0069607b-aa90-49fd-9766-4368bcd168c4.txt

import os
import sys
with open(sys.argv[0]) as f:
    code = f.read() # read the code of this file ASAP, for logging
import uuid
import time
from dataclasses import dataclass
from pathlib import Path

import torch
from torch import nn
import torch.nn.functional as F
import torch.distributed as dist
import torch._inductor.config as config
from torch.nn.parallel import DistributedDataParallel as DDP
# Use of FlexAttention contributed by @KoszarskyB
from torch.nn.attention.flex_attention import BlockMask, flex_attention

# -----------------------------------------------------------------------------
# Muon optimizer

def zeropower_via_svd(G, steps=None):
    U, S, V = G.svd()
    return U @ V.T

@torch.compile
def zeropower_via_newtonschulz5(G, steps=10, eps=1e-7):
    """
    Newton-Schulz iteration to compute the zeroth power / orthogonalization of G. We opt to use a
    quintic iteration whose coefficients are selected to maximize the slope at zero. For the purpose
    of minimizing steps, it turns out to be empirically effective to keep increasing the slope at
    zero even beyond the point where the iteration no longer converges all the way to one everywhere
    on the interval. This iteration therefore does not produce UV^T but rather something like US'V^T
    where S' is diagonal with S_{ii}' ~ Uniform(0.5, 1.5), which turns out not to hurt model
    performance at all relative to UV^T, where USV^T = G is the SVD.
    """
    assert len(G.shape) == 2
    a, b, c = (3.4445, -4.7750,  2.0315)
    X = G.bfloat16()
    X /= (X.norm() + eps) # ensure top singular value <= 1
    if G.size(0) > G.size(1):
        X = X.T
    for _ in range(steps):
        A = X @ X.T
        B = b * A + c * A @ A # adapted from suggestion by @jxbz, @leloykun, and @YouJiacheng
        X = a * X + B @ X
    if G.size(0) > G.size(1):
        X = X.T
    return X

zeropower_backends = dict(svd=zeropower_via_svd, newtonschulz5=zeropower_via_newtonschulz5)

class Muon(torch.optim.Optimizer):
    """
    Muon - MomentUm Orthogonalized by Newton-schulz

    Muon internally runs standard SGD-momentum, and then performs an orthogonalization post-
    processing step, in which each 2D parameter's update is replaced with the nearest orthogonal
    matrix. To efficiently orthogonalize each update, we use a Newton-Schulz iteration, which has
    the advantage that it can be stably run in bfloat16 on the GPU.

    Some warnings:
    - This optimizer assumes that all parameters passed in are 2D.
    - It should not be used for the embedding layer, the final fully connected layer, or any {0,1}-D
    parameters; those should all be optimized by a standard method (e.g., AdamW).
    - To use it with 4D convolutional filters, it works well to just flatten their last 3 dimensions.
    - We believe it is unlikely to work well for training with small batch size.
    - We believe it may not work well for finetuning pretrained models, but we haven't tested this.
    - We have not yet tried this optimizer for training scenarios larger than NanoGPT (124M).

    Arguments:
        lr: The learning rate used by the internal SGD.
        momentum: The momentum used by the internal SGD.
        nesterov: Whether to use Nesterov-style momentum in the internal SGD. (recommended)
        backend: The chosen backend for the orthogonalization step. (recommended: 'newtonschulz5')
        backend_steps: The number of iteration steps to use in the backend, if it is iterative.
    """
    def __init__(self, params, lr=0.02, momentum=0.95, nesterov=True,
                 backend='newtonschulz5', backend_steps=5):
        self.num_process = int(os.environ['WORLD_SIZE'])
        self.rank = int(os.environ["RANK"])
        defaults = dict(lr=lr, momentum=momentum, nesterov=nesterov, backend=backend, backend_steps=backend_steps)
        params: "list[torch.Tensor]" = list(params)
        assert all(isinstance(p, torch.Tensor) for p in params)
        sizes = {p.numel() for p in params}
        param_groups = [
            {
                "params": [p for p in params if p.numel() == size],
                "update_buffer": [
                    torch.empty(size, device="cuda", dtype=torch.bfloat16)
                    for _ in range(self.num_process)
                ],
            }
            for size in sizes
        ]
        super().__init__(param_groups, defaults)

    def step(self):
        for group in self.param_groups:
            lr: float = group["lr"]
            momentum: float = group["momentum"]
            nesterov: bool = group["nesterov"]
            zeropower_backend = zeropower_backends[group["backend"]]
            backend_steps: int = group["backend_steps"]
            update_buffers: "list[torch.Tensor]" = group["update_buffer"]
            # generate weight updates in distributed fashion
            params: "list[torch.Tensor]" = group["params"]
            assert len(params) % self.num_process == 0
            handle = None
            params_world = None
            def update_prev():
                if params_world is None:
                    return
                assert handle is not None
                handle.wait()
                for p_world, g_world in zip(params_world, update_buffers):
                    p_world.data.add_(
                        g_world.view_as(p_world),
                        alpha=-lr * max(1, p_world.size(0) / p_world.size(1)) ** 0.5,
                    )
            for base_i in range(len(params))[::self.num_process]:
                p = params[base_i + self.rank]
                g = p.grad
                assert g is not None
                state = self.state[p] 
                if "momentum_buffer" not in state:
                    state["momentum_buffer"] = torch.zeros_like(g)
                buf: torch.Tensor = state["momentum_buffer"]
                buf.lerp_(g, 1 - momentum)
                g = g.lerp_(buf, momentum) if nesterov else buf
                g = zeropower_backend(g, steps=backend_steps).flatten()
                update_prev()
                handle = dist.all_gather(update_buffers, g, async_op=True)
                params_world = params[base_i : base_i + self.num_process]
            update_prev()


# -----------------------------------------------------------------------------
# PyTorch nn.Module definitions for the GPT-2 model

def norm(x):
    return F.rms_norm(x, (x.size(-1),))

class CastedLinear(nn.Linear):

    def __init__(self, in_features, out_features):
        super().__init__(in_features, out_features, bias=False)

    def forward(self, x):
        return F.linear(x, self.weight.to(x.dtype))

class Rotary(torch.nn.Module):

    def __init__(self, dim, base=10000):
        super().__init__()
        self.register_buffer('inv_freq', (1 / base) ** (torch.arange(0, dim, 2) / dim))
        self.seq_len_cached = None
        self.cos_cached = None
        self.sin_cached = None

    def forward(self, x):
        seq_len = x.shape[1]
        if seq_len != self.seq_len_cached:
            t = torch.arange(seq_len, device=x.device)
            freqs = torch.outer(t, self.inv_freq)
            self.seq_len_cached = seq_len
            self.cos_cached = freqs.cos()
            self.sin_cached = freqs.sin()
        cos, sin = self.cos_cached[None, :, None, :], self.sin_cached[None, :, None, :]
        # apply_rotary_emb(x, cos, sin)
        x1, x2 = x.chunk(2, dim=3)
        y1 = x1 * cos + x2 * sin
        y2 = x1 * (-sin) + x2 * cos
        return torch.cat((y1, y2), 3).type_as(x)

class CausalSelfAttention(nn.Module):

    def __init__(self, dim, n_head):
        super().__init__()
        assert dim % n_head == 0
        self.n_head = n_head
        self.c_q = CastedLinear(dim, dim)
        self.c_k = CastedLinear(dim, dim)
        self.c_v = CastedLinear(dim, dim)
        # value residual lambda
        self.lambdas = nn.Parameter(torch.tensor([0.5, 0.5])) # @Grad62304977
        # rotary embeddings
        self.rotary = Rotary(dim // n_head) # dim // n_head = head_dim
        # output projection
        self.c_proj = CastedLinear(dim, dim)
        self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977

    def forward(self, x: torch.Tensor, vi: torch.Tensor, block_mask: BlockMask) -> torch.Tensor:
        B, T = x.size(0), x.size(1) # batch size, sequence length
        assert B == 1, "Must use batch size = 1 for FlexAttention"
        q: torch.Tensor = self.c_q(x).view(B, T, self.n_head, -1)
        k: torch.Tensor = self.c_k(x).view(B, T, self.n_head, -1)
        v: torch.Tensor = self.c_v(x).view(B, T, self.n_head, -1)
        v = self.lambdas[0] * v + self.lambdas[1] * vi.view_as(v) # @Grad62304977
        q, k = norm(q), norm(k) # QK norm suggested by @Grad62304977
        q, k = self.rotary(q), self.rotary(k)
        y = flex_attention(q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2), block_mask=block_mask)
        y = y.transpose(1, 2).contiguous().view_as(x) # re-assemble all head outputs side by side
        y = self.c_proj(y)
        return y

class MLP(nn.Module):

    def __init__(self, dim: int):
        super().__init__()
        self.c_fc   = CastedLinear(dim, 4 * dim)
        self.c_proj = CastedLinear(4 * dim, dim)
        self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        x = self.c_fc(x)
        x = F.relu(x).square() # https://arxiv.org/abs/2109.08668v2; ~1-2% better than GELU; suggested by @SKYLINEZ007 and @Grad62304977
        x = self.c_proj(x)
        return x

class Block(nn.Module):

    def __init__(self, config):
        super().__init__()
        self.attn = CausalSelfAttention(config.n_embd, config.n_head)
        self.mlp = MLP(config.n_embd)
        self.lambdas = nn.Parameter(torch.tensor([1., 0.]))

    def forward(self, x: torch.Tensor, vi: torch.Tensor, x0: torch.Tensor, block_mask: BlockMask) -> torch.Tensor:
        x = self.lambdas[0] * x + self.lambdas[1] * x0
        x = x + self.attn(norm(x), vi, block_mask)
        x = x + self.mlp(norm(x))
        return x

# -----------------------------------------------------------------------------
# The main GPT-2 model

@dataclass
class GPTConfig:
    vocab_size : int = 50304
    n_layer : int = 12
    n_head : int = 6 # head dim 128 suggested by @Grad62304977
    n_embd : int = 768
    lm_head_softcap : int = 30

class GPT(nn.Module):

    def __init__(self, config: GPTConfig):
        super().__init__()
        self.n_layer = config.n_layer
        self.lm_head_softcap = config.lm_head_softcap

        # U-net design by @brendanh0gan
        self.num_encoder_layers = config.n_layer // 2 # Half of the layers for encoder
        self.num_decoder_layers = config.n_layer - self.num_encoder_layers # Remaining for decoder
        # Add learnable skip connection weights for decoder layers
        self.skip_weights = nn.Parameter(torch.ones(self.num_decoder_layers))

        self.transformer = nn.ModuleDict(dict(
            wte = nn.Embedding(config.vocab_size, config.n_embd),
            # token value embeddings by @KoszarskyB - inspired by @Grad62304977's value residual learning
            # U-net structure on token value embeddings by @leloykun
            vte = nn.Embedding(config.vocab_size, config.n_embd*self.num_encoder_layers),
            h = nn.ModuleList([Block(config) for _ in range(config.n_layer)]),
        ))
        self.lm_head = CastedLinear(config.n_embd, config.vocab_size)
        self.lm_head.weight.data.zero_() # @Grad62304977

    def forward(self, idx: torch.Tensor, target: torch.Tensor, sliding_window: torch.Tensor) -> torch.Tensor:
        BLOCK_SIZE = 128
        assert idx.ndim == 1
        docs = (idx == 50256).cumsum(0)
        docs_low = docs.reshape(-1, BLOCK_SIZE)[:, 0].contiguous()
        docs_high = docs.reshape(-1, BLOCK_SIZE)[:, -1].contiguous()
        def document_sliding_window_causal(b, h, q_idx, kv_idx):
            causal_mask = q_idx >= kv_idx
            document_mask = docs[q_idx] == docs[kv_idx]
            window_mask = q_idx - kv_idx < sliding_window
            return causal_mask & document_mask & window_mask

        S = len(idx)
        def create_sliding_window_causal_mask(S: int, sliding_window: torch.Tensor):
            kv_idx = block_idx = torch.arange(S // BLOCK_SIZE, dtype=torch.int32, device="cuda")
            q_idx = block_idx[:, None]
            causal_mask = q_idx >= kv_idx
            document_mask = (docs_low[q_idx] <= docs_high[kv_idx]) & (docs_low[kv_idx] <= docs_high[q_idx])
            window_mask = q_idx - kv_idx < ((sliding_window + BLOCK_SIZE - 1) // BLOCK_SIZE)
            dense_mask = causal_mask & document_mask & window_mask
            dense_mask = dense_mask.to(torch.int32)
            num_blocks = dense_mask.sum(dim=-1).to(torch.int32)
            indices = torch.argsort(dense_mask, dim=-1, descending=True, stable=True).to(torch.int32)
            num_blocks = num_blocks[None, None, :].contiguous()
            indices = indices[None, None, :].contiguous()
            return BlockMask.from_kv_blocks(num_blocks, indices, BLOCK_SIZE=BLOCK_SIZE, mask_mod=document_sliding_window_causal)
        block_mask = create_sliding_window_causal_mask(S, sliding_window)

        # forward the GPT model itself
        x = self.transformer.wte(idx[None]) # token embeddings of shape (b, t, n_embd)
        x = norm(x) # @Grad62304977
        x0 = x
        vi = self.transformer.vte(idx[None]).chunk(self.num_encoder_layers, dim=-1)

        # Store outputs for U-Net skip connections
        skip_connections = []
        # Encoder pass - process only the first half of the blocks
        for i in range(self.num_encoder_layers):
            x = self.transformer.h[i](x, vi[i], x0, block_mask)
            skip_connections.append(x)
        # Decoder pass - process the remaining blocks with weighted skip connections
        for i in range(self.num_decoder_layers):
            x = x + self.skip_weights[i] * skip_connections.pop()
            # U-net structure on token value embeddings by @leloykun
            x = self.transformer.h[self.num_encoder_layers + i](x, vi[self.num_encoder_layers-1-i], x0, block_mask)

        x = norm(x)
        logits = self.lm_head(x)
        logits = self.lm_head_softcap * torch.tanh(logits / self.lm_head_softcap) # @Grad62304977
        logits = logits.float()
        loss = F.cross_entropy(logits.view(-1, logits.size(-1)), target.view(-1))
        return loss

# -----------------------------------------------------------------------------
# Our own simple Distributed Data Loader

def _peek_data_shard(file: Path):
    # only reads the header, returns header data
    # header is 256 int32
    header = torch.from_file(f"{file}", False, 256, dtype=torch.int32)
    assert header[0] == 20240520, "magic number mismatch in the data .bin file"
    assert header[1] == 1, "unsupported version"
    return int(header[2]) # number of tokens (claimed)

def _load_data_shard(file: Path, ntok: int):
    with file.open("rb") as f:
        tokens = torch.empty(ntok, dtype=torch.uint16, pin_memory=True)
        f.seek(256 * 4)
        nbytes = f.readinto(tokens.numpy())
        assert nbytes == 2 * ntok, "number of tokens read does not match header?"
    return tokens

class DistributedDataLoader:
    def __init__(self, filename_pattern, T, process_rank, num_processes):
        self.process_rank = process_rank
        self.num_processes = num_processes
        self.T = T

        # glob files that match the pattern
        self.files = sorted(Path.cwd().glob(filename_pattern))
        assert len(self.files) > 0, f"did not find any files that match the pattern {filename_pattern}"

        # load and validate all data shards, count number of tokens in total
        self.ntoks = [_peek_data_shard(file) for file in self.files]
        assert min(self.ntoks) >= num_processes * T + 1
        self.ntok_total = sum(self.ntoks)

        self.reset()

    def reset(self):
        self.current_shard = -1
        self.advance()

    def advance(self): # advance to next data shard
        self.current_shard = (self.current_shard + 1) % len(self.files)
        self.current_position = self.process_rank * self.T
        self.tokens = _load_data_shard(self.files[self.current_shard], self.ntoks[self.current_shard])

    def next_batch(self):
        batch_size = self.T * self.num_processes
        buf = self.tokens[self.current_position:self.current_position+self.T+1]
        # host side async is sufficient;
        # no performance improvement was observed when introducing a separate stream.
        x = buf[:-1].to(device="cuda", dtype=torch.int32, non_blocking=True) # inputs
        y = buf[1:].to(device="cuda", dtype=torch.int64, non_blocking=True) # targets
        # advance current position and load next shard if necessary
        self.current_position += batch_size
        if self.current_position + batch_size + 1 >= len(self.tokens):
            self.advance()
        return x, y

# -----------------------------------------------------------------------------
# int main

@dataclass
class Hyperparameters:
    # data hyperparams
    input_bin : str = 'data/fineweb10B/fineweb_train_*.bin' # input .bin to train on
    input_val_bin : str = 'data/fineweb10B/fineweb_val_*.bin' # input .bin to eval validation loss on
    # optimization hyperparams
    batch_size : int = 8 # batch size, in sequences, across all devices
    sequence_length : int = 64*1024 # sequence length, in tokens
    num_iterations : int = 1480 # number of iterations to run
    warmup_iters : int = 0
    cooldown_iters : int = 600 # number of iterations of linear warmup/cooldown for triangular or trapezoidal schedule
    weight_decay : float = 0
    # evaluation and logging hyperparams
    val_loss_every : int = 125 # every how many steps to evaluate val loss? 0 for only at the end
    val_tokens : int = 10485760 # how many tokens of validation data? it's important to keep this fixed for consistent comparisons
    save_every : int = 0 # every how many steps to save the checkpoint? 0 for only at the end
args = Hyperparameters()

# set up DDP (distributed data parallel). torchrun sets this env variable
assert torch.cuda.is_available()
dist.init_process_group(backend='nccl')
ddp_rank = int(os.environ['RANK'])
ddp_local_rank = int(os.environ['LOCAL_RANK'])
ddp_world_size = int(os.environ['WORLD_SIZE'])
device = f'cuda:{ddp_local_rank}'
torch.cuda.set_device(device)
print(f"using device: {device}")
master_process = (ddp_rank == 0) # this process will do logging, checkpointing etc.

# begin logging
logfile = None
if master_process:
    run_id = str(uuid.uuid4())
    logdir = 'logs/%s/' % run_id
    # os.makedirs(logdir, exist_ok=True)
    logfile = 'logs/%s.txt' % run_id
    # create the log file
    with open(logfile, "w") as f:
        # begin the log by printing this file (the Python code)
        f.write(code)
        f.write('='*100 + '\n')
def print0(s, logonly=False):
    if master_process:
        with open(logfile, "a") as f:
            if not logonly:
                print(s)
            f.write(s+'\n')
# log information about the hardware/software environment this is running on
# and print the full `nvidia-smi` to file
print0(f"Running pytorch {torch.version.__version__} compiled for CUDA {torch.version.cuda}\nnvidia-smi:")
import subprocess
result = subprocess.run(['nvidia-smi'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
print0(f'{result.stdout}', logonly=True)
print0('='*100, logonly=True)

# convenience variables
T = args.sequence_length
# calculate the number of steps to take in the val loop.
assert args.val_tokens % (T * ddp_world_size) == 0
val_steps = args.val_tokens // (T * ddp_world_size)
# calculate the steps of gradient accumulation required to attain the desired global batch size.
assert args.batch_size % (ddp_world_size) == 0
train_accumulation_steps = args.batch_size // ddp_world_size
assert train_accumulation_steps == 1

# load tokens
train_loader = DistributedDataLoader(args.input_bin, T, ddp_rank, ddp_world_size)
val_loader = DistributedDataLoader(args.input_val_bin, T, ddp_rank, ddp_world_size)
print0(f"Training DataLoader: total number of tokens: {train_loader.ntok_total} across {len(train_loader.files)} files")
print0(f"Validation DataLoader: total number of tokens: {val_loader.ntok_total} across {len(val_loader.files)} files")
print0('='*100, logonly=True)
x, y = train_loader.next_batch()

# there are only 50257 unique GPT-2 tokens; we extend to nearest multiple of 128 for efficiency. suggested to me by @Grad62304977.
# this originates from Karpathy's experiments.
num_vocab = 50304
model = GPT(GPTConfig(vocab_size=num_vocab, n_layer=12, n_head=6, n_embd=768))
model = model.cuda().bfloat16()
for m in model.modules():
    if isinstance(m, CastedLinear):
        m.float()
if hasattr(config, "coordinate_descent_tuning"):
    config.coordinate_descent_tuning = True # suggested by @Chillee
model = torch.compile(model)
# here we wrap model into DDP container
model = DDP(model, device_ids=[ddp_local_rank])
raw_model = model.module # always contains the "raw" unwrapped model

# init the optimizer(s)
optimizer1 = torch.optim.Adam([raw_model.transformer.wte.weight, raw_model.transformer.vte.weight], lr=0.6, betas=(0.8, 0.95), fused=True)
optimizer2 = torch.optim.Adam([raw_model.lm_head.weight], lr=0.008, betas=(0.8, 0.95), fused=True)
params = list(raw_model.transformer.h.parameters())
matrix_params = [p for p in params if p.ndim == 2]
scalar_params = [p for p in params if p.ndim < 2] + [raw_model.skip_weights]
optimizer3 = Muon(matrix_params, lr=0.05, momentum=0.95)
optimizer4 = torch.optim.Adam(scalar_params, lr=0.04, betas=(0.8, 0.95), fused=True)
optimizers = [optimizer1, optimizer2, optimizer3, optimizer4]
# learning rate decay scheduler (linear warmup and cooldown)
def get_lr(it):
    assert it <= args.num_iterations
    # 1) linear warmup for warmup_iters steps
    if it < args.warmup_iters:
        return (it+1) / args.warmup_iters
    # 2) constant lr for a while
    elif it < args.num_iterations - args.cooldown_iters:
        return 1.0
    # 3) linear cooldown
    else:
        decay_ratio = (args.num_iterations - it) / args.cooldown_iters
        return decay_ratio
schedulers = [torch.optim.lr_scheduler.LambdaLR(opt, get_lr) for opt in optimizers]

sliding_window_size = torch.tensor(64, dtype=torch.int32, device="cuda")
sw_size_prev = 64
# Start training loop
training_time_ms = 0
# start the clock
torch.cuda.synchronize()
t0 = time.perf_counter()
# begin training
for step in range(args.num_iterations + 1):
    last_step = (step == args.num_iterations)
    # This effectively ignores timing first 10 steps, which are slower for weird reasons.
    # Alternately, and slightly more correctly in terms of benchmarking, we could do 10
    # steps with dummy data first, and then re-initialize the model and reset the loader.
    if step == 10:
        training_time_ms = 0
        t0 = time.perf_counter()
    timed_steps = float('nan') if step <= 11 else (step - 10) + 1 # <= 11 to avoid bug in val

    # Set the sliding window size for the current step, in chunks of 64. By @fernbear.bsky.social
    sw_size =  64 * int((64 + (1792 - 64) * step / args.num_iterations) // 64)
    if sw_size != sw_size_prev:
        sliding_window_size.copy_(sw_size, non_blocking=True)
        sw_size_prev = sw_size

    # once in a while evaluate the validation dataset
    if (last_step or (args.val_loss_every > 0 and step % args.val_loss_every == 0)):
        # stop the clock
        torch.cuda.synchronize()
        training_time_ms += 1000 * (time.perf_counter() - t0)
        # run validation batches
        model.eval()
        val_loader.reset()
        val_loss = 0.0
        for _ in range(val_steps):
            with torch.no_grad():
                x_val, y_val = val_loader.next_batch()
                val_loss += model(x_val, y_val, sliding_window=sliding_window_size)
        dist.all_reduce(val_loss, op=dist.ReduceOp.AVG)
        val_loss /= val_steps
        # log val loss to console and to logfile
        print0(f'step:{step}/{args.num_iterations} val_loss:{val_loss:.4f} train_time:{training_time_ms:.0f}ms step_avg:{training_time_ms/(timed_steps-1):.2f}ms')
        # start the clock again
        torch.cuda.synchronize()
        t0 = time.perf_counter()

    if master_process and (last_step or (args.save_every > 0 and step % args.save_every == 0)):
        # stop the clock
        torch.cuda.synchronize()
        training_time_ms += 1000 * (time.perf_counter() - t0)
        # save the state of the training process
        log = dict(step=step, code=code, model=raw_model.state_dict(), optimizers=[opt.state_dict() for opt in optimizers])
        # torch.save(log, 'logs/%s/state_step%06d.pt' % (run_id, step))
        # start the clock again
        torch.cuda.synchronize()
        t0 = time.perf_counter()

    # bit confusing: we want to make sure to eval on 0th iteration
    # but also after the very last iteration. so we loop for step <= num_iterations
    # instead of just < num_iterations (one extra due to <=), only to do
    # the validation/sampling one last time, and then we break right here as we're done.
    if last_step:
        break

    # --------------- TRAINING SECTION BEGIN -----------------
    model.train()
    loss = model(x, y, sliding_window=sliding_window_size)
    loss.backward()
    del loss
    # advance the dataset for the next batch
    x, y = train_loader.next_batch()
    # momentum warmup for Muon
    frac = min(step/300, 1)
    for group in optimizer3.param_groups:
        group['momentum'] = (1 - frac) * 0.85 + frac * 0.95
    # step the optimizers and schedulers
    for opt, sched in zip(optimizers, schedulers):
        opt.step()
        sched.step()
    # null the gradients
    model.zero_grad(set_to_none=True)
    # --------------- TRAINING SECTION END -------------------
    # everything that follows now is just diagnostics, prints, logging, etc.
    approx_time = training_time_ms + 1000 * (time.perf_counter() - t0)
    print0(f"step:{step+1}/{args.num_iterations} train_time:{approx_time:.0f}ms step_avg:{approx_time/timed_steps:.2f}ms")

if master_process:
    print(f"peak memory consumption: {torch.cuda.max_memory_allocated() // 1024 // 1024} MiB")

# -------------------------------------------------------------------------
# clean up nice
dist.destroy_process_group()
====================================================================================================
Running pytorch 2.6.0.dev20241203+cu124 compiled for CUDA 12.4
nvidia-smi:
Sun Dec  8 12:24:25 2024       
+---------------------------------------------------------------------------------------+
| NVIDIA-SMI 535.129.03             Driver Version: 535.129.03   CUDA Version: 12.6     |
|-----------------------------------------+----------------------+----------------------+
| GPU  Name                 Persistence-M | Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp   Perf          Pwr:Usage/Cap |         Memory-Usage | GPU-Util  Compute M. |
|                                         |                      |               MIG M. |
|=========================================+======================+======================|
|   0  NVIDIA H100 80GB HBM3          On  | 00000000:65:02.0 Off |                    0 |
| N/A   36C    P0              74W / 700W |      7MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   1  NVIDIA H100 80GB HBM3          On  | 00000000:67:02.0 Off |                    0 |
| N/A   45C    P0             128W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   2  NVIDIA H100 80GB HBM3          On  | 00000000:69:02.0 Off |                    0 |
| N/A   45C    P0             123W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   3  NVIDIA H100 80GB HBM3          On  | 00000000:6B:02.0 Off |                    0 |
| N/A   39C    P0             114W / 700W |    533MiB / 81559MiB |      1%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   4  NVIDIA H100 80GB HBM3          On  | 00000000:6F:02.0 Off |                    0 |
| N/A   38C    P0             117W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   5  NVIDIA H100 80GB HBM3          On  | 00000000:71:02.0 Off |                    0 |
| N/A   44C    P0             121W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   6  NVIDIA H100 80GB HBM3          On  | 00000000:73:02.0 Off |                    0 |
| N/A   45C    P0             126W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   7  NVIDIA H100 80GB HBM3          On  | 00000000:75:02.0 Off |                    0 |
| N/A   38C    P0             123W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
                                                                                         
+---------------------------------------------------------------------------------------+
| Processes:                                                                            |
|  GPU   GI   CI        PID   Type   Process name                            GPU Memory |
|        ID   ID                                                             Usage      |
|=======================================================================================|
+---------------------------------------------------------------------------------------+

====================================================================================================
Training DataLoader: total number of tokens: 3200000000 across 32 files
Validation DataLoader: total number of tokens: 100000000 across 1 files
====================================================================================================
step:0/1480 val_loss:10.8258 train_time:0ms step_avg:nanms
step:1/1480 train_time:23764ms step_avg:nanms
step:2/1480 train_time:23887ms step_avg:nanms
step:3/1480 train_time:24025ms step_avg:nanms
step:4/1480 train_time:24167ms step_avg:nanms
step:5/1480 train_time:24307ms step_avg:nanms
step:6/1480 train_time:24448ms step_avg:nanms
step:7/1480 train_time:24589ms step_avg:nanms
step:8/1480 train_time:24731ms step_avg:nanms
step:9/1480 train_time:24877ms step_avg:nanms
step:10/1480 train_time:25020ms step_avg:nanms
step:11/1480 train_time:143ms step_avg:nanms
step:12/1480 train_time:285ms step_avg:nanms
step:13/1480 train_time:427ms step_avg:142.18ms
step:14/1480 train_time:567ms step_avg:141.70ms
step:15/1480 train_time:708ms step_avg:141.61ms
step:16/1480 train_time:852ms step_avg:142.01ms
step:17/1480 train_time:996ms step_avg:142.34ms
step:18/1480 train_time:1141ms step_avg:142.57ms
step:19/1480 train_time:1284ms step_avg:142.64ms
step:20/1480 train_time:1426ms step_avg:142.59ms
step:21/1480 train_time:1567ms step_avg:142.45ms
step:22/1480 train_time:1708ms step_avg:142.30ms
step:23/1480 train_time:1852ms step_avg:142.46ms
step:24/1480 train_time:1996ms step_avg:142.54ms
step:25/1480 train_time:2140ms step_avg:142.69ms
step:26/1480 train_time:2283ms step_avg:142.69ms
step:27/1480 train_time:2425ms step_avg:142.68ms
step:28/1480 train_time:2567ms step_avg:142.61ms
step:29/1480 train_time:2707ms step_avg:142.47ms
step:30/1480 train_time:2851ms step_avg:142.53ms
step:31/1480 train_time:2994ms step_avg:142.58ms
step:32/1480 train_time:3137ms step_avg:142.58ms
step:33/1480 train_time:3280ms step_avg:142.63ms
step:34/1480 train_time:3424ms step_avg:142.67ms
step:35/1480 train_time:3566ms step_avg:142.65ms
step:36/1480 train_time:3707ms step_avg:142.58ms
step:37/1480 train_time:3851ms step_avg:142.62ms
step:38/1480 train_time:3994ms step_avg:142.63ms
step:39/1480 train_time:4136ms step_avg:142.63ms
step:40/1480 train_time:4281ms step_avg:142.70ms
step:41/1480 train_time:4425ms step_avg:142.74ms
step:42/1480 train_time:4568ms step_avg:142.75ms
step:43/1480 train_time:4709ms step_avg:142.70ms
step:44/1480 train_time:4852ms step_avg:142.70ms
step:45/1480 train_time:4994ms step_avg:142.69ms
step:46/1480 train_time:5136ms step_avg:142.68ms
step:47/1480 train_time:5281ms step_avg:142.72ms
step:48/1480 train_time:5426ms step_avg:142.78ms
step:49/1480 train_time:5568ms step_avg:142.77ms
step:50/1480 train_time:5710ms step_avg:142.75ms
step:51/1480 train_time:5853ms step_avg:142.76ms
step:52/1480 train_time:5995ms step_avg:142.74ms
step:53/1480 train_time:6137ms step_avg:142.73ms
step:54/1480 train_time:6279ms step_avg:142.72ms
step:55/1480 train_time:6424ms step_avg:142.75ms
step:56/1480 train_time:6568ms step_avg:142.78ms
step:57/1480 train_time:6709ms step_avg:142.75ms
step:58/1480 train_time:6853ms step_avg:142.77ms
step:59/1480 train_time:6997ms step_avg:142.80ms
step:60/1480 train_time:7141ms step_avg:142.82ms
step:61/1480 train_time:7284ms step_avg:142.83ms
step:62/1480 train_time:7427ms step_avg:142.82ms
step:63/1480 train_time:7568ms step_avg:142.79ms
step:64/1480 train_time:7710ms step_avg:142.77ms
step:65/1480 train_time:7853ms step_avg:142.77ms
step:66/1480 train_time:7995ms step_avg:142.77ms
step:67/1480 train_time:8139ms step_avg:142.79ms
step:68/1480 train_time:8282ms step_avg:142.79ms
step:69/1480 train_time:8425ms step_avg:142.80ms
step:70/1480 train_time:8568ms step_avg:142.80ms
step:71/1480 train_time:8710ms step_avg:142.79ms
step:72/1480 train_time:8854ms step_avg:142.80ms
step:73/1480 train_time:8996ms step_avg:142.79ms
step:74/1480 train_time:9140ms step_avg:142.82ms
step:75/1480 train_time:9284ms step_avg:142.83ms
step:76/1480 train_time:9426ms step_avg:142.82ms
step:77/1480 train_time:9568ms step_avg:142.81ms
step:78/1480 train_time:9710ms step_avg:142.79ms
step:79/1480 train_time:9853ms step_avg:142.80ms
step:80/1480 train_time:9996ms step_avg:142.80ms
step:81/1480 train_time:10140ms step_avg:142.82ms
step:82/1480 train_time:10283ms step_avg:142.82ms
step:83/1480 train_time:10426ms step_avg:142.82ms
step:84/1480 train_time:10567ms step_avg:142.80ms
step:85/1480 train_time:10709ms step_avg:142.79ms
step:86/1480 train_time:10851ms step_avg:142.78ms
step:87/1480 train_time:10993ms step_avg:142.77ms
step:88/1480 train_time:11135ms step_avg:142.75ms
step:89/1480 train_time:11278ms step_avg:142.75ms
step:90/1480 train_time:11423ms step_avg:142.78ms
step:91/1480 train_time:11565ms step_avg:142.78ms
step:92/1480 train_time:11708ms step_avg:142.78ms
step:93/1480 train_time:11850ms step_avg:142.77ms
step:94/1480 train_time:11993ms step_avg:142.77ms
step:95/1480 train_time:12134ms step_avg:142.75ms
step:96/1480 train_time:12276ms step_avg:142.74ms
step:97/1480 train_time:12420ms step_avg:142.76ms
step:98/1480 train_time:12564ms step_avg:142.78ms
step:99/1480 train_time:12706ms step_avg:142.76ms
step:100/1480 train_time:12850ms step_avg:142.77ms
step:101/1480 train_time:12991ms step_avg:142.75ms
step:102/1480 train_time:13132ms step_avg:142.74ms
step:103/1480 train_time:13274ms step_avg:142.73ms
step:104/1480 train_time:13419ms step_avg:142.76ms
step:105/1480 train_time:13563ms step_avg:142.77ms
step:106/1480 train_time:13706ms step_avg:142.78ms
step:107/1480 train_time:13849ms step_avg:142.78ms
step:108/1480 train_time:13990ms step_avg:142.76ms
step:109/1480 train_time:14132ms step_avg:142.75ms
step:110/1480 train_time:14273ms step_avg:142.73ms
step:111/1480 train_time:14420ms step_avg:142.78ms
step:112/1480 train_time:14568ms step_avg:142.82ms
step:113/1480 train_time:14716ms step_avg:142.87ms
step:114/1480 train_time:14864ms step_avg:142.92ms
step:115/1480 train_time:15011ms step_avg:142.96ms
step:116/1480 train_time:15159ms step_avg:143.01ms
step:117/1480 train_time:15306ms step_avg:143.04ms
step:118/1480 train_time:15452ms step_avg:143.08ms
step:119/1480 train_time:15599ms step_avg:143.11ms
step:120/1480 train_time:15747ms step_avg:143.16ms
step:121/1480 train_time:15893ms step_avg:143.18ms
step:122/1480 train_time:16041ms step_avg:143.22ms
step:123/1480 train_time:16188ms step_avg:143.26ms
step:124/1480 train_time:16334ms step_avg:143.28ms
step:125/1480 train_time:16482ms step_avg:143.32ms
step:125/1480 val_loss:4.4041 train_time:16539ms step_avg:143.82ms
step:126/1480 train_time:16635ms step_avg:143.40ms
step:127/1480 train_time:16783ms step_avg:143.44ms
step:128/1480 train_time:16931ms step_avg:143.48ms
step:129/1480 train_time:17077ms step_avg:143.50ms
step:130/1480 train_time:17223ms step_avg:143.53ms
step:131/1480 train_time:17369ms step_avg:143.55ms
step:132/1480 train_time:17515ms step_avg:143.57ms
step:133/1480 train_time:17666ms step_avg:143.62ms
step:134/1480 train_time:17813ms step_avg:143.65ms
step:135/1480 train_time:17959ms step_avg:143.67ms
step:136/1480 train_time:18106ms step_avg:143.70ms
step:137/1480 train_time:18253ms step_avg:143.72ms
step:138/1480 train_time:18398ms step_avg:143.73ms
step:139/1480 train_time:18546ms step_avg:143.77ms
step:140/1480 train_time:18694ms step_avg:143.80ms
step:141/1480 train_time:18841ms step_avg:143.82ms
step:142/1480 train_time:18989ms step_avg:143.85ms
step:143/1480 train_time:19135ms step_avg:143.87ms
step:144/1480 train_time:19284ms step_avg:143.91ms
step:145/1480 train_time:19429ms step_avg:143.92ms
step:146/1480 train_time:19575ms step_avg:143.94ms
step:147/1480 train_time:19724ms step_avg:143.97ms
step:148/1480 train_time:19871ms step_avg:143.99ms
step:149/1480 train_time:20016ms step_avg:144.00ms
step:150/1480 train_time:20163ms step_avg:144.02ms
step:151/1480 train_time:20310ms step_avg:144.04ms
step:152/1480 train_time:20456ms step_avg:144.06ms
step:153/1480 train_time:20602ms step_avg:144.07ms
step:154/1480 train_time:20750ms step_avg:144.10ms
step:155/1480 train_time:20896ms step_avg:144.11ms
step:156/1480 train_time:21043ms step_avg:144.13ms
step:157/1480 train_time:21193ms step_avg:144.17ms
step:158/1480 train_time:21337ms step_avg:144.17ms
step:159/1480 train_time:21485ms step_avg:144.20ms
step:160/1480 train_time:21631ms step_avg:144.21ms
step:161/1480 train_time:21778ms step_avg:144.23ms
step:162/1480 train_time:21925ms step_avg:144.24ms
step:163/1480 train_time:22072ms step_avg:144.26ms
step:164/1480 train_time:22217ms step_avg:144.27ms
step:165/1480 train_time:22366ms step_avg:144.29ms
step:166/1480 train_time:22512ms step_avg:144.31ms
step:167/1480 train_time:22659ms step_avg:144.32ms
step:168/1480 train_time:22806ms step_avg:144.34ms
step:169/1480 train_time:22952ms step_avg:144.35ms
step:170/1480 train_time:23098ms step_avg:144.36ms
step:171/1480 train_time:23245ms step_avg:144.38ms
step:172/1480 train_time:23393ms step_avg:144.40ms
step:173/1480 train_time:23540ms step_avg:144.41ms
step:174/1480 train_time:23686ms step_avg:144.43ms
step:175/1480 train_time:23832ms step_avg:144.44ms
step:176/1480 train_time:23980ms step_avg:144.46ms
step:177/1480 train_time:24126ms step_avg:144.47ms
step:178/1480 train_time:24273ms step_avg:144.48ms
step:179/1480 train_time:24418ms step_avg:144.48ms
step:180/1480 train_time:24565ms step_avg:144.50ms
step:181/1480 train_time:24712ms step_avg:144.51ms
step:182/1480 train_time:24858ms step_avg:144.52ms
step:183/1480 train_time:25005ms step_avg:144.54ms
step:184/1480 train_time:25153ms step_avg:144.56ms
step:185/1480 train_time:25300ms step_avg:144.57ms
step:186/1480 train_time:25447ms step_avg:144.58ms
step:187/1480 train_time:25593ms step_avg:144.60ms
step:188/1480 train_time:25741ms step_avg:144.61ms
step:189/1480 train_time:25888ms step_avg:144.63ms
step:190/1480 train_time:26034ms step_avg:144.63ms
step:191/1480 train_time:26181ms step_avg:144.64ms
step:192/1480 train_time:26327ms step_avg:144.66ms
step:193/1480 train_time:26474ms step_avg:144.67ms
step:194/1480 train_time:26620ms step_avg:144.67ms
step:195/1480 train_time:26767ms step_avg:144.69ms
step:196/1480 train_time:26913ms step_avg:144.69ms
step:197/1480 train_time:27061ms step_avg:144.71ms
step:198/1480 train_time:27207ms step_avg:144.72ms
step:199/1480 train_time:27354ms step_avg:144.73ms
step:200/1480 train_time:27501ms step_avg:144.74ms
step:201/1480 train_time:27648ms step_avg:144.75ms
step:202/1480 train_time:27794ms step_avg:144.76ms
step:203/1480 train_time:27940ms step_avg:144.77ms
step:204/1480 train_time:28087ms step_avg:144.78ms
step:205/1480 train_time:28234ms step_avg:144.79ms
step:206/1480 train_time:28379ms step_avg:144.79ms
step:207/1480 train_time:28527ms step_avg:144.80ms
step:208/1480 train_time:28674ms step_avg:144.82ms
step:209/1480 train_time:28820ms step_avg:144.82ms
step:210/1480 train_time:28967ms step_avg:144.83ms
step:211/1480 train_time:29113ms step_avg:144.84ms
step:212/1480 train_time:29260ms step_avg:144.85ms
step:213/1480 train_time:29407ms step_avg:144.86ms
step:214/1480 train_time:29554ms step_avg:144.87ms
step:215/1480 train_time:29700ms step_avg:144.88ms
step:216/1480 train_time:29848ms step_avg:144.89ms
step:217/1480 train_time:29994ms step_avg:144.90ms
step:218/1480 train_time:30141ms step_avg:144.91ms
step:219/1480 train_time:30290ms step_avg:144.93ms
step:220/1480 train_time:30435ms step_avg:144.93ms
step:221/1480 train_time:30584ms step_avg:144.95ms
step:222/1480 train_time:30734ms step_avg:144.97ms
step:223/1480 train_time:30883ms step_avg:144.99ms
step:224/1480 train_time:31033ms step_avg:145.01ms
step:225/1480 train_time:31184ms step_avg:145.04ms
step:226/1480 train_time:31333ms step_avg:145.06ms
step:227/1480 train_time:31484ms step_avg:145.09ms
step:228/1480 train_time:31634ms step_avg:145.11ms
step:229/1480 train_time:31786ms step_avg:145.14ms
step:230/1480 train_time:31936ms step_avg:145.16ms
step:231/1480 train_time:32087ms step_avg:145.19ms
step:232/1480 train_time:32237ms step_avg:145.21ms
step:233/1480 train_time:32387ms step_avg:145.23ms
step:234/1480 train_time:32539ms step_avg:145.26ms
step:235/1480 train_time:32690ms step_avg:145.29ms
step:236/1480 train_time:32839ms step_avg:145.30ms
step:237/1480 train_time:32989ms step_avg:145.33ms
step:238/1480 train_time:33139ms step_avg:145.35ms
step:239/1480 train_time:33290ms step_avg:145.37ms
step:240/1480 train_time:33438ms step_avg:145.38ms
step:241/1480 train_time:33590ms step_avg:145.41ms
step:242/1480 train_time:33740ms step_avg:145.43ms
step:243/1480 train_time:33891ms step_avg:145.45ms
step:244/1480 train_time:34041ms step_avg:145.47ms
step:245/1480 train_time:34192ms step_avg:145.50ms
step:246/1480 train_time:34342ms step_avg:145.52ms
step:247/1480 train_time:34491ms step_avg:145.53ms
step:248/1480 train_time:34641ms step_avg:145.55ms
step:249/1480 train_time:34791ms step_avg:145.57ms
step:250/1480 train_time:34942ms step_avg:145.59ms
step:250/1480 val_loss:3.9878 train_time:35000ms step_avg:145.84ms
step:251/1480 train_time:35098ms step_avg:145.64ms
step:252/1480 train_time:35248ms step_avg:145.65ms
step:253/1480 train_time:35398ms step_avg:145.67ms
step:254/1480 train_time:35547ms step_avg:145.69ms
step:255/1480 train_time:35697ms step_avg:145.70ms
step:256/1480 train_time:35846ms step_avg:145.71ms
step:257/1480 train_time:35996ms step_avg:145.73ms
step:258/1480 train_time:36147ms step_avg:145.76ms
step:259/1480 train_time:36299ms step_avg:145.78ms
step:260/1480 train_time:36450ms step_avg:145.80ms
step:261/1480 train_time:36600ms step_avg:145.82ms
step:262/1480 train_time:36749ms step_avg:145.83ms
step:263/1480 train_time:36900ms step_avg:145.85ms
step:264/1480 train_time:37050ms step_avg:145.86ms
step:265/1480 train_time:37204ms step_avg:145.90ms
step:266/1480 train_time:37352ms step_avg:145.91ms
step:267/1480 train_time:37503ms step_avg:145.93ms
step:268/1480 train_time:37652ms step_avg:145.94ms
step:269/1480 train_time:37803ms step_avg:145.96ms
step:270/1480 train_time:37951ms step_avg:145.96ms
step:271/1480 train_time:38101ms step_avg:145.98ms
step:272/1480 train_time:38251ms step_avg:146.00ms
step:273/1480 train_time:38404ms step_avg:146.02ms
step:274/1480 train_time:38553ms step_avg:146.03ms
step:275/1480 train_time:38704ms step_avg:146.05ms
step:276/1480 train_time:38855ms step_avg:146.07ms
step:277/1480 train_time:39006ms step_avg:146.09ms
step:278/1480 train_time:39155ms step_avg:146.10ms
step:279/1480 train_time:39308ms step_avg:146.13ms
step:280/1480 train_time:39457ms step_avg:146.14ms
step:281/1480 train_time:39607ms step_avg:146.15ms
step:282/1480 train_time:39758ms step_avg:146.17ms
step:283/1480 train_time:39909ms step_avg:146.19ms
step:284/1480 train_time:40061ms step_avg:146.21ms
step:285/1480 train_time:40211ms step_avg:146.22ms
step:286/1480 train_time:40362ms step_avg:146.24ms
step:287/1480 train_time:40512ms step_avg:146.25ms
step:288/1480 train_time:40663ms step_avg:146.27ms
step:289/1480 train_time:40813ms step_avg:146.28ms
step:290/1480 train_time:40964ms step_avg:146.30ms
step:291/1480 train_time:41115ms step_avg:146.32ms
step:292/1480 train_time:41266ms step_avg:146.33ms
step:293/1480 train_time:41416ms step_avg:146.35ms
step:294/1480 train_time:41567ms step_avg:146.36ms
step:295/1480 train_time:41715ms step_avg:146.37ms
step:296/1480 train_time:41866ms step_avg:146.39ms
step:297/1480 train_time:42016ms step_avg:146.40ms
step:298/1480 train_time:42167ms step_avg:146.41ms
step:299/1480 train_time:42317ms step_avg:146.43ms
step:300/1480 train_time:42469ms step_avg:146.44ms
step:301/1480 train_time:42619ms step_avg:146.46ms
step:302/1480 train_time:42768ms step_avg:146.47ms
step:303/1480 train_time:42918ms step_avg:146.48ms
step:304/1480 train_time:43069ms step_avg:146.49ms
step:305/1480 train_time:43220ms step_avg:146.51ms
step:306/1480 train_time:43370ms step_avg:146.52ms
step:307/1480 train_time:43521ms step_avg:146.54ms
step:308/1480 train_time:43672ms step_avg:146.55ms
step:309/1480 train_time:43822ms step_avg:146.56ms
step:310/1480 train_time:43972ms step_avg:146.57ms
step:311/1480 train_time:44122ms step_avg:146.59ms
step:312/1480 train_time:44274ms step_avg:146.60ms
step:313/1480 train_time:44424ms step_avg:146.61ms
step:314/1480 train_time:44573ms step_avg:146.62ms
step:315/1480 train_time:44724ms step_avg:146.64ms
step:316/1480 train_time:44873ms step_avg:146.64ms
step:317/1480 train_time:45024ms step_avg:146.66ms
step:318/1480 train_time:45174ms step_avg:146.67ms
step:319/1480 train_time:45325ms step_avg:146.68ms
step:320/1480 train_time:45476ms step_avg:146.70ms
step:321/1480 train_time:45627ms step_avg:146.71ms
step:322/1480 train_time:45779ms step_avg:146.73ms
step:323/1480 train_time:45929ms step_avg:146.74ms
step:324/1480 train_time:46081ms step_avg:146.75ms
step:325/1480 train_time:46231ms step_avg:146.77ms
step:326/1480 train_time:46382ms step_avg:146.78ms
step:327/1480 train_time:46531ms step_avg:146.79ms
step:328/1480 train_time:46682ms step_avg:146.80ms
step:329/1480 train_time:46833ms step_avg:146.81ms
step:330/1480 train_time:46985ms step_avg:146.83ms
step:331/1480 train_time:47138ms step_avg:146.85ms
step:332/1480 train_time:47291ms step_avg:146.87ms
step:333/1480 train_time:47443ms step_avg:146.88ms
step:334/1480 train_time:47597ms step_avg:146.90ms
step:335/1480 train_time:47750ms step_avg:146.92ms
step:336/1480 train_time:47905ms step_avg:146.95ms
step:337/1480 train_time:48060ms step_avg:146.97ms
step:338/1480 train_time:48213ms step_avg:146.99ms
step:339/1480 train_time:48366ms step_avg:147.01ms
step:340/1480 train_time:48521ms step_avg:147.03ms
step:341/1480 train_time:48675ms step_avg:147.05ms
step:342/1480 train_time:48827ms step_avg:147.07ms
step:343/1480 train_time:48982ms step_avg:147.09ms
step:344/1480 train_time:49136ms step_avg:147.12ms
step:345/1480 train_time:49290ms step_avg:147.14ms
step:346/1480 train_time:49444ms step_avg:147.16ms
step:347/1480 train_time:49599ms step_avg:147.18ms
step:348/1480 train_time:49752ms step_avg:147.20ms
step:349/1480 train_time:49906ms step_avg:147.22ms
step:350/1480 train_time:50060ms step_avg:147.23ms
step:351/1480 train_time:50213ms step_avg:147.25ms
step:352/1480 train_time:50367ms step_avg:147.27ms
step:353/1480 train_time:50521ms step_avg:147.29ms
step:354/1480 train_time:50676ms step_avg:147.31ms
step:355/1480 train_time:50829ms step_avg:147.33ms
step:356/1480 train_time:50983ms step_avg:147.35ms
step:357/1480 train_time:51138ms step_avg:147.37ms
step:358/1480 train_time:51291ms step_avg:147.39ms
step:359/1480 train_time:51445ms step_avg:147.41ms
step:360/1480 train_time:51602ms step_avg:147.43ms
step:361/1480 train_time:51757ms step_avg:147.46ms
step:362/1480 train_time:51910ms step_avg:147.47ms
step:363/1480 train_time:52064ms step_avg:147.49ms
step:364/1480 train_time:52219ms step_avg:147.51ms
step:365/1480 train_time:52373ms step_avg:147.53ms
step:366/1480 train_time:52527ms step_avg:147.55ms
step:367/1480 train_time:52681ms step_avg:147.57ms
step:368/1480 train_time:52835ms step_avg:147.58ms
step:369/1480 train_time:52987ms step_avg:147.60ms
step:370/1480 train_time:53140ms step_avg:147.61ms
step:371/1480 train_time:53293ms step_avg:147.63ms
step:372/1480 train_time:53446ms step_avg:147.64ms
step:373/1480 train_time:53600ms step_avg:147.66ms
step:374/1480 train_time:53755ms step_avg:147.68ms
step:375/1480 train_time:53908ms step_avg:147.69ms
step:375/1480 val_loss:3.8060 train_time:53968ms step_avg:147.86ms
step:376/1480 train_time:54065ms step_avg:147.72ms
step:377/1480 train_time:54219ms step_avg:147.74ms
step:378/1480 train_time:54372ms step_avg:147.75ms
step:379/1480 train_time:54525ms step_avg:147.76ms
step:380/1480 train_time:54677ms step_avg:147.78ms
step:381/1480 train_time:54829ms step_avg:147.79ms
step:382/1480 train_time:54983ms step_avg:147.81ms
step:383/1480 train_time:55139ms step_avg:147.82ms
step:384/1480 train_time:55294ms step_avg:147.84ms
step:385/1480 train_time:55446ms step_avg:147.86ms
step:386/1480 train_time:55600ms step_avg:147.87ms
step:387/1480 train_time:55753ms step_avg:147.89ms
step:388/1480 train_time:55907ms step_avg:147.90ms
step:389/1480 train_time:56060ms step_avg:147.92ms
step:390/1480 train_time:56215ms step_avg:147.93ms
step:391/1480 train_time:56369ms step_avg:147.95ms
step:392/1480 train_time:56522ms step_avg:147.96ms
step:393/1480 train_time:56677ms step_avg:147.98ms
step:394/1480 train_time:56830ms step_avg:147.99ms
step:395/1480 train_time:56983ms step_avg:148.01ms
step:396/1480 train_time:57137ms step_avg:148.02ms
step:397/1480 train_time:57291ms step_avg:148.04ms
step:398/1480 train_time:57445ms step_avg:148.05ms
step:399/1480 train_time:57600ms step_avg:148.07ms
step:400/1480 train_time:57755ms step_avg:148.09ms
step:401/1480 train_time:57907ms step_avg:148.10ms
step:402/1480 train_time:58060ms step_avg:148.11ms
step:403/1480 train_time:58215ms step_avg:148.13ms
step:404/1480 train_time:58368ms step_avg:148.14ms
step:405/1480 train_time:58522ms step_avg:148.16ms
step:406/1480 train_time:58675ms step_avg:148.17ms
step:407/1480 train_time:58828ms step_avg:148.18ms
step:408/1480 train_time:58982ms step_avg:148.19ms
step:409/1480 train_time:59135ms step_avg:148.21ms
step:410/1480 train_time:59289ms step_avg:148.22ms
step:411/1480 train_time:59443ms step_avg:148.24ms
step:412/1480 train_time:59599ms step_avg:148.26ms
step:413/1480 train_time:59753ms step_avg:148.27ms
step:414/1480 train_time:59906ms step_avg:148.28ms
step:415/1480 train_time:60062ms step_avg:148.30ms
step:416/1480 train_time:60217ms step_avg:148.32ms
step:417/1480 train_time:60370ms step_avg:148.33ms
step:418/1480 train_time:60523ms step_avg:148.34ms
step:419/1480 train_time:60678ms step_avg:148.36ms
step:420/1480 train_time:60832ms step_avg:148.37ms
step:421/1480 train_time:60985ms step_avg:148.38ms
step:422/1480 train_time:61138ms step_avg:148.39ms
step:423/1480 train_time:61293ms step_avg:148.41ms
step:424/1480 train_time:61446ms step_avg:148.42ms
step:425/1480 train_time:61601ms step_avg:148.44ms
step:426/1480 train_time:61754ms step_avg:148.45ms
step:427/1480 train_time:61906ms step_avg:148.46ms
step:428/1480 train_time:62060ms step_avg:148.47ms
step:429/1480 train_time:62215ms step_avg:148.48ms
step:430/1480 train_time:62369ms step_avg:148.50ms
step:431/1480 train_time:62523ms step_avg:148.51ms
step:432/1480 train_time:62677ms step_avg:148.52ms
step:433/1480 train_time:62830ms step_avg:148.53ms
step:434/1480 train_time:62983ms step_avg:148.54ms
step:435/1480 train_time:63136ms step_avg:148.56ms
step:436/1480 train_time:63292ms step_avg:148.57ms
step:437/1480 train_time:63445ms step_avg:148.58ms
step:438/1480 train_time:63599ms step_avg:148.60ms
step:439/1480 train_time:63754ms step_avg:148.61ms
step:440/1480 train_time:63907ms step_avg:148.62ms
step:441/1480 train_time:64063ms step_avg:148.64ms
step:442/1480 train_time:64220ms step_avg:148.66ms
step:443/1480 train_time:64378ms step_avg:148.68ms
step:444/1480 train_time:64534ms step_avg:148.70ms
step:445/1480 train_time:64691ms step_avg:148.72ms
step:446/1480 train_time:64847ms step_avg:148.73ms
step:447/1480 train_time:65003ms step_avg:148.75ms
step:448/1480 train_time:65160ms step_avg:148.77ms
step:449/1480 train_time:65318ms step_avg:148.79ms
step:450/1480 train_time:65476ms step_avg:148.81ms
step:451/1480 train_time:65634ms step_avg:148.83ms
step:452/1480 train_time:65791ms step_avg:148.85ms
step:453/1480 train_time:65947ms step_avg:148.86ms
step:454/1480 train_time:66103ms step_avg:148.88ms
step:455/1480 train_time:66260ms step_avg:148.90ms
step:456/1480 train_time:66418ms step_avg:148.92ms
step:457/1480 train_time:66575ms step_avg:148.94ms
step:458/1480 train_time:66732ms step_avg:148.95ms
step:459/1480 train_time:66890ms step_avg:148.97ms
step:460/1480 train_time:67046ms step_avg:148.99ms
step:461/1480 train_time:67204ms step_avg:149.01ms
step:462/1480 train_time:67360ms step_avg:149.03ms
step:463/1480 train_time:67518ms step_avg:149.05ms
step:464/1480 train_time:67676ms step_avg:149.07ms
step:465/1480 train_time:67833ms step_avg:149.08ms
step:466/1480 train_time:67990ms step_avg:149.10ms
step:467/1480 train_time:68146ms step_avg:149.12ms
step:468/1480 train_time:68302ms step_avg:149.13ms
step:469/1480 train_time:68459ms step_avg:149.15ms
step:470/1480 train_time:68617ms step_avg:149.17ms
step:471/1480 train_time:68772ms step_avg:149.18ms
step:472/1480 train_time:68928ms step_avg:149.20ms
step:473/1480 train_time:69085ms step_avg:149.21ms
step:474/1480 train_time:69240ms step_avg:149.22ms
step:475/1480 train_time:69398ms step_avg:149.24ms
step:476/1480 train_time:69555ms step_avg:149.26ms
step:477/1480 train_time:69712ms step_avg:149.28ms
step:478/1480 train_time:69867ms step_avg:149.29ms
step:479/1480 train_time:70023ms step_avg:149.30ms
step:480/1480 train_time:70180ms step_avg:149.32ms
step:481/1480 train_time:70338ms step_avg:149.34ms
step:482/1480 train_time:70495ms step_avg:149.35ms
step:483/1480 train_time:70651ms step_avg:149.37ms
step:484/1480 train_time:70807ms step_avg:149.38ms
step:485/1480 train_time:70964ms step_avg:149.40ms
step:486/1480 train_time:71120ms step_avg:149.41ms
step:487/1480 train_time:71279ms step_avg:149.43ms
step:488/1480 train_time:71435ms step_avg:149.45ms
step:489/1480 train_time:71591ms step_avg:149.46ms
step:490/1480 train_time:71747ms step_avg:149.47ms
step:491/1480 train_time:71904ms step_avg:149.49ms
step:492/1480 train_time:72060ms step_avg:149.50ms
step:493/1480 train_time:72218ms step_avg:149.52ms
step:494/1480 train_time:72377ms step_avg:149.54ms
step:495/1480 train_time:72535ms step_avg:149.56ms
step:496/1480 train_time:72694ms step_avg:149.58ms
step:497/1480 train_time:72851ms step_avg:149.59ms
step:498/1480 train_time:73006ms step_avg:149.60ms
step:499/1480 train_time:73163ms step_avg:149.62ms
step:500/1480 train_time:73321ms step_avg:149.63ms
step:500/1480 val_loss:3.6861 train_time:73384ms step_avg:149.76ms
step:501/1480 train_time:73483ms step_avg:149.66ms
step:502/1480 train_time:73641ms step_avg:149.68ms
step:503/1480 train_time:73796ms step_avg:149.69ms
step:504/1480 train_time:73952ms step_avg:149.70ms
step:505/1480 train_time:74107ms step_avg:149.71ms
step:506/1480 train_time:74265ms step_avg:149.73ms
step:507/1480 train_time:74423ms step_avg:149.74ms
step:508/1480 train_time:74581ms step_avg:149.76ms
step:509/1480 train_time:74739ms step_avg:149.78ms
step:510/1480 train_time:74894ms step_avg:149.79ms
step:511/1480 train_time:75051ms step_avg:149.80ms
step:512/1480 train_time:75209ms step_avg:149.82ms
step:513/1480 train_time:75366ms step_avg:149.83ms
step:514/1480 train_time:75524ms step_avg:149.85ms
step:515/1480 train_time:75682ms step_avg:149.86ms
step:516/1480 train_time:75840ms step_avg:149.88ms
step:517/1480 train_time:75996ms step_avg:149.89ms
step:518/1480 train_time:76152ms step_avg:149.91ms
step:519/1480 train_time:76309ms step_avg:149.92ms
step:520/1480 train_time:76468ms step_avg:149.94ms
step:521/1480 train_time:76627ms step_avg:149.95ms
step:522/1480 train_time:76785ms step_avg:149.97ms
step:523/1480 train_time:76943ms step_avg:149.99ms
step:524/1480 train_time:77100ms step_avg:150.00ms
step:525/1480 train_time:77255ms step_avg:150.01ms
step:526/1480 train_time:77411ms step_avg:150.02ms
step:527/1480 train_time:77567ms step_avg:150.03ms
step:528/1480 train_time:77724ms step_avg:150.05ms
step:529/1480 train_time:77880ms step_avg:150.06ms
step:530/1480 train_time:78038ms step_avg:150.07ms
step:531/1480 train_time:78195ms step_avg:150.09ms
step:532/1480 train_time:78351ms step_avg:150.10ms
step:533/1480 train_time:78507ms step_avg:150.11ms
step:534/1480 train_time:78663ms step_avg:150.12ms
step:535/1480 train_time:78821ms step_avg:150.13ms
step:536/1480 train_time:78977ms step_avg:150.15ms
step:537/1480 train_time:79134ms step_avg:150.16ms
step:538/1480 train_time:79294ms step_avg:150.18ms
step:539/1480 train_time:79451ms step_avg:150.19ms
step:540/1480 train_time:79608ms step_avg:150.20ms
step:541/1480 train_time:79765ms step_avg:150.22ms
step:542/1480 train_time:79920ms step_avg:150.23ms
step:543/1480 train_time:80076ms step_avg:150.24ms
step:544/1480 train_time:80232ms step_avg:150.25ms
step:545/1480 train_time:80390ms step_avg:150.26ms
step:546/1480 train_time:80547ms step_avg:150.27ms
step:547/1480 train_time:80705ms step_avg:150.29ms
step:548/1480 train_time:80864ms step_avg:150.30ms
step:549/1480 train_time:81020ms step_avg:150.31ms
step:550/1480 train_time:81177ms step_avg:150.33ms
step:551/1480 train_time:81334ms step_avg:150.34ms
step:552/1480 train_time:81492ms step_avg:150.35ms
step:553/1480 train_time:81651ms step_avg:150.37ms
step:554/1480 train_time:81810ms step_avg:150.39ms
step:555/1480 train_time:81970ms step_avg:150.40ms
step:556/1480 train_time:82130ms step_avg:150.42ms
step:557/1480 train_time:82291ms step_avg:150.44ms
step:558/1480 train_time:82452ms step_avg:150.46ms
step:559/1480 train_time:82610ms step_avg:150.47ms
step:560/1480 train_time:82770ms step_avg:150.49ms
step:561/1480 train_time:82930ms step_avg:150.51ms
step:562/1480 train_time:83090ms step_avg:150.53ms
step:563/1480 train_time:83249ms step_avg:150.54ms
step:564/1480 train_time:83409ms step_avg:150.56ms
step:565/1480 train_time:83569ms step_avg:150.57ms
step:566/1480 train_time:83729ms step_avg:150.59ms
step:567/1480 train_time:83889ms step_avg:150.61ms
step:568/1480 train_time:84048ms step_avg:150.62ms
step:569/1480 train_time:84207ms step_avg:150.64ms
step:570/1480 train_time:84368ms step_avg:150.66ms
step:571/1480 train_time:84529ms step_avg:150.68ms
step:572/1480 train_time:84690ms step_avg:150.69ms
step:573/1480 train_time:84852ms step_avg:150.71ms
step:574/1480 train_time:85013ms step_avg:150.73ms
step:575/1480 train_time:85172ms step_avg:150.75ms
step:576/1480 train_time:85330ms step_avg:150.76ms
step:577/1480 train_time:85490ms step_avg:150.78ms
step:578/1480 train_time:85650ms step_avg:150.79ms
step:579/1480 train_time:85809ms step_avg:150.81ms
step:580/1480 train_time:85970ms step_avg:150.82ms
step:581/1480 train_time:86130ms step_avg:150.84ms
step:582/1480 train_time:86291ms step_avg:150.86ms
step:583/1480 train_time:86449ms step_avg:150.87ms
step:584/1480 train_time:86608ms step_avg:150.89ms
step:585/1480 train_time:86768ms step_avg:150.90ms
step:586/1480 train_time:86927ms step_avg:150.92ms
step:587/1480 train_time:87088ms step_avg:150.93ms
step:588/1480 train_time:87246ms step_avg:150.95ms
step:589/1480 train_time:87407ms step_avg:150.96ms
step:590/1480 train_time:87569ms step_avg:150.98ms
step:591/1480 train_time:87728ms step_avg:150.99ms
step:592/1480 train_time:87889ms step_avg:151.01ms
step:593/1480 train_time:88050ms step_avg:151.03ms
step:594/1480 train_time:88210ms step_avg:151.05ms
step:595/1480 train_time:88371ms step_avg:151.06ms
step:596/1480 train_time:88532ms step_avg:151.08ms
step:597/1480 train_time:88692ms step_avg:151.09ms
step:598/1480 train_time:88849ms step_avg:151.10ms
step:599/1480 train_time:89008ms step_avg:151.12ms
step:600/1480 train_time:89169ms step_avg:151.13ms
step:601/1480 train_time:89329ms step_avg:151.15ms
step:602/1480 train_time:89490ms step_avg:151.17ms
step:603/1480 train_time:89651ms step_avg:151.18ms
step:604/1480 train_time:89809ms step_avg:151.19ms
step:605/1480 train_time:89970ms step_avg:151.21ms
step:606/1480 train_time:90132ms step_avg:151.23ms
step:607/1480 train_time:90295ms step_avg:151.25ms
step:608/1480 train_time:90456ms step_avg:151.27ms
step:609/1480 train_time:90615ms step_avg:151.28ms
step:610/1480 train_time:90773ms step_avg:151.29ms
step:611/1480 train_time:90932ms step_avg:151.30ms
step:612/1480 train_time:91092ms step_avg:151.31ms
step:613/1480 train_time:91252ms step_avg:151.33ms
step:614/1480 train_time:91411ms step_avg:151.34ms
step:615/1480 train_time:91570ms step_avg:151.36ms
step:616/1480 train_time:91729ms step_avg:151.37ms
step:617/1480 train_time:91890ms step_avg:151.38ms
step:618/1480 train_time:92050ms step_avg:151.40ms
step:619/1480 train_time:92210ms step_avg:151.41ms
step:620/1480 train_time:92371ms step_avg:151.43ms
step:621/1480 train_time:92530ms step_avg:151.44ms
step:622/1480 train_time:92690ms step_avg:151.45ms
step:623/1480 train_time:92851ms step_avg:151.47ms
step:624/1480 train_time:93010ms step_avg:151.48ms
step:625/1480 train_time:93170ms step_avg:151.50ms
step:625/1480 val_loss:3.6068 train_time:93233ms step_avg:151.60ms
step:626/1480 train_time:93334ms step_avg:151.52ms
step:627/1480 train_time:93495ms step_avg:151.53ms
step:628/1480 train_time:93654ms step_avg:151.54ms
step:629/1480 train_time:93812ms step_avg:151.55ms
step:630/1480 train_time:93970ms step_avg:151.57ms
step:631/1480 train_time:94128ms step_avg:151.57ms
step:632/1480 train_time:94289ms step_avg:151.59ms
step:633/1480 train_time:94450ms step_avg:151.60ms
step:634/1480 train_time:94609ms step_avg:151.62ms
step:635/1480 train_time:94766ms step_avg:151.63ms
step:636/1480 train_time:94924ms step_avg:151.64ms
step:637/1480 train_time:95083ms step_avg:151.65ms
step:638/1480 train_time:95240ms step_avg:151.66ms
step:639/1480 train_time:95398ms step_avg:151.67ms
step:640/1480 train_time:95558ms step_avg:151.68ms
step:641/1480 train_time:95718ms step_avg:151.69ms
step:642/1480 train_time:95879ms step_avg:151.71ms
step:643/1480 train_time:96039ms step_avg:151.72ms
step:644/1480 train_time:96197ms step_avg:151.73ms
step:645/1480 train_time:96358ms step_avg:151.74ms
step:646/1480 train_time:96517ms step_avg:151.76ms
step:647/1480 train_time:96676ms step_avg:151.77ms
step:648/1480 train_time:96839ms step_avg:151.79ms
step:649/1480 train_time:96998ms step_avg:151.80ms
step:650/1480 train_time:97159ms step_avg:151.81ms
step:651/1480 train_time:97320ms step_avg:151.83ms
step:652/1480 train_time:97479ms step_avg:151.84ms
step:653/1480 train_time:97639ms step_avg:151.85ms
step:654/1480 train_time:97798ms step_avg:151.86ms
step:655/1480 train_time:97959ms step_avg:151.87ms
step:656/1480 train_time:98119ms step_avg:151.89ms
step:657/1480 train_time:98280ms step_avg:151.90ms
step:658/1480 train_time:98439ms step_avg:151.91ms
step:659/1480 train_time:98599ms step_avg:151.92ms
step:660/1480 train_time:98761ms step_avg:151.94ms
step:661/1480 train_time:98921ms step_avg:151.95ms
step:662/1480 train_time:99081ms step_avg:151.97ms
step:663/1480 train_time:99240ms step_avg:151.98ms
step:664/1480 train_time:99402ms step_avg:151.99ms
step:665/1480 train_time:99564ms step_avg:152.01ms
step:666/1480 train_time:99723ms step_avg:152.02ms
step:667/1480 train_time:99885ms step_avg:152.03ms
step:668/1480 train_time:100047ms step_avg:152.05ms
step:669/1480 train_time:100209ms step_avg:152.06ms
step:670/1480 train_time:100369ms step_avg:152.07ms
step:671/1480 train_time:100529ms step_avg:152.09ms
step:672/1480 train_time:100693ms step_avg:152.10ms
step:673/1480 train_time:100855ms step_avg:152.12ms
step:674/1480 train_time:101019ms step_avg:152.14ms
step:675/1480 train_time:101182ms step_avg:152.15ms
step:676/1480 train_time:101344ms step_avg:152.17ms
step:677/1480 train_time:101504ms step_avg:152.18ms
step:678/1480 train_time:101663ms step_avg:152.19ms
step:679/1480 train_time:101826ms step_avg:152.21ms
step:680/1480 train_time:101988ms step_avg:152.22ms
step:681/1480 train_time:102148ms step_avg:152.23ms
step:682/1480 train_time:102310ms step_avg:152.25ms
step:683/1480 train_time:102472ms step_avg:152.26ms
step:684/1480 train_time:102632ms step_avg:152.27ms
step:685/1480 train_time:102797ms step_avg:152.29ms
step:686/1480 train_time:102959ms step_avg:152.31ms
step:687/1480 train_time:103121ms step_avg:152.32ms
step:688/1480 train_time:103284ms step_avg:152.34ms
step:689/1480 train_time:103446ms step_avg:152.35ms
step:690/1480 train_time:103608ms step_avg:152.36ms
step:691/1480 train_time:103767ms step_avg:152.37ms
step:692/1480 train_time:103927ms step_avg:152.38ms
step:693/1480 train_time:104087ms step_avg:152.40ms
step:694/1480 train_time:104248ms step_avg:152.41ms
step:695/1480 train_time:104408ms step_avg:152.42ms
step:696/1480 train_time:104566ms step_avg:152.43ms
step:697/1480 train_time:104728ms step_avg:152.44ms
step:698/1480 train_time:104889ms step_avg:152.45ms
step:699/1480 train_time:105053ms step_avg:152.47ms
step:700/1480 train_time:105216ms step_avg:152.49ms
step:701/1480 train_time:105377ms step_avg:152.50ms
step:702/1480 train_time:105539ms step_avg:152.51ms
step:703/1480 train_time:105700ms step_avg:152.53ms
step:704/1480 train_time:105861ms step_avg:152.54ms
step:705/1480 train_time:106023ms step_avg:152.55ms
step:706/1480 train_time:106186ms step_avg:152.57ms
step:707/1480 train_time:106346ms step_avg:152.58ms
step:708/1480 train_time:106507ms step_avg:152.59ms
step:709/1480 train_time:106667ms step_avg:152.60ms
step:710/1480 train_time:106828ms step_avg:152.61ms
step:711/1480 train_time:106991ms step_avg:152.63ms
step:712/1480 train_time:107158ms step_avg:152.65ms
step:713/1480 train_time:107322ms step_avg:152.66ms
step:714/1480 train_time:107482ms step_avg:152.67ms
step:715/1480 train_time:107642ms step_avg:152.68ms
step:716/1480 train_time:107801ms step_avg:152.69ms
step:717/1480 train_time:107964ms step_avg:152.71ms
step:718/1480 train_time:108123ms step_avg:152.72ms
step:719/1480 train_time:108283ms step_avg:152.73ms
step:720/1480 train_time:108447ms step_avg:152.74ms
step:721/1480 train_time:108608ms step_avg:152.75ms
step:722/1480 train_time:108768ms step_avg:152.76ms
step:723/1480 train_time:108929ms step_avg:152.78ms
step:724/1480 train_time:109091ms step_avg:152.79ms
step:725/1480 train_time:109258ms step_avg:152.81ms
step:726/1480 train_time:109421ms step_avg:152.82ms
step:727/1480 train_time:109584ms step_avg:152.84ms
step:728/1480 train_time:109744ms step_avg:152.85ms
step:729/1480 train_time:109904ms step_avg:152.86ms
step:730/1480 train_time:110066ms step_avg:152.87ms
step:731/1480 train_time:110227ms step_avg:152.88ms
step:732/1480 train_time:110387ms step_avg:152.89ms
step:733/1480 train_time:110549ms step_avg:152.90ms
step:734/1480 train_time:110710ms step_avg:152.91ms
step:735/1480 train_time:110871ms step_avg:152.93ms
step:736/1480 train_time:111032ms step_avg:152.94ms
step:737/1480 train_time:111194ms step_avg:152.95ms
step:738/1480 train_time:111357ms step_avg:152.96ms
step:739/1480 train_time:111520ms step_avg:152.98ms
step:740/1480 train_time:111685ms step_avg:152.99ms
step:741/1480 train_time:111848ms step_avg:153.01ms
step:742/1480 train_time:112008ms step_avg:153.02ms
step:743/1480 train_time:112168ms step_avg:153.03ms
step:744/1480 train_time:112331ms step_avg:153.04ms
step:745/1480 train_time:112498ms step_avg:153.06ms
step:746/1480 train_time:112660ms step_avg:153.07ms
step:747/1480 train_time:112823ms step_avg:153.08ms
step:748/1480 train_time:112987ms step_avg:153.10ms
step:749/1480 train_time:113152ms step_avg:153.11ms
step:750/1480 train_time:113311ms step_avg:153.12ms
step:750/1480 val_loss:3.5482 train_time:113377ms step_avg:153.21ms
step:751/1480 train_time:113479ms step_avg:153.14ms
step:752/1480 train_time:113640ms step_avg:153.15ms
step:753/1480 train_time:113801ms step_avg:153.16ms
step:754/1480 train_time:113960ms step_avg:153.17ms
step:755/1480 train_time:114121ms step_avg:153.18ms
step:756/1480 train_time:114282ms step_avg:153.19ms
step:757/1480 train_time:114447ms step_avg:153.21ms
step:758/1480 train_time:114606ms step_avg:153.22ms
step:759/1480 train_time:114769ms step_avg:153.23ms
step:760/1480 train_time:114931ms step_avg:153.24ms
step:761/1480 train_time:115094ms step_avg:153.25ms
step:762/1480 train_time:115257ms step_avg:153.27ms
step:763/1480 train_time:115420ms step_avg:153.28ms
step:764/1480 train_time:115581ms step_avg:153.29ms
step:765/1480 train_time:115742ms step_avg:153.30ms
step:766/1480 train_time:115905ms step_avg:153.31ms
step:767/1480 train_time:116066ms step_avg:153.32ms
step:768/1480 train_time:116227ms step_avg:153.33ms
step:769/1480 train_time:116390ms step_avg:153.35ms
step:770/1480 train_time:116553ms step_avg:153.36ms
step:771/1480 train_time:116718ms step_avg:153.37ms
step:772/1480 train_time:116880ms step_avg:153.39ms
step:773/1480 train_time:117043ms step_avg:153.40ms
step:774/1480 train_time:117206ms step_avg:153.41ms
step:775/1480 train_time:117368ms step_avg:153.42ms
step:776/1480 train_time:117532ms step_avg:153.44ms
step:777/1480 train_time:117699ms step_avg:153.45ms
step:778/1480 train_time:117861ms step_avg:153.47ms
step:779/1480 train_time:118023ms step_avg:153.48ms
step:780/1480 train_time:118187ms step_avg:153.49ms
step:781/1480 train_time:118349ms step_avg:153.50ms
step:782/1480 train_time:118513ms step_avg:153.51ms
step:783/1480 train_time:118676ms step_avg:153.53ms
step:784/1480 train_time:118840ms step_avg:153.54ms
step:785/1480 train_time:119002ms step_avg:153.55ms
step:786/1480 train_time:119167ms step_avg:153.57ms
step:787/1480 train_time:119329ms step_avg:153.58ms
step:788/1480 train_time:119494ms step_avg:153.59ms
step:789/1480 train_time:119657ms step_avg:153.60ms
step:790/1480 train_time:119822ms step_avg:153.62ms
step:791/1480 train_time:119988ms step_avg:153.63ms
step:792/1480 train_time:120152ms step_avg:153.65ms
step:793/1480 train_time:120316ms step_avg:153.66ms
step:794/1480 train_time:120480ms step_avg:153.67ms
step:795/1480 train_time:120645ms step_avg:153.69ms
step:796/1480 train_time:120811ms step_avg:153.70ms
step:797/1480 train_time:120976ms step_avg:153.72ms
step:798/1480 train_time:121141ms step_avg:153.73ms
step:799/1480 train_time:121307ms step_avg:153.75ms
step:800/1480 train_time:121471ms step_avg:153.76ms
step:801/1480 train_time:121636ms step_avg:153.77ms
step:802/1480 train_time:121805ms step_avg:153.79ms
step:803/1480 train_time:121967ms step_avg:153.80ms
step:804/1480 train_time:122129ms step_avg:153.81ms
step:805/1480 train_time:122295ms step_avg:153.83ms
step:806/1480 train_time:122457ms step_avg:153.84ms
step:807/1480 train_time:122620ms step_avg:153.85ms
step:808/1480 train_time:122784ms step_avg:153.86ms
step:809/1480 train_time:122946ms step_avg:153.87ms
step:810/1480 train_time:123107ms step_avg:153.88ms
step:811/1480 train_time:123268ms step_avg:153.89ms
step:812/1480 train_time:123432ms step_avg:153.91ms
step:813/1480 train_time:123594ms step_avg:153.91ms
step:814/1480 train_time:123757ms step_avg:153.93ms
step:815/1480 train_time:123920ms step_avg:153.94ms
step:816/1480 train_time:124084ms step_avg:153.95ms
step:817/1480 train_time:124245ms step_avg:153.96ms
step:818/1480 train_time:124406ms step_avg:153.97ms
step:819/1480 train_time:124571ms step_avg:153.98ms
step:820/1480 train_time:124734ms step_avg:153.99ms
step:821/1480 train_time:124896ms step_avg:154.00ms
step:822/1480 train_time:125059ms step_avg:154.01ms
step:823/1480 train_time:125221ms step_avg:154.02ms
step:824/1480 train_time:125383ms step_avg:154.03ms
step:825/1480 train_time:125548ms step_avg:154.05ms
step:826/1480 train_time:125714ms step_avg:154.06ms
step:827/1480 train_time:125879ms step_avg:154.07ms
step:828/1480 train_time:126041ms step_avg:154.08ms
step:829/1480 train_time:126206ms step_avg:154.10ms
step:830/1480 train_time:126370ms step_avg:154.11ms
step:831/1480 train_time:126535ms step_avg:154.12ms
step:832/1480 train_time:126699ms step_avg:154.14ms
step:833/1480 train_time:126863ms step_avg:154.15ms
step:834/1480 train_time:127028ms step_avg:154.16ms
step:835/1480 train_time:127193ms step_avg:154.17ms
step:836/1480 train_time:127358ms step_avg:154.19ms
step:837/1480 train_time:127520ms step_avg:154.20ms
step:838/1480 train_time:127684ms step_avg:154.21ms
step:839/1480 train_time:127846ms step_avg:154.22ms
step:840/1480 train_time:128005ms step_avg:154.22ms
step:841/1480 train_time:128166ms step_avg:154.23ms
step:842/1480 train_time:128328ms step_avg:154.24ms
step:843/1480 train_time:128492ms step_avg:154.25ms
step:844/1480 train_time:128655ms step_avg:154.26ms
step:845/1480 train_time:128819ms step_avg:154.27ms
step:846/1480 train_time:128983ms step_avg:154.29ms
step:847/1480 train_time:129145ms step_avg:154.29ms
step:848/1480 train_time:129306ms step_avg:154.30ms
step:849/1480 train_time:129470ms step_avg:154.32ms
step:850/1480 train_time:129633ms step_avg:154.32ms
step:851/1480 train_time:129798ms step_avg:154.34ms
step:852/1480 train_time:129960ms step_avg:154.35ms
step:853/1480 train_time:130122ms step_avg:154.36ms
step:854/1480 train_time:130285ms step_avg:154.37ms
step:855/1480 train_time:130449ms step_avg:154.38ms
step:856/1480 train_time:130609ms step_avg:154.38ms
step:857/1480 train_time:130773ms step_avg:154.40ms
step:858/1480 train_time:130940ms step_avg:154.41ms
step:859/1480 train_time:131104ms step_avg:154.42ms
step:860/1480 train_time:131265ms step_avg:154.43ms
step:861/1480 train_time:131432ms step_avg:154.44ms
step:862/1480 train_time:131601ms step_avg:154.46ms
step:863/1480 train_time:131767ms step_avg:154.48ms
step:864/1480 train_time:131931ms step_avg:154.49ms
step:865/1480 train_time:132094ms step_avg:154.50ms
step:866/1480 train_time:132260ms step_avg:154.51ms
step:867/1480 train_time:132424ms step_avg:154.52ms
step:868/1480 train_time:132585ms step_avg:154.53ms
step:869/1480 train_time:132746ms step_avg:154.54ms
step:870/1480 train_time:132912ms step_avg:154.55ms
step:871/1480 train_time:133075ms step_avg:154.56ms
step:872/1480 train_time:133240ms step_avg:154.57ms
step:873/1480 train_time:133403ms step_avg:154.58ms
step:874/1480 train_time:133568ms step_avg:154.59ms
step:875/1480 train_time:133734ms step_avg:154.61ms
step:875/1480 val_loss:3.5035 train_time:133798ms step_avg:154.68ms
step:876/1480 train_time:133901ms step_avg:154.62ms
step:877/1480 train_time:134067ms step_avg:154.63ms
step:878/1480 train_time:134229ms step_avg:154.64ms
step:879/1480 train_time:134392ms step_avg:154.65ms
step:880/1480 train_time:134553ms step_avg:154.66ms
step:881/1480 train_time:134716ms step_avg:154.67ms
step:882/1480 train_time:134883ms step_avg:154.68ms
step:883/1480 train_time:135049ms step_avg:154.70ms
step:884/1480 train_time:135215ms step_avg:154.71ms
step:885/1480 train_time:135381ms step_avg:154.72ms
step:886/1480 train_time:135548ms step_avg:154.73ms
step:887/1480 train_time:135714ms step_avg:154.75ms
step:888/1480 train_time:135888ms step_avg:154.77ms
step:889/1480 train_time:136056ms step_avg:154.78ms
step:890/1480 train_time:136219ms step_avg:154.79ms
step:891/1480 train_time:136386ms step_avg:154.81ms
step:892/1480 train_time:136551ms step_avg:154.82ms
step:893/1480 train_time:136713ms step_avg:154.83ms
step:894/1480 train_time:136880ms step_avg:154.84ms
step:895/1480 train_time:137046ms step_avg:154.85ms
step:896/1480 train_time:137210ms step_avg:154.86ms
step:897/1480 train_time:137376ms step_avg:154.88ms
step:898/1480 train_time:137545ms step_avg:154.89ms
step:899/1480 train_time:137709ms step_avg:154.90ms
step:900/1480 train_time:137872ms step_avg:154.91ms
step:901/1480 train_time:138035ms step_avg:154.92ms
step:902/1480 train_time:138200ms step_avg:154.93ms
step:903/1480 train_time:138370ms step_avg:154.95ms
step:904/1480 train_time:138536ms step_avg:154.96ms
step:905/1480 train_time:138699ms step_avg:154.97ms
step:906/1480 train_time:138866ms step_avg:154.98ms
step:907/1480 train_time:139034ms step_avg:155.00ms
step:908/1480 train_time:139196ms step_avg:155.01ms
step:909/1480 train_time:139362ms step_avg:155.02ms
step:910/1480 train_time:139532ms step_avg:155.04ms
step:911/1480 train_time:139697ms step_avg:155.05ms
step:912/1480 train_time:139864ms step_avg:155.06ms
step:913/1480 train_time:140031ms step_avg:155.07ms
step:914/1480 train_time:140198ms step_avg:155.09ms
step:915/1480 train_time:140367ms step_avg:155.10ms
step:916/1480 train_time:140531ms step_avg:155.11ms
step:917/1480 train_time:140694ms step_avg:155.12ms
step:918/1480 train_time:140862ms step_avg:155.13ms
step:919/1480 train_time:141032ms step_avg:155.15ms
step:920/1480 train_time:141196ms step_avg:155.16ms
step:921/1480 train_time:141363ms step_avg:155.17ms
step:922/1480 train_time:141530ms step_avg:155.19ms
step:923/1480 train_time:141691ms step_avg:155.19ms
step:924/1480 train_time:141856ms step_avg:155.20ms
step:925/1480 train_time:142022ms step_avg:155.22ms
step:926/1480 train_time:142185ms step_avg:155.22ms
step:927/1480 train_time:142351ms step_avg:155.24ms
step:928/1480 train_time:142519ms step_avg:155.25ms
step:929/1480 train_time:142684ms step_avg:155.26ms
step:930/1480 train_time:142850ms step_avg:155.27ms
step:931/1480 train_time:143013ms step_avg:155.28ms
step:932/1480 train_time:143179ms step_avg:155.29ms
step:933/1480 train_time:143346ms step_avg:155.30ms
step:934/1480 train_time:143512ms step_avg:155.32ms
step:935/1480 train_time:143683ms step_avg:155.33ms
step:936/1480 train_time:143851ms step_avg:155.35ms
step:937/1480 train_time:144023ms step_avg:155.36ms
step:938/1480 train_time:144186ms step_avg:155.37ms
step:939/1480 train_time:144355ms step_avg:155.39ms
step:940/1480 train_time:144522ms step_avg:155.40ms
step:941/1480 train_time:144686ms step_avg:155.41ms
step:942/1480 train_time:144850ms step_avg:155.42ms
step:943/1480 train_time:145021ms step_avg:155.44ms
step:944/1480 train_time:145192ms step_avg:155.45ms
step:945/1480 train_time:145358ms step_avg:155.46ms
step:946/1480 train_time:145527ms step_avg:155.48ms
step:947/1480 train_time:145693ms step_avg:155.49ms
step:948/1480 train_time:145860ms step_avg:155.50ms
step:949/1480 train_time:146026ms step_avg:155.51ms
step:950/1480 train_time:146190ms step_avg:155.52ms
step:951/1480 train_time:146358ms step_avg:155.53ms
step:952/1480 train_time:146524ms step_avg:155.55ms
step:953/1480 train_time:146693ms step_avg:155.56ms
step:954/1480 train_time:146863ms step_avg:155.58ms
step:955/1480 train_time:147027ms step_avg:155.58ms
step:956/1480 train_time:147192ms step_avg:155.59ms
step:957/1480 train_time:147361ms step_avg:155.61ms
step:958/1480 train_time:147530ms step_avg:155.62ms
step:959/1480 train_time:147694ms step_avg:155.63ms
step:960/1480 train_time:147863ms step_avg:155.65ms
step:961/1480 train_time:148028ms step_avg:155.66ms
step:962/1480 train_time:148192ms step_avg:155.66ms
step:963/1480 train_time:148357ms step_avg:155.67ms
step:964/1480 train_time:148526ms step_avg:155.69ms
step:965/1480 train_time:148690ms step_avg:155.70ms
step:966/1480 train_time:148855ms step_avg:155.71ms
step:967/1480 train_time:149019ms step_avg:155.71ms
step:968/1480 train_time:149185ms step_avg:155.73ms
step:969/1480 train_time:149350ms step_avg:155.74ms
step:970/1480 train_time:149512ms step_avg:155.74ms
step:971/1480 train_time:149675ms step_avg:155.75ms
step:972/1480 train_time:149841ms step_avg:155.76ms
step:973/1480 train_time:150006ms step_avg:155.77ms
step:974/1480 train_time:150174ms step_avg:155.78ms
step:975/1480 train_time:150341ms step_avg:155.79ms
step:976/1480 train_time:150507ms step_avg:155.80ms
step:977/1480 train_time:150671ms step_avg:155.81ms
step:978/1480 train_time:150837ms step_avg:155.82ms
step:979/1480 train_time:151003ms step_avg:155.83ms
step:980/1480 train_time:151169ms step_avg:155.84ms
step:981/1480 train_time:151337ms step_avg:155.86ms
step:982/1480 train_time:151500ms step_avg:155.86ms
step:983/1480 train_time:151665ms step_avg:155.87ms
step:984/1480 train_time:151829ms step_avg:155.88ms
step:985/1480 train_time:151996ms step_avg:155.89ms
step:986/1480 train_time:152163ms step_avg:155.91ms
step:987/1480 train_time:152328ms step_avg:155.91ms
step:988/1480 train_time:152494ms step_avg:155.92ms
step:989/1480 train_time:152661ms step_avg:155.94ms
step:990/1480 train_time:152829ms step_avg:155.95ms
step:991/1480 train_time:152996ms step_avg:155.96ms
step:992/1480 train_time:153171ms step_avg:155.98ms
step:993/1480 train_time:153348ms step_avg:156.00ms
step:994/1480 train_time:153512ms step_avg:156.01ms
step:995/1480 train_time:153675ms step_avg:156.02ms
step:996/1480 train_time:153838ms step_avg:156.02ms
step:997/1480 train_time:154004ms step_avg:156.03ms
step:998/1480 train_time:154168ms step_avg:156.04ms
step:999/1480 train_time:154333ms step_avg:156.05ms
step:1000/1480 train_time:154501ms step_avg:156.06ms
step:1000/1480 val_loss:3.4389 train_time:154569ms step_avg:156.13ms
step:1001/1480 train_time:154673ms step_avg:156.08ms
step:1002/1480 train_time:154839ms step_avg:156.09ms
step:1003/1480 train_time:155010ms step_avg:156.10ms
step:1004/1480 train_time:155180ms step_avg:156.12ms
step:1005/1480 train_time:155347ms step_avg:156.13ms
step:1006/1480 train_time:155515ms step_avg:156.14ms
step:1007/1480 train_time:155681ms step_avg:156.15ms
step:1008/1480 train_time:155848ms step_avg:156.16ms
step:1009/1480 train_time:156021ms step_avg:156.18ms
step:1010/1480 train_time:156186ms step_avg:156.19ms
step:1011/1480 train_time:156352ms step_avg:156.20ms
step:1012/1480 train_time:156518ms step_avg:156.21ms
step:1013/1480 train_time:156689ms step_avg:156.22ms
step:1014/1480 train_time:156857ms step_avg:156.23ms
step:1015/1480 train_time:157026ms step_avg:156.24ms
step:1016/1480 train_time:157196ms step_avg:156.26ms
step:1017/1480 train_time:157368ms step_avg:156.27ms
step:1018/1480 train_time:157536ms step_avg:156.29ms
step:1019/1480 train_time:157704ms step_avg:156.30ms
step:1020/1480 train_time:157874ms step_avg:156.31ms
step:1021/1480 train_time:158040ms step_avg:156.32ms
step:1022/1480 train_time:158206ms step_avg:156.33ms
step:1023/1480 train_time:158375ms step_avg:156.34ms
step:1024/1480 train_time:158542ms step_avg:156.35ms
step:1025/1480 train_time:158714ms step_avg:156.37ms
step:1026/1480 train_time:158880ms step_avg:156.38ms
step:1027/1480 train_time:159047ms step_avg:156.39ms
step:1028/1480 train_time:159220ms step_avg:156.40ms
step:1029/1480 train_time:159394ms step_avg:156.42ms
step:1030/1480 train_time:159561ms step_avg:156.43ms
step:1031/1480 train_time:159725ms step_avg:156.44ms
step:1032/1480 train_time:159896ms step_avg:156.45ms
step:1033/1480 train_time:160063ms step_avg:156.46ms
step:1034/1480 train_time:160231ms step_avg:156.48ms
step:1035/1480 train_time:160399ms step_avg:156.49ms
step:1036/1480 train_time:160564ms step_avg:156.50ms
step:1037/1480 train_time:160732ms step_avg:156.51ms
step:1038/1480 train_time:160900ms step_avg:156.52ms
step:1039/1480 train_time:161070ms step_avg:156.53ms
step:1040/1480 train_time:161237ms step_avg:156.54ms
step:1041/1480 train_time:161405ms step_avg:156.55ms
step:1042/1480 train_time:161569ms step_avg:156.56ms
step:1043/1480 train_time:161733ms step_avg:156.57ms
step:1044/1480 train_time:161899ms step_avg:156.58ms
step:1045/1480 train_time:162069ms step_avg:156.59ms
step:1046/1480 train_time:162237ms step_avg:156.60ms
step:1047/1480 train_time:162405ms step_avg:156.61ms
step:1048/1480 train_time:162571ms step_avg:156.62ms
step:1049/1480 train_time:162737ms step_avg:156.63ms
step:1050/1480 train_time:162906ms step_avg:156.64ms
step:1051/1480 train_time:163076ms step_avg:156.65ms
step:1052/1480 train_time:163242ms step_avg:156.66ms
step:1053/1480 train_time:163407ms step_avg:156.67ms
step:1054/1480 train_time:163577ms step_avg:156.68ms
step:1055/1480 train_time:163743ms step_avg:156.69ms
step:1056/1480 train_time:163907ms step_avg:156.70ms
step:1057/1480 train_time:164074ms step_avg:156.71ms
step:1058/1480 train_time:164242ms step_avg:156.72ms
step:1059/1480 train_time:164415ms step_avg:156.73ms
step:1060/1480 train_time:164583ms step_avg:156.75ms
step:1061/1480 train_time:164747ms step_avg:156.75ms
step:1062/1480 train_time:164914ms step_avg:156.76ms
step:1063/1480 train_time:165079ms step_avg:156.77ms
step:1064/1480 train_time:165242ms step_avg:156.78ms
step:1065/1480 train_time:165409ms step_avg:156.79ms
step:1066/1480 train_time:165577ms step_avg:156.80ms
step:1067/1480 train_time:165745ms step_avg:156.81ms
step:1068/1480 train_time:165911ms step_avg:156.82ms
step:1069/1480 train_time:166082ms step_avg:156.83ms
step:1070/1480 train_time:166247ms step_avg:156.84ms
step:1071/1480 train_time:166421ms step_avg:156.85ms
step:1072/1480 train_time:166586ms step_avg:156.86ms
step:1073/1480 train_time:166750ms step_avg:156.87ms
step:1074/1480 train_time:166916ms step_avg:156.88ms
step:1075/1480 train_time:167089ms step_avg:156.89ms
step:1076/1480 train_time:167257ms step_avg:156.90ms
step:1077/1480 train_time:167424ms step_avg:156.91ms
step:1078/1480 train_time:167599ms step_avg:156.93ms
step:1079/1480 train_time:167772ms step_avg:156.94ms
step:1080/1480 train_time:167942ms step_avg:156.96ms
step:1081/1480 train_time:168108ms step_avg:156.96ms
step:1082/1480 train_time:168274ms step_avg:156.97ms
step:1083/1480 train_time:168440ms step_avg:156.98ms
step:1084/1480 train_time:168606ms step_avg:156.99ms
step:1085/1480 train_time:168774ms step_avg:157.00ms
step:1086/1480 train_time:168941ms step_avg:157.01ms
step:1087/1480 train_time:169106ms step_avg:157.02ms
step:1088/1480 train_time:169277ms step_avg:157.03ms
step:1089/1480 train_time:169448ms step_avg:157.04ms
step:1090/1480 train_time:169620ms step_avg:157.06ms
step:1091/1480 train_time:169788ms step_avg:157.07ms
step:1092/1480 train_time:169956ms step_avg:157.08ms
step:1093/1480 train_time:170123ms step_avg:157.08ms
step:1094/1480 train_time:170289ms step_avg:157.09ms
step:1095/1480 train_time:170453ms step_avg:157.10ms
step:1096/1480 train_time:170622ms step_avg:157.11ms
step:1097/1480 train_time:170790ms step_avg:157.12ms
step:1098/1480 train_time:170961ms step_avg:157.13ms
step:1099/1480 train_time:171132ms step_avg:157.15ms
step:1100/1480 train_time:171304ms step_avg:157.16ms
step:1101/1480 train_time:171474ms step_avg:157.17ms
step:1102/1480 train_time:171644ms step_avg:157.18ms
step:1103/1480 train_time:171821ms step_avg:157.20ms
step:1104/1480 train_time:171988ms step_avg:157.21ms
step:1105/1480 train_time:172158ms step_avg:157.22ms
step:1106/1480 train_time:172326ms step_avg:157.23ms
step:1107/1480 train_time:172495ms step_avg:157.24ms
step:1108/1480 train_time:172660ms step_avg:157.25ms
step:1109/1480 train_time:172825ms step_avg:157.26ms
step:1110/1480 train_time:172991ms step_avg:157.26ms
step:1111/1480 train_time:173159ms step_avg:157.27ms
step:1112/1480 train_time:173329ms step_avg:157.29ms
step:1113/1480 train_time:173511ms step_avg:157.31ms
step:1114/1480 train_time:173684ms step_avg:157.32ms
step:1115/1480 train_time:173858ms step_avg:157.34ms
step:1116/1480 train_time:174024ms step_avg:157.35ms
step:1117/1480 train_time:174198ms step_avg:157.36ms
step:1118/1480 train_time:174372ms step_avg:157.38ms
step:1119/1480 train_time:174539ms step_avg:157.38ms
step:1120/1480 train_time:174706ms step_avg:157.39ms
step:1121/1480 train_time:174877ms step_avg:157.41ms
step:1122/1480 train_time:175043ms step_avg:157.41ms
step:1123/1480 train_time:175209ms step_avg:157.42ms
step:1124/1480 train_time:175379ms step_avg:157.43ms
step:1125/1480 train_time:175547ms step_avg:157.44ms
step:1125/1480 val_loss:3.3843 train_time:175616ms step_avg:157.50ms
step:1126/1480 train_time:175718ms step_avg:157.45ms
step:1127/1480 train_time:175887ms step_avg:157.46ms
step:1128/1480 train_time:176058ms step_avg:157.48ms
step:1129/1480 train_time:176231ms step_avg:157.49ms
step:1130/1480 train_time:176402ms step_avg:157.50ms
step:1131/1480 train_time:176581ms step_avg:157.52ms
step:1132/1480 train_time:176746ms step_avg:157.53ms
step:1133/1480 train_time:176919ms step_avg:157.54ms
step:1134/1480 train_time:177089ms step_avg:157.55ms
step:1135/1480 train_time:177257ms step_avg:157.56ms
step:1136/1480 train_time:177427ms step_avg:157.57ms
step:1137/1480 train_time:177595ms step_avg:157.58ms
step:1138/1480 train_time:177765ms step_avg:157.59ms
step:1139/1480 train_time:177934ms step_avg:157.60ms
step:1140/1480 train_time:178102ms step_avg:157.61ms
step:1141/1480 train_time:178274ms step_avg:157.63ms
step:1142/1480 train_time:178442ms step_avg:157.63ms
step:1143/1480 train_time:178613ms step_avg:157.65ms
step:1144/1480 train_time:178782ms step_avg:157.66ms
step:1145/1480 train_time:178947ms step_avg:157.66ms
step:1146/1480 train_time:179119ms step_avg:157.68ms
step:1147/1480 train_time:179288ms step_avg:157.68ms
step:1148/1480 train_time:179457ms step_avg:157.70ms
step:1149/1480 train_time:179628ms step_avg:157.71ms
step:1150/1480 train_time:179796ms step_avg:157.72ms
step:1151/1480 train_time:179967ms step_avg:157.73ms
step:1152/1480 train_time:180138ms step_avg:157.74ms
step:1153/1480 train_time:180310ms step_avg:157.75ms
step:1154/1480 train_time:180478ms step_avg:157.76ms
step:1155/1480 train_time:180649ms step_avg:157.77ms
step:1156/1480 train_time:180827ms step_avg:157.79ms
step:1157/1480 train_time:180997ms step_avg:157.80ms
step:1158/1480 train_time:181163ms step_avg:157.81ms
step:1159/1480 train_time:181330ms step_avg:157.82ms
step:1160/1480 train_time:181497ms step_avg:157.82ms
step:1161/1480 train_time:181667ms step_avg:157.83ms
step:1162/1480 train_time:181837ms step_avg:157.84ms
step:1163/1480 train_time:182008ms step_avg:157.86ms
step:1164/1480 train_time:182177ms step_avg:157.87ms
step:1165/1480 train_time:182344ms step_avg:157.87ms
step:1166/1480 train_time:182512ms step_avg:157.88ms
step:1167/1480 train_time:182681ms step_avg:157.89ms
step:1168/1480 train_time:182849ms step_avg:157.90ms
step:1169/1480 train_time:183019ms step_avg:157.91ms
step:1170/1480 train_time:183187ms step_avg:157.92ms
step:1171/1480 train_time:183355ms step_avg:157.93ms
step:1172/1480 train_time:183521ms step_avg:157.94ms
step:1173/1480 train_time:183692ms step_avg:157.95ms
step:1174/1480 train_time:183873ms step_avg:157.97ms
step:1175/1480 train_time:184045ms step_avg:157.98ms
step:1176/1480 train_time:184217ms step_avg:157.99ms
step:1177/1480 train_time:184395ms step_avg:158.01ms
step:1178/1480 train_time:184561ms step_avg:158.01ms
step:1179/1480 train_time:184727ms step_avg:158.02ms
step:1180/1480 train_time:184908ms step_avg:158.04ms
step:1181/1480 train_time:185079ms step_avg:158.05ms
step:1182/1480 train_time:185247ms step_avg:158.06ms
step:1183/1480 train_time:185419ms step_avg:158.07ms
step:1184/1480 train_time:185586ms step_avg:158.08ms
step:1185/1480 train_time:185758ms step_avg:158.09ms
step:1186/1480 train_time:185929ms step_avg:158.10ms
step:1187/1480 train_time:186112ms step_avg:158.12ms
step:1188/1480 train_time:186279ms step_avg:158.13ms
step:1189/1480 train_time:186449ms step_avg:158.14ms
step:1190/1480 train_time:186617ms step_avg:158.15ms
step:1191/1480 train_time:186788ms step_avg:158.16ms
step:1192/1480 train_time:186955ms step_avg:158.17ms
step:1193/1480 train_time:187122ms step_avg:158.18ms
step:1194/1480 train_time:187290ms step_avg:158.18ms
step:1195/1480 train_time:187463ms step_avg:158.20ms
step:1196/1480 train_time:187645ms step_avg:158.22ms
step:1197/1480 train_time:187818ms step_avg:158.23ms
step:1198/1480 train_time:188001ms step_avg:158.25ms
step:1199/1480 train_time:188171ms step_avg:158.26ms
step:1200/1480 train_time:188341ms step_avg:158.27ms
step:1201/1480 train_time:188508ms step_avg:158.28ms
step:1202/1480 train_time:188689ms step_avg:158.30ms
step:1203/1480 train_time:188865ms step_avg:158.31ms
step:1204/1480 train_time:189041ms step_avg:158.33ms
step:1205/1480 train_time:189209ms step_avg:158.33ms
step:1206/1480 train_time:189377ms step_avg:158.34ms
step:1207/1480 train_time:189546ms step_avg:158.35ms
step:1208/1480 train_time:189714ms step_avg:158.36ms
step:1209/1480 train_time:189886ms step_avg:158.37ms
step:1210/1480 train_time:190062ms step_avg:158.39ms
step:1211/1480 train_time:190236ms step_avg:158.40ms
step:1212/1480 train_time:190408ms step_avg:158.41ms
step:1213/1480 train_time:190582ms step_avg:158.42ms
step:1214/1480 train_time:190761ms step_avg:158.44ms
step:1215/1480 train_time:190933ms step_avg:158.45ms
step:1216/1480 train_time:191103ms step_avg:158.46ms
step:1217/1480 train_time:191277ms step_avg:158.47ms
step:1218/1480 train_time:191447ms step_avg:158.48ms
step:1219/1480 train_time:191625ms step_avg:158.50ms
step:1220/1480 train_time:191794ms step_avg:158.51ms
step:1221/1480 train_time:191962ms step_avg:158.52ms
step:1222/1480 train_time:192128ms step_avg:158.52ms
step:1223/1480 train_time:192299ms step_avg:158.53ms
step:1224/1480 train_time:192478ms step_avg:158.55ms
step:1225/1480 train_time:192650ms step_avg:158.56ms
step:1226/1480 train_time:192824ms step_avg:158.57ms
step:1227/1480 train_time:192995ms step_avg:158.58ms
step:1228/1480 train_time:193165ms step_avg:158.59ms
step:1229/1480 train_time:193337ms step_avg:158.60ms
step:1230/1480 train_time:193517ms step_avg:158.62ms
step:1231/1480 train_time:193693ms step_avg:158.63ms
step:1232/1480 train_time:193869ms step_avg:158.65ms
step:1233/1480 train_time:194040ms step_avg:158.66ms
step:1234/1480 train_time:194209ms step_avg:158.67ms
step:1235/1480 train_time:194385ms step_avg:158.68ms
step:1236/1480 train_time:194552ms step_avg:158.69ms
step:1237/1480 train_time:194724ms step_avg:158.70ms
step:1238/1480 train_time:194909ms step_avg:158.72ms
step:1239/1480 train_time:195082ms step_avg:158.73ms
step:1240/1480 train_time:195251ms step_avg:158.74ms
step:1241/1480 train_time:195425ms step_avg:158.75ms
step:1242/1480 train_time:195594ms step_avg:158.76ms
step:1243/1480 train_time:195765ms step_avg:158.77ms
step:1244/1480 train_time:195933ms step_avg:158.78ms
step:1245/1480 train_time:196102ms step_avg:158.79ms
step:1246/1480 train_time:196272ms step_avg:158.80ms
step:1247/1480 train_time:196442ms step_avg:158.80ms
step:1248/1480 train_time:196612ms step_avg:158.81ms
step:1249/1480 train_time:196781ms step_avg:158.82ms
step:1250/1480 train_time:196950ms step_avg:158.83ms
step:1250/1480 val_loss:3.3344 train_time:197022ms step_avg:158.89ms
step:1251/1480 train_time:197131ms step_avg:158.85ms
step:1252/1480 train_time:197302ms step_avg:158.86ms
step:1253/1480 train_time:197469ms step_avg:158.87ms
step:1254/1480 train_time:197640ms step_avg:158.87ms
step:1255/1480 train_time:197827ms step_avg:158.90ms
step:1256/1480 train_time:198001ms step_avg:158.91ms
step:1257/1480 train_time:198171ms step_avg:158.92ms
step:1258/1480 train_time:198348ms step_avg:158.93ms
step:1259/1480 train_time:198520ms step_avg:158.94ms
step:1260/1480 train_time:198687ms step_avg:158.95ms
step:1261/1480 train_time:198858ms step_avg:158.96ms
step:1262/1480 train_time:199034ms step_avg:158.97ms
step:1263/1480 train_time:199207ms step_avg:158.98ms
step:1264/1480 train_time:199372ms step_avg:158.99ms
step:1265/1480 train_time:199540ms step_avg:159.00ms
step:1266/1480 train_time:199710ms step_avg:159.00ms
step:1267/1480 train_time:199882ms step_avg:159.02ms
step:1268/1480 train_time:200052ms step_avg:159.02ms
step:1269/1480 train_time:200228ms step_avg:159.04ms
step:1270/1480 train_time:200398ms step_avg:159.05ms
step:1271/1480 train_time:200568ms step_avg:159.06ms
step:1272/1480 train_time:200735ms step_avg:159.06ms
step:1273/1480 train_time:200908ms step_avg:159.07ms
step:1274/1480 train_time:201080ms step_avg:159.08ms
step:1275/1480 train_time:201249ms step_avg:159.09ms
step:1276/1480 train_time:201414ms step_avg:159.10ms
step:1277/1480 train_time:201586ms step_avg:159.11ms
step:1278/1480 train_time:201754ms step_avg:159.11ms
step:1279/1480 train_time:201925ms step_avg:159.12ms
step:1280/1480 train_time:202105ms step_avg:159.14ms
step:1281/1480 train_time:202273ms step_avg:159.14ms
step:1282/1480 train_time:202439ms step_avg:159.15ms
step:1283/1480 train_time:202610ms step_avg:159.16ms
step:1284/1480 train_time:202780ms step_avg:159.17ms
step:1285/1480 train_time:202950ms step_avg:159.18ms
step:1286/1480 train_time:203119ms step_avg:159.18ms
step:1287/1480 train_time:203291ms step_avg:159.19ms
step:1288/1480 train_time:203462ms step_avg:159.20ms
step:1289/1480 train_time:203647ms step_avg:159.22ms
step:1290/1480 train_time:203825ms step_avg:159.24ms
step:1291/1480 train_time:203999ms step_avg:159.25ms
step:1292/1480 train_time:204173ms step_avg:159.26ms
step:1293/1480 train_time:204349ms step_avg:159.27ms
step:1294/1480 train_time:204521ms step_avg:159.28ms
step:1295/1480 train_time:204692ms step_avg:159.29ms
step:1296/1480 train_time:204867ms step_avg:159.31ms
step:1297/1480 train_time:205039ms step_avg:159.32ms
step:1298/1480 train_time:205210ms step_avg:159.32ms
step:1299/1480 train_time:205380ms step_avg:159.33ms
step:1300/1480 train_time:205548ms step_avg:159.34ms
step:1301/1480 train_time:205718ms step_avg:159.35ms
step:1302/1480 train_time:205894ms step_avg:159.36ms
step:1303/1480 train_time:206071ms step_avg:159.37ms
step:1304/1480 train_time:206247ms step_avg:159.39ms
step:1305/1480 train_time:206416ms step_avg:159.39ms
step:1306/1480 train_time:206590ms step_avg:159.41ms
step:1307/1480 train_time:206757ms step_avg:159.41ms
step:1308/1480 train_time:206926ms step_avg:159.42ms
step:1309/1480 train_time:207098ms step_avg:159.43ms
step:1310/1480 train_time:207267ms step_avg:159.44ms
step:1311/1480 train_time:207435ms step_avg:159.44ms
step:1312/1480 train_time:207607ms step_avg:159.45ms
step:1313/1480 train_time:207775ms step_avg:159.46ms
step:1314/1480 train_time:207949ms step_avg:159.47ms
step:1315/1480 train_time:208121ms step_avg:159.48ms
step:1316/1480 train_time:208289ms step_avg:159.49ms
step:1317/1480 train_time:208460ms step_avg:159.49ms
step:1318/1480 train_time:208640ms step_avg:159.51ms
step:1319/1480 train_time:208816ms step_avg:159.52ms
step:1320/1480 train_time:208992ms step_avg:159.54ms
step:1321/1480 train_time:209163ms step_avg:159.54ms
step:1322/1480 train_time:209346ms step_avg:159.56ms
step:1323/1480 train_time:209518ms step_avg:159.57ms
step:1324/1480 train_time:209692ms step_avg:159.58ms
step:1325/1480 train_time:209873ms step_avg:159.60ms
step:1326/1480 train_time:210049ms step_avg:159.61ms
step:1327/1480 train_time:210219ms step_avg:159.62ms
step:1328/1480 train_time:210389ms step_avg:159.63ms
step:1329/1480 train_time:210585ms step_avg:159.65ms
step:1330/1480 train_time:210764ms step_avg:159.67ms
step:1331/1480 train_time:210934ms step_avg:159.68ms
step:1332/1480 train_time:211109ms step_avg:159.69ms
step:1333/1480 train_time:211285ms step_avg:159.70ms
step:1334/1480 train_time:211455ms step_avg:159.71ms
step:1335/1480 train_time:211624ms step_avg:159.72ms
step:1336/1480 train_time:211807ms step_avg:159.73ms
step:1337/1480 train_time:211982ms step_avg:159.75ms
step:1338/1480 train_time:212154ms step_avg:159.75ms
step:1339/1480 train_time:212327ms step_avg:159.76ms
step:1340/1480 train_time:212500ms step_avg:159.77ms
step:1341/1480 train_time:212668ms step_avg:159.78ms
step:1342/1480 train_time:212841ms step_avg:159.79ms
step:1343/1480 train_time:213011ms step_avg:159.80ms
step:1344/1480 train_time:213184ms step_avg:159.81ms
step:1345/1480 train_time:213363ms step_avg:159.82ms
step:1346/1480 train_time:213533ms step_avg:159.83ms
step:1347/1480 train_time:213703ms step_avg:159.84ms
step:1348/1480 train_time:213872ms step_avg:159.84ms
step:1349/1480 train_time:214042ms step_avg:159.85ms
step:1350/1480 train_time:214217ms step_avg:159.86ms
step:1351/1480 train_time:214387ms step_avg:159.87ms
step:1352/1480 train_time:214558ms step_avg:159.88ms
step:1353/1480 train_time:214734ms step_avg:159.89ms
step:1354/1480 train_time:214905ms step_avg:159.90ms
step:1355/1480 train_time:215073ms step_avg:159.91ms
step:1356/1480 train_time:215247ms step_avg:159.92ms
step:1357/1480 train_time:215421ms step_avg:159.93ms
step:1358/1480 train_time:215592ms step_avg:159.94ms
step:1359/1480 train_time:215766ms step_avg:159.94ms
step:1360/1480 train_time:215939ms step_avg:159.95ms
step:1361/1480 train_time:216116ms step_avg:159.97ms
step:1362/1480 train_time:216292ms step_avg:159.98ms
step:1363/1480 train_time:216475ms step_avg:160.00ms
step:1364/1480 train_time:216645ms step_avg:160.00ms
step:1365/1480 train_time:216812ms step_avg:160.01ms
step:1366/1480 train_time:216985ms step_avg:160.02ms
step:1367/1480 train_time:217155ms step_avg:160.03ms
step:1368/1480 train_time:217328ms step_avg:160.04ms
step:1369/1480 train_time:217510ms step_avg:160.05ms
step:1370/1480 train_time:217688ms step_avg:160.06ms
step:1371/1480 train_time:217860ms step_avg:160.07ms
step:1372/1480 train_time:218038ms step_avg:160.09ms
step:1373/1480 train_time:218208ms step_avg:160.09ms
step:1374/1480 train_time:218385ms step_avg:160.11ms
step:1375/1480 train_time:218555ms step_avg:160.11ms
step:1375/1480 val_loss:3.2962 train_time:218622ms step_avg:160.16ms
step:1376/1480 train_time:218726ms step_avg:160.12ms
step:1377/1480 train_time:218899ms step_avg:160.13ms
step:1378/1480 train_time:219067ms step_avg:160.14ms
step:1379/1480 train_time:219242ms step_avg:160.15ms
step:1380/1480 train_time:219416ms step_avg:160.16ms
step:1381/1480 train_time:219596ms step_avg:160.17ms
step:1382/1480 train_time:219766ms step_avg:160.18ms
step:1383/1480 train_time:219938ms step_avg:160.19ms
step:1384/1480 train_time:220114ms step_avg:160.20ms
step:1385/1480 train_time:220280ms step_avg:160.20ms
step:1386/1480 train_time:220450ms step_avg:160.21ms
step:1387/1480 train_time:220623ms step_avg:160.22ms
step:1388/1480 train_time:220792ms step_avg:160.23ms
step:1389/1480 train_time:220966ms step_avg:160.24ms
step:1390/1480 train_time:221133ms step_avg:160.24ms
step:1391/1480 train_time:221303ms step_avg:160.25ms
step:1392/1480 train_time:221475ms step_avg:160.26ms
step:1393/1480 train_time:221646ms step_avg:160.26ms
step:1394/1480 train_time:221817ms step_avg:160.27ms
step:1395/1480 train_time:221986ms step_avg:160.28ms
step:1396/1480 train_time:222155ms step_avg:160.28ms
step:1397/1480 train_time:222323ms step_avg:160.29ms
step:1398/1480 train_time:222489ms step_avg:160.29ms
step:1399/1480 train_time:222658ms step_avg:160.30ms
step:1400/1480 train_time:222836ms step_avg:160.31ms
step:1401/1480 train_time:223002ms step_avg:160.32ms
step:1402/1480 train_time:223175ms step_avg:160.33ms
step:1403/1480 train_time:223351ms step_avg:160.34ms
step:1404/1480 train_time:223523ms step_avg:160.35ms
step:1405/1480 train_time:223698ms step_avg:160.36ms
step:1406/1480 train_time:223874ms step_avg:160.37ms
step:1407/1480 train_time:224043ms step_avg:160.37ms
step:1408/1480 train_time:224212ms step_avg:160.38ms
step:1409/1480 train_time:224397ms step_avg:160.40ms
step:1410/1480 train_time:224566ms step_avg:160.40ms
step:1411/1480 train_time:224735ms step_avg:160.41ms
step:1412/1480 train_time:224904ms step_avg:160.42ms
step:1413/1480 train_time:225074ms step_avg:160.42ms
step:1414/1480 train_time:225245ms step_avg:160.43ms
step:1415/1480 train_time:225421ms step_avg:160.44ms
step:1416/1480 train_time:225606ms step_avg:160.46ms
step:1417/1480 train_time:225781ms step_avg:160.47ms
step:1418/1480 train_time:225952ms step_avg:160.48ms
step:1419/1480 train_time:226125ms step_avg:160.49ms
step:1420/1480 train_time:226301ms step_avg:160.50ms
step:1421/1480 train_time:226474ms step_avg:160.51ms
step:1422/1480 train_time:226646ms step_avg:160.51ms
step:1423/1480 train_time:226816ms step_avg:160.52ms
step:1424/1480 train_time:226993ms step_avg:160.53ms
step:1425/1480 train_time:227172ms step_avg:160.55ms
step:1426/1480 train_time:227344ms step_avg:160.55ms
step:1427/1480 train_time:227519ms step_avg:160.56ms
step:1428/1480 train_time:227689ms step_avg:160.57ms
step:1429/1480 train_time:227857ms step_avg:160.58ms
step:1430/1480 train_time:228030ms step_avg:160.58ms
step:1431/1480 train_time:228205ms step_avg:160.59ms
step:1432/1480 train_time:228382ms step_avg:160.61ms
step:1433/1480 train_time:228563ms step_avg:160.62ms
step:1434/1480 train_time:228744ms step_avg:160.63ms
step:1435/1480 train_time:228921ms step_avg:160.65ms
step:1436/1480 train_time:229096ms step_avg:160.66ms
step:1437/1480 train_time:229266ms step_avg:160.66ms
step:1438/1480 train_time:229435ms step_avg:160.67ms
step:1439/1480 train_time:229609ms step_avg:160.68ms
step:1440/1480 train_time:229779ms step_avg:160.68ms
step:1441/1480 train_time:229950ms step_avg:160.69ms
step:1442/1480 train_time:230127ms step_avg:160.70ms
step:1443/1480 train_time:230316ms step_avg:160.72ms
step:1444/1480 train_time:230486ms step_avg:160.73ms
step:1445/1480 train_time:230660ms step_avg:160.74ms
step:1446/1480 train_time:230834ms step_avg:160.75ms
step:1447/1480 train_time:231014ms step_avg:160.76ms
step:1448/1480 train_time:231184ms step_avg:160.77ms
step:1449/1480 train_time:231359ms step_avg:160.78ms
step:1450/1480 train_time:231532ms step_avg:160.79ms
step:1451/1480 train_time:231702ms step_avg:160.79ms
step:1452/1480 train_time:231875ms step_avg:160.80ms
step:1453/1480 train_time:232044ms step_avg:160.81ms
step:1454/1480 train_time:232215ms step_avg:160.81ms
step:1455/1480 train_time:232391ms step_avg:160.82ms
step:1456/1480 train_time:232564ms step_avg:160.83ms
step:1457/1480 train_time:232735ms step_avg:160.84ms
step:1458/1480 train_time:232906ms step_avg:160.85ms
step:1459/1480 train_time:233084ms step_avg:160.86ms
step:1460/1480 train_time:233255ms step_avg:160.87ms
step:1461/1480 train_time:233428ms step_avg:160.87ms
step:1462/1480 train_time:233599ms step_avg:160.88ms
step:1463/1480 train_time:233776ms step_avg:160.89ms
step:1464/1480 train_time:233951ms step_avg:160.90ms
step:1465/1480 train_time:234124ms step_avg:160.91ms
step:1466/1480 train_time:234294ms step_avg:160.92ms
step:1467/1480 train_time:234468ms step_avg:160.93ms
step:1468/1480 train_time:234639ms step_avg:160.93ms
step:1469/1480 train_time:234811ms step_avg:160.94ms
step:1470/1480 train_time:234989ms step_avg:160.95ms
step:1471/1480 train_time:235177ms step_avg:160.97ms
step:1472/1480 train_time:235359ms step_avg:160.98ms
step:1473/1480 train_time:235530ms step_avg:160.99ms
step:1474/1480 train_time:235707ms step_avg:161.00ms
step:1475/1480 train_time:235886ms step_avg:161.01ms
step:1476/1480 train_time:236058ms step_avg:161.02ms
step:1477/1480 train_time:236242ms step_avg:161.04ms
step:1478/1480 train_time:236423ms step_avg:161.05ms
step:1479/1480 train_time:236597ms step_avg:161.06ms
step:1480/1480 train_time:236772ms step_avg:161.07ms
step:1480/1480 val_loss:3.2773 train_time:236843ms step_avg:161.12ms