f2fda86f-4418-4ff8-a3a3-8112731cbcf9.txt

import os
import sys
with open(sys.argv[0]) as f:
    code = f.read() # read the code of this file ASAP, for logging
import uuid
import time
from dataclasses import dataclass
from pathlib import Path

import torch
from torch import nn
import torch.nn.functional as F
import torch.distributed as dist
import torch._inductor.config as config
from torch.nn.parallel import DistributedDataParallel as DDP
# Use of FlexAttention contributed by @KoszarskyB
from torch.nn.attention.flex_attention import BlockMask, flex_attention

# -----------------------------------------------------------------------------
# Muon optimizer

def zeropower_via_svd(G, steps=None):
    U, S, V = G.svd()
    return U @ V.T

@torch.compile
def zeropower_via_newtonschulz5(G, steps=10, eps=1e-7):
    """
    Newton-Schulz iteration to compute the zeroth power / orthogonalization of G. We opt to use a
    quintic iteration whose coefficients are selected to maximize the slope at zero. For the purpose
    of minimizing steps, it turns out to be empirically effective to keep increasing the slope at
    zero even beyond the point where the iteration no longer converges all the way to one everywhere
    on the interval. This iteration therefore does not produce UV^T but rather something like US'V^T
    where S' is diagonal with S_{ii}' ~ Uniform(0.5, 1.5), which turns out not to hurt model
    performance at all relative to UV^T, where USV^T = G is the SVD.
    """
    assert len(G.shape) == 2
    a, b, c = (3.4445, -4.7750,  2.0315)
    X = G.bfloat16()
    X /= (X.norm() + eps) # ensure top singular value <= 1
    if G.size(0) > G.size(1):
        X = X.T
    for _ in range(steps):
        A = X @ X.T
        B = b * A + c * A @ A # adapted from suggestion by @jxbz, @leloykun, and @YouJiacheng
        X = a * X + B @ X
    if G.size(0) > G.size(1):
        X = X.T
    return X

zeropower_backends = dict(svd=zeropower_via_svd, newtonschulz5=zeropower_via_newtonschulz5)

class Muon(torch.optim.Optimizer):
    """
    Muon - MomentUm Orthogonalized by Newton-schulz

    Muon internally runs standard SGD-momentum, and then performs an orthogonalization post-
    processing step, in which each 2D parameter's update is replaced with the nearest orthogonal
    matrix. To efficiently orthogonalize each update, we use a Newton-Schulz iteration, which has
    the advantage that it can be stably run in bfloat16 on the GPU.

    Some warnings:
    - This optimizer assumes that all parameters passed in are 2D.
    - It should not be used for the embedding layer, the final fully connected layer, or any {0,1}-D
    parameters; those should all be optimized by a standard method (e.g., AdamW).
    - To use it with 4D convolutional filters, it works well to just flatten their last 3 dimensions.
    - We believe it is unlikely to work well for training with small batch size.
    - We believe it may not work well for finetuning pretrained models, but we haven't tested this.
    - We have not yet tried this optimizer for training scenarios larger than NanoGPT (124M).

    Arguments:
        lr: The learning rate used by the internal SGD.
        momentum: The momentum used by the internal SGD.
        nesterov: Whether to use Nesterov-style momentum in the internal SGD. (recommended)
        backend: The chosen backend for the orthogonalization step. (recommended: 'newtonschulz5')
        backend_steps: The number of iteration steps to use in the backend, if it is iterative.
    """
    def __init__(self, params, lr=0.02, momentum=0.95, nesterov=True,
                 backend='newtonschulz5', backend_steps=5):
        self.num_process = int(os.environ['WORLD_SIZE'])
        self.rank = int(os.environ["RANK"])
        defaults = dict(lr=lr, momentum=momentum, nesterov=nesterov, backend=backend, backend_steps=backend_steps)
        params: "list[torch.Tensor]" = list(params)
        assert all(isinstance(p, torch.Tensor) for p in params)
        sizes = {p.numel() for p in params}
        param_groups = [
            {
                "params": [p for p in params if p.numel() == size],
                "update_buffer": [
                    torch.empty(size, device="cuda", dtype=torch.bfloat16)
                    for _ in range(self.num_process)
                ],
            }
            for size in sizes
        ]
        super().__init__(param_groups, defaults)

    def step(self):
        for group in self.param_groups:
            lr: float = group["lr"]
            momentum: float = group["momentum"]
            nesterov: bool = group["nesterov"]
            zeropower_backend = zeropower_backends[group["backend"]]
            backend_steps: int = group["backend_steps"]
            update_buffers: "list[torch.Tensor]" = group["update_buffer"]
            # generate weight updates in distributed fashion
            params: "list[torch.Tensor]" = group["params"]
            assert len(params) % self.num_process == 0
            handle = None
            params_world = None
            def update_prev():
                if params_world is None:
                    return
                assert handle is not None
                handle.wait()
                for p_world, g_world in zip(params_world, update_buffers):
                    p_world.data.add_(
                        g_world.view_as(p_world),
                        alpha=-lr * max(1, p_world.size(0) / p_world.size(1)) ** 0.5,
                    )
            for base_i in range(len(params))[::self.num_process]:
                p = params[base_i + self.rank]
                g = p.grad
                assert g is not None
                state = self.state[p] 
                if "momentum_buffer" not in state:
                    state["momentum_buffer"] = torch.zeros_like(g)
                buf: torch.Tensor = state["momentum_buffer"]
                buf.lerp_(g, 1 - momentum)
                g = g.lerp_(buf, momentum) if nesterov else buf
                g = zeropower_backend(g, steps=backend_steps).flatten()
                update_prev()
                handle = dist.all_gather(update_buffers, g, async_op=True)
                params_world = params[base_i : base_i + self.num_process]
            update_prev()


# -----------------------------------------------------------------------------
# PyTorch nn.Module definitions for the GPT-2 model

def norm(x):
    return F.rms_norm(x, (x.size(-1),))

class CastedLinear(nn.Linear):

    def __init__(self, in_features, out_features):
        super().__init__(in_features, out_features, bias=False)

    def forward(self, x):
        return F.linear(x, self.weight.to(x.dtype))

class Rotary(torch.nn.Module):

    def __init__(self, dim, base=10000):
        super().__init__()
        self.register_buffer('inv_freq', (1 / base) ** (torch.arange(0, dim, 2) / dim))
        self.seq_len_cached = None
        self.cos_cached = None
        self.sin_cached = None

    def forward(self, x):
        seq_len = x.shape[1]
        if seq_len != self.seq_len_cached:
            t = torch.arange(seq_len, device=x.device)
            freqs = torch.outer(t, self.inv_freq)
            self.seq_len_cached = seq_len
            self.cos_cached = freqs.cos()
            self.sin_cached = freqs.sin()
        cos, sin = self.cos_cached[None, :, None, :], self.sin_cached[None, :, None, :]
        # apply_rotary_emb(x, cos, sin)
        x1, x2 = x.chunk(2, dim=3)
        y1 = x1 * cos + x2 * sin
        y2 = x1 * (-sin) + x2 * cos
        return torch.cat((y1, y2), 3).type_as(x)

class CausalSelfAttention(nn.Module):

    def __init__(self, dim, n_head):
        super().__init__()
        assert dim % n_head == 0
        self.n_head = n_head
        self.c_q = CastedLinear(dim, dim)
        self.c_k = CastedLinear(dim, dim)
        self.c_v = CastedLinear(dim, dim)
        # value residual lambda
        self.lambdas = nn.Parameter(torch.tensor([0.5, 0.5])) # @Grad62304977
        # rotary embeddings
        self.rotary = Rotary(dim // n_head) # dim // n_head = head_dim
        # output projection
        self.c_proj = CastedLinear(dim, dim)
        self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977

    def forward(self, x: torch.Tensor, vi: torch.Tensor, block_mask: BlockMask) -> torch.Tensor:
        B, T = x.size(0), x.size(1) # batch size, sequence length
        assert B == 1, "Must use batch size = 1 for FlexAttention"
        q: torch.Tensor = self.c_q(x).view(B, T, self.n_head, -1)
        k: torch.Tensor = self.c_k(x).view(B, T, self.n_head, -1)
        v: torch.Tensor = self.c_v(x).view(B, T, self.n_head, -1)
        v = self.lambdas[0] * v + self.lambdas[1] * vi.view_as(v) # @Grad62304977
        q, k = norm(q), norm(k) # QK norm suggested by @Grad62304977
        q, k = self.rotary(q), self.rotary(k)
        y = flex_attention(q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2), block_mask=block_mask)
        y = y.transpose(1, 2).contiguous().view_as(x) # re-assemble all head outputs side by side
        y = self.c_proj(y)
        return y

class MLP(nn.Module):

    def __init__(self, dim: int):
        super().__init__()
        self.c_fc   = CastedLinear(dim, 4 * dim)
        self.c_proj = CastedLinear(4 * dim, dim)
        self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        x = self.c_fc(x)
        x = F.relu(x).square() # https://arxiv.org/abs/2109.08668v2; ~1-2% better than GELU; suggested by @SKYLINEZ007 and @Grad62304977
        x = self.c_proj(x)
        return x

class Block(nn.Module):

    def __init__(self, config):
        super().__init__()
        self.attn = CausalSelfAttention(config.n_embd, config.n_head)
        self.mlp = MLP(config.n_embd)
        self.lambdas = nn.Parameter(torch.tensor([1., 0.]))

    def forward(self, x: torch.Tensor, vi: torch.Tensor, x0: torch.Tensor, block_mask: BlockMask) -> torch.Tensor:
        x = self.lambdas[0] * x + self.lambdas[1] * x0
        x = x + self.attn(norm(x), vi, block_mask)
        x = x + self.mlp(norm(x))
        return x

# -----------------------------------------------------------------------------
# The main GPT-2 model

@dataclass
class GPTConfig:
    vocab_size : int = 50304
    n_layer : int = 12
    n_head : int = 6 # head dim 128 suggested by @Grad62304977
    n_embd : int = 768
    lm_head_softcap : int = 30

class GPT(nn.Module):

    def __init__(self, config: GPTConfig):
        super().__init__()
        self.n_layer = config.n_layer
        self.lm_head_softcap = config.lm_head_softcap

        # U-net design by @brendanh0gan
        self.num_encoder_layers = config.n_layer // 2 # Half of the layers for encoder
        self.num_decoder_layers = config.n_layer - self.num_encoder_layers # Remaining for decoder
        # Add learnable skip connection weights for decoder layers
        self.skip_weights = nn.Parameter(torch.ones(self.num_decoder_layers))

        self.transformer = nn.ModuleDict(dict(
            wte = nn.Embedding(config.vocab_size, config.n_embd),
            # token value embeddings by @KoszarskyB - inspired by @Grad62304977's value residual learning
            # U-net structure on token value embeddings by @leloykun
            vte = nn.Embedding(config.vocab_size, config.n_embd*self.num_encoder_layers),
            h = nn.ModuleList([Block(config) for _ in range(config.n_layer)]),
        ))
        self.lm_head = CastedLinear(config.n_embd, config.vocab_size)
        self.lm_head.weight.data.zero_() # @Grad62304977

    def forward(self, idx: torch.Tensor, target: torch.Tensor, sliding_window: torch.Tensor) -> torch.Tensor:
        BLOCK_SIZE = 128
        assert idx.ndim == 1
        docs = (idx == 50256).cumsum(0)
        docs_low = docs.reshape(-1, BLOCK_SIZE)[:, 0].contiguous()
        docs_high = docs.reshape(-1, BLOCK_SIZE)[:, -1].contiguous()
        def document_sliding_window_causal(b, h, q_idx, kv_idx):
            causal_mask = q_idx >= kv_idx
            document_mask = docs[q_idx] == docs[kv_idx]
            window_mask = q_idx - kv_idx < sliding_window
            return causal_mask & document_mask & window_mask

        S = len(idx)
        def create_sliding_window_causal_mask(S: int, sliding_window: torch.Tensor):
            kv_idx = block_idx = torch.arange(S // BLOCK_SIZE, dtype=torch.int32, device="cuda")
            q_idx = block_idx[:, None]
            causal_mask = q_idx >= kv_idx
            document_mask = (docs_low[q_idx] <= docs_high[kv_idx]) & (docs_low[kv_idx] <= docs_high[q_idx])
            window_mask = q_idx - kv_idx < ((sliding_window + BLOCK_SIZE - 1) // BLOCK_SIZE)
            dense_mask = causal_mask & document_mask & window_mask
            dense_mask = dense_mask.to(torch.int32)
            num_blocks = dense_mask.sum(dim=-1).to(torch.int32)
            indices = torch.argsort(dense_mask, dim=-1, descending=True, stable=True).to(torch.int32)
            num_blocks = num_blocks[None, None, :].contiguous()
            indices = indices[None, None, :].contiguous()
            return BlockMask.from_kv_blocks(num_blocks, indices, BLOCK_SIZE=BLOCK_SIZE, mask_mod=document_sliding_window_causal)
        block_mask = create_sliding_window_causal_mask(S, sliding_window)

        # forward the GPT model itself
        x = self.transformer.wte(idx[None]) # token embeddings of shape (b, t, n_embd)
        x = norm(x) # @Grad62304977
        x0 = x
        vi = self.transformer.vte(idx[None]).chunk(self.num_encoder_layers, dim=-1)

        # Store outputs for U-Net skip connections
        skip_connections = []
        # Encoder pass - process only the first half of the blocks
        for i in range(self.num_encoder_layers):
            x = self.transformer.h[i](x, vi[i], x0, block_mask)
            skip_connections.append(x)
        # Decoder pass - process the remaining blocks with weighted skip connections
        for i in range(self.num_decoder_layers):
            x = x + self.skip_weights[i] * skip_connections.pop()
            # U-net structure on token value embeddings by @leloykun
            x = self.transformer.h[self.num_encoder_layers + i](x, vi[self.num_encoder_layers-1-i], x0, block_mask)

        x = norm(x)
        logits = self.lm_head(x)
        logits = self.lm_head_softcap * torch.tanh(logits / self.lm_head_softcap) # @Grad62304977
        logits = logits.float()
        loss = F.cross_entropy(logits.view(-1, logits.size(-1)), target.view(-1))
        return loss

# -----------------------------------------------------------------------------
# Our own simple Distributed Data Loader

def _peek_data_shard(file: Path):
    # only reads the header, returns header data
    # header is 256 int32
    header = torch.from_file(f"{file}", False, 256, dtype=torch.int32)
    assert header[0] == 20240520, "magic number mismatch in the data .bin file"
    assert header[1] == 1, "unsupported version"
    return int(header[2]) # number of tokens (claimed)

def _load_data_shard(file: Path, ntok: int):
    with file.open("rb") as f:
        tokens = torch.empty(ntok, dtype=torch.uint16, pin_memory=True)
        f.seek(256 * 4)
        nbytes = f.readinto(tokens.numpy())
        assert nbytes == 2 * ntok, "number of tokens read does not match header?"
    return tokens

class DistributedDataLoader:
    def __init__(self, filename_pattern, T, process_rank, num_processes):
        self.process_rank = process_rank
        self.num_processes = num_processes
        self.T = T

        # glob files that match the pattern
        self.files = sorted(Path.cwd().glob(filename_pattern))
        assert len(self.files) > 0, f"did not find any files that match the pattern {filename_pattern}"

        # load and validate all data shards, count number of tokens in total
        self.ntoks = [_peek_data_shard(file) for file in self.files]
        assert min(self.ntoks) >= num_processes * T + 1
        self.ntok_total = sum(self.ntoks)

        self.reset()

    def reset(self):
        self.current_shard = -1
        self.advance()

    def advance(self): # advance to next data shard
        self.current_shard = (self.current_shard + 1) % len(self.files)
        self.current_position = self.process_rank * self.T
        self.tokens = _load_data_shard(self.files[self.current_shard], self.ntoks[self.current_shard])

    def next_batch(self):
        batch_size = self.T * self.num_processes
        buf = self.tokens[self.current_position:self.current_position+self.T+1]
        # host side async is sufficient;
        # no performance improvement was observed when introducing a separate stream.
        x = buf[:-1].to(device="cuda", dtype=torch.int32, non_blocking=True) # inputs
        y = buf[1:].to(device="cuda", dtype=torch.int64, non_blocking=True) # targets
        # advance current position and load next shard if necessary
        self.current_position += batch_size
        if self.current_position + batch_size + 1 >= len(self.tokens):
            self.advance()
        return x, y

# -----------------------------------------------------------------------------
# int main

@dataclass
class Hyperparameters:
    # data hyperparams
    input_bin : str = 'data/fineweb10B/fineweb_train_*.bin' # input .bin to train on
    input_val_bin : str = 'data/fineweb10B/fineweb_val_*.bin' # input .bin to eval validation loss on
    # optimization hyperparams
    batch_size : int = 8 # batch size, in sequences, across all devices
    sequence_length : int = 64*1024 # sequence length, in tokens
    num_iterations : int = 1480 # number of iterations to run
    warmup_iters : int = 0
    cooldown_iters : int = 600 # number of iterations of linear warmup/cooldown for triangular or trapezoidal schedule
    weight_decay : float = 0
    # evaluation and logging hyperparams
    val_loss_every : int = 125 # every how many steps to evaluate val loss? 0 for only at the end
    val_tokens : int = 10485760 # how many tokens of validation data? it's important to keep this fixed for consistent comparisons
    save_every : int = 0 # every how many steps to save the checkpoint? 0 for only at the end
args = Hyperparameters()

# set up DDP (distributed data parallel). torchrun sets this env variable
assert torch.cuda.is_available()
dist.init_process_group(backend='nccl')
ddp_rank = int(os.environ['RANK'])
ddp_local_rank = int(os.environ['LOCAL_RANK'])
ddp_world_size = int(os.environ['WORLD_SIZE'])
device = f'cuda:{ddp_local_rank}'
torch.cuda.set_device(device)
print(f"using device: {device}")
master_process = (ddp_rank == 0) # this process will do logging, checkpointing etc.

# begin logging
logfile = None
if master_process:
    run_id = str(uuid.uuid4())
    logdir = 'logs/%s/' % run_id
    # os.makedirs(logdir, exist_ok=True)
    logfile = 'logs/%s.txt' % run_id
    # create the log file
    with open(logfile, "w") as f:
        # begin the log by printing this file (the Python code)
        f.write(code)
        f.write('='*100 + '\n')
def print0(s, logonly=False):
    if master_process:
        with open(logfile, "a") as f:
            if not logonly:
                print(s)
            f.write(s+'\n')
# log information about the hardware/software environment this is running on
# and print the full `nvidia-smi` to file
print0(f"Running pytorch {torch.version.__version__} compiled for CUDA {torch.version.cuda}\nnvidia-smi:")
import subprocess
result = subprocess.run(['nvidia-smi'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
print0(f'{result.stdout}', logonly=True)
print0('='*100, logonly=True)

# convenience variables
T = args.sequence_length
# calculate the number of steps to take in the val loop.
assert args.val_tokens % (T * ddp_world_size) == 0
val_steps = args.val_tokens // (T * ddp_world_size)
# calculate the steps of gradient accumulation required to attain the desired global batch size.
assert args.batch_size % (ddp_world_size) == 0
train_accumulation_steps = args.batch_size // ddp_world_size
assert train_accumulation_steps == 1

# load tokens
train_loader = DistributedDataLoader(args.input_bin, T, ddp_rank, ddp_world_size)
val_loader = DistributedDataLoader(args.input_val_bin, T, ddp_rank, ddp_world_size)
print0(f"Training DataLoader: total number of tokens: {train_loader.ntok_total} across {len(train_loader.files)} files")
print0(f"Validation DataLoader: total number of tokens: {val_loader.ntok_total} across {len(val_loader.files)} files")
print0('='*100, logonly=True)
x, y = train_loader.next_batch()

# there are only 50257 unique GPT-2 tokens; we extend to nearest multiple of 128 for efficiency. suggested to me by @Grad62304977.
# this originates from Karpathy's experiments.
num_vocab = 50304
model = GPT(GPTConfig(vocab_size=num_vocab, n_layer=12, n_head=6, n_embd=768))
model = model.cuda().bfloat16()
for m in model.modules():
    if isinstance(m, CastedLinear):
        m.float()
if hasattr(config, "coordinate_descent_tuning"):
    config.coordinate_descent_tuning = True # suggested by @Chillee
model = torch.compile(model)
# here we wrap model into DDP container
model = DDP(model, device_ids=[ddp_local_rank])
raw_model = model.module # always contains the "raw" unwrapped model

# init the optimizer(s)
optimizer1 = torch.optim.Adam([raw_model.transformer.wte.weight, raw_model.transformer.vte.weight], lr=0.6, betas=(0.8, 0.95), fused=True)
optimizer2 = torch.optim.Adam([raw_model.lm_head.weight], lr=0.008, betas=(0.8, 0.95), fused=True)
params = list(raw_model.transformer.h.parameters())
matrix_params = [p for p in params if p.ndim == 2]
scalar_params = [p for p in params if p.ndim < 2] + [raw_model.skip_weights]
optimizer3 = Muon(matrix_params, lr=0.05, momentum=0.95)
optimizer4 = torch.optim.Adam(scalar_params, lr=0.04, betas=(0.8, 0.95), fused=True)
optimizers = [optimizer1, optimizer2, optimizer3, optimizer4]
# learning rate decay scheduler (linear warmup and cooldown)
def get_lr(it):
    assert it <= args.num_iterations
    # 1) linear warmup for warmup_iters steps
    if it < args.warmup_iters:
        return (it+1) / args.warmup_iters
    # 2) constant lr for a while
    elif it < args.num_iterations - args.cooldown_iters:
        return 1.0
    # 3) linear cooldown
    else:
        decay_ratio = (args.num_iterations - it) / args.cooldown_iters
        return decay_ratio
schedulers = [torch.optim.lr_scheduler.LambdaLR(opt, get_lr) for opt in optimizers]

sliding_window_size = torch.tensor(64, dtype=torch.int32, device="cuda")
sw_size_prev = 64
# Start training loop
training_time_ms = 0
# start the clock
torch.cuda.synchronize()
t0 = time.perf_counter()
# begin training
for step in range(args.num_iterations + 1):
    last_step = (step == args.num_iterations)
    # This effectively ignores timing first 10 steps, which are slower for weird reasons.
    # Alternately, and slightly more correctly in terms of benchmarking, we could do 10
    # steps with dummy data first, and then re-initialize the model and reset the loader.
    if step == 10:
        training_time_ms = 0
        t0 = time.perf_counter()
    timed_steps = float('nan') if step <= 11 else (step - 10) + 1 # <= 11 to avoid bug in val

    # Set the sliding window size for the current step, in chunks of 64. By @fernbear.bsky.social
    sw_size =  64 * int((64 + (1792 - 64) * step / args.num_iterations) // 64)
    if sw_size != sw_size_prev:
        sliding_window_size.copy_(sw_size, non_blocking=True)
        sw_size_prev = sw_size

    # once in a while evaluate the validation dataset
    if (last_step or (args.val_loss_every > 0 and step % args.val_loss_every == 0)):
        # stop the clock
        torch.cuda.synchronize()
        training_time_ms += 1000 * (time.perf_counter() - t0)
        # run validation batches
        model.eval()
        val_loader.reset()
        val_loss = 0.0
        for _ in range(val_steps):
            with torch.no_grad():
                x_val, y_val = val_loader.next_batch()
                val_loss += model(x_val, y_val, sliding_window=sliding_window_size)
        dist.all_reduce(val_loss, op=dist.ReduceOp.AVG)
        val_loss /= val_steps
        # log val loss to console and to logfile
        print0(f'step:{step}/{args.num_iterations} val_loss:{val_loss:.4f} train_time:{training_time_ms:.0f}ms step_avg:{training_time_ms/(timed_steps-1):.2f}ms')
        # start the clock again
        torch.cuda.synchronize()
        t0 = time.perf_counter()

    if master_process and (last_step or (args.save_every > 0 and step % args.save_every == 0)):
        # stop the clock
        torch.cuda.synchronize()
        training_time_ms += 1000 * (time.perf_counter() - t0)
        # save the state of the training process
        log = dict(step=step, code=code, model=raw_model.state_dict(), optimizers=[opt.state_dict() for opt in optimizers])
        # torch.save(log, 'logs/%s/state_step%06d.pt' % (run_id, step))
        # start the clock again
        torch.cuda.synchronize()
        t0 = time.perf_counter()

    # bit confusing: we want to make sure to eval on 0th iteration
    # but also after the very last iteration. so we loop for step <= num_iterations
    # instead of just < num_iterations (one extra due to <=), only to do
    # the validation/sampling one last time, and then we break right here as we're done.
    if last_step:
        break

    # --------------- TRAINING SECTION BEGIN -----------------
    model.train()
    loss = model(x, y, sliding_window=sliding_window_size)
    loss.backward()
    del loss
    # advance the dataset for the next batch
    x, y = train_loader.next_batch()
    # momentum warmup for Muon
    frac = min(step/300, 1)
    for group in optimizer3.param_groups:
        group['momentum'] = (1 - frac) * 0.85 + frac * 0.95
    # step the optimizers and schedulers
    for opt, sched in zip(optimizers, schedulers):
        opt.step()
        sched.step()
    # null the gradients
    model.zero_grad(set_to_none=True)
    # --------------- TRAINING SECTION END -------------------
    # everything that follows now is just diagnostics, prints, logging, etc.
    approx_time = training_time_ms + 1000 * (time.perf_counter() - t0)
    print0(f"step:{step+1}/{args.num_iterations} train_time:{approx_time:.0f}ms step_avg:{approx_time/timed_steps:.2f}ms")

if master_process:
    print(f"peak memory consumption: {torch.cuda.max_memory_allocated() // 1024 // 1024} MiB")

# -------------------------------------------------------------------------
# clean up nice
dist.destroy_process_group()
====================================================================================================
Running pytorch 2.6.0.dev20241203+cu124 compiled for CUDA 12.4
nvidia-smi:
Sun Dec  8 09:59:16 2024       
+---------------------------------------------------------------------------------------+
| NVIDIA-SMI 535.129.03             Driver Version: 535.129.03   CUDA Version: 12.6     |
|-----------------------------------------+----------------------+----------------------+
| GPU  Name                 Persistence-M | Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp   Perf          Pwr:Usage/Cap |         Memory-Usage | GPU-Util  Compute M. |
|                                         |                      |               MIG M. |
|=========================================+======================+======================|
|   0  NVIDIA H100 80GB HBM3          On  | 00000000:65:02.0 Off |                    0 |
| N/A   36C    P0              74W / 700W |      7MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   1  NVIDIA H100 80GB HBM3          On  | 00000000:67:02.0 Off |                    0 |
| N/A   45C    P0              96W / 700W |     26MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   2  NVIDIA H100 80GB HBM3          On  | 00000000:69:02.0 Off |                    0 |
| N/A   46C    P0             123W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   3  NVIDIA H100 80GB HBM3          On  | 00000000:6B:02.0 Off |                    0 |
| N/A   39C    P0             105W / 700W |     27MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   4  NVIDIA H100 80GB HBM3          On  | 00000000:6F:02.0 Off |                    0 |
| N/A   39C    P0             117W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   5  NVIDIA H100 80GB HBM3          On  | 00000000:71:02.0 Off |                    0 |
| N/A   45C    P0             121W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   6  NVIDIA H100 80GB HBM3          On  | 00000000:73:02.0 Off |                    0 |
| N/A   46C    P0             120W / 700W |     47MiB / 81559MiB |      1%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   7  NVIDIA H100 80GB HBM3          On  | 00000000:75:02.0 Off |                    0 |
| N/A   38C    P0             124W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
                                                                                         
+---------------------------------------------------------------------------------------+
| Processes:                                                                            |
|  GPU   GI   CI        PID   Type   Process name                            GPU Memory |
|        ID   ID                                                             Usage      |
|=======================================================================================|
+---------------------------------------------------------------------------------------+

====================================================================================================
Training DataLoader: total number of tokens: 3200000000 across 32 files
Validation DataLoader: total number of tokens: 100000000 across 1 files
====================================================================================================
step:0/1480 val_loss:10.8258 train_time:0ms step_avg:nanms
step:1/1480 train_time:23353ms step_avg:nanms
step:2/1480 train_time:23439ms step_avg:nanms
step:3/1480 train_time:23577ms step_avg:nanms
step:4/1480 train_time:23717ms step_avg:nanms
step:5/1480 train_time:23857ms step_avg:nanms
step:6/1480 train_time:23997ms step_avg:nanms
step:7/1480 train_time:24137ms step_avg:nanms
step:8/1480 train_time:24279ms step_avg:nanms
step:9/1480 train_time:24426ms step_avg:nanms
step:10/1480 train_time:24571ms step_avg:nanms
step:11/1480 train_time:142ms step_avg:nanms
step:12/1480 train_time:284ms step_avg:nanms
step:13/1480 train_time:425ms step_avg:141.66ms
step:14/1480 train_time:566ms step_avg:141.60ms
step:15/1480 train_time:709ms step_avg:141.78ms
step:16/1480 train_time:854ms step_avg:142.38ms
step:17/1480 train_time:998ms step_avg:142.60ms
step:18/1480 train_time:1142ms step_avg:142.77ms
step:19/1480 train_time:1285ms step_avg:142.73ms
step:20/1480 train_time:1426ms step_avg:142.55ms
step:21/1480 train_time:1567ms step_avg:142.45ms
step:22/1480 train_time:1709ms step_avg:142.40ms
step:23/1480 train_time:1853ms step_avg:142.51ms
step:24/1480 train_time:1996ms step_avg:142.56ms
step:25/1480 train_time:2141ms step_avg:142.73ms
step:26/1480 train_time:2284ms step_avg:142.77ms
step:27/1480 train_time:2425ms step_avg:142.67ms
step:28/1480 train_time:2567ms step_avg:142.60ms
step:29/1480 train_time:2709ms step_avg:142.55ms
step:30/1480 train_time:2850ms step_avg:142.48ms
step:31/1480 train_time:2992ms step_avg:142.47ms
step:32/1480 train_time:3138ms step_avg:142.63ms
step:33/1480 train_time:3282ms step_avg:142.71ms
step:34/1480 train_time:3424ms step_avg:142.65ms
step:35/1480 train_time:3567ms step_avg:142.67ms
step:36/1480 train_time:3708ms step_avg:142.62ms
step:37/1480 train_time:3850ms step_avg:142.58ms
step:38/1480 train_time:3991ms step_avg:142.54ms
step:39/1480 train_time:4137ms step_avg:142.66ms
step:40/1480 train_time:4281ms step_avg:142.71ms
step:41/1480 train_time:4423ms step_avg:142.69ms
step:42/1480 train_time:4566ms step_avg:142.68ms
step:43/1480 train_time:4707ms step_avg:142.63ms
step:44/1480 train_time:4850ms step_avg:142.65ms
step:45/1480 train_time:4991ms step_avg:142.60ms
step:46/1480 train_time:5134ms step_avg:142.61ms
step:47/1480 train_time:5279ms step_avg:142.68ms
step:48/1480 train_time:5422ms step_avg:142.69ms
step:49/1480 train_time:5565ms step_avg:142.69ms
step:50/1480 train_time:5706ms step_avg:142.66ms
step:51/1480 train_time:5848ms step_avg:142.64ms
step:52/1480 train_time:5989ms step_avg:142.60ms
step:53/1480 train_time:6134ms step_avg:142.65ms
step:54/1480 train_time:6280ms step_avg:142.72ms
step:55/1480 train_time:6424ms step_avg:142.76ms
step:56/1480 train_time:6566ms step_avg:142.74ms
step:57/1480 train_time:6706ms step_avg:142.68ms
step:58/1480 train_time:6848ms step_avg:142.66ms
step:59/1480 train_time:6988ms step_avg:142.61ms
step:60/1480 train_time:7130ms step_avg:142.60ms
step:61/1480 train_time:7274ms step_avg:142.63ms
step:62/1480 train_time:7420ms step_avg:142.69ms
step:63/1480 train_time:7563ms step_avg:142.70ms
step:64/1480 train_time:7705ms step_avg:142.69ms
step:65/1480 train_time:7846ms step_avg:142.66ms
step:66/1480 train_time:7987ms step_avg:142.63ms
step:67/1480 train_time:8129ms step_avg:142.61ms
step:68/1480 train_time:8272ms step_avg:142.62ms
step:69/1480 train_time:8416ms step_avg:142.65ms
step:70/1480 train_time:8561ms step_avg:142.68ms
step:71/1480 train_time:8704ms step_avg:142.69ms
step:72/1480 train_time:8847ms step_avg:142.69ms
step:73/1480 train_time:8987ms step_avg:142.65ms
step:74/1480 train_time:9128ms step_avg:142.63ms
step:75/1480 train_time:9270ms step_avg:142.62ms
step:76/1480 train_time:9416ms step_avg:142.66ms
step:77/1480 train_time:9560ms step_avg:142.69ms
step:78/1480 train_time:9704ms step_avg:142.70ms
step:79/1480 train_time:9845ms step_avg:142.69ms
step:80/1480 train_time:9986ms step_avg:142.66ms
step:81/1480 train_time:10127ms step_avg:142.63ms
step:82/1480 train_time:10270ms step_avg:142.64ms
step:83/1480 train_time:10413ms step_avg:142.65ms
step:84/1480 train_time:10558ms step_avg:142.68ms
step:85/1480 train_time:10701ms step_avg:142.68ms
step:86/1480 train_time:10844ms step_avg:142.68ms
step:87/1480 train_time:10985ms step_avg:142.67ms
step:88/1480 train_time:11127ms step_avg:142.66ms
step:89/1480 train_time:11270ms step_avg:142.66ms
step:90/1480 train_time:11412ms step_avg:142.65ms
step:91/1480 train_time:11556ms step_avg:142.67ms
step:92/1480 train_time:11699ms step_avg:142.67ms
step:93/1480 train_time:11842ms step_avg:142.67ms
step:94/1480 train_time:11984ms step_avg:142.66ms
step:95/1480 train_time:12125ms step_avg:142.65ms
step:96/1480 train_time:12267ms step_avg:142.64ms
step:97/1480 train_time:12408ms step_avg:142.62ms
step:98/1480 train_time:12550ms step_avg:142.61ms
step:99/1480 train_time:12691ms step_avg:142.59ms
step:100/1480 train_time:12834ms step_avg:142.60ms
step:101/1480 train_time:12977ms step_avg:142.60ms
step:102/1480 train_time:13120ms step_avg:142.60ms
step:103/1480 train_time:13262ms step_avg:142.61ms
step:104/1480 train_time:13405ms step_avg:142.61ms
step:105/1480 train_time:13547ms step_avg:142.60ms
step:106/1480 train_time:13688ms step_avg:142.59ms
step:107/1480 train_time:13831ms step_avg:142.59ms
step:108/1480 train_time:13975ms step_avg:142.60ms
step:109/1480 train_time:14118ms step_avg:142.61ms
step:110/1480 train_time:14261ms step_avg:142.61ms
step:111/1480 train_time:14406ms step_avg:142.63ms
step:112/1480 train_time:14552ms step_avg:142.67ms
step:113/1480 train_time:14699ms step_avg:142.71ms
step:114/1480 train_time:14846ms step_avg:142.75ms
step:115/1480 train_time:14992ms step_avg:142.78ms
step:116/1480 train_time:15141ms step_avg:142.84ms
step:117/1480 train_time:15288ms step_avg:142.88ms
step:118/1480 train_time:15436ms step_avg:142.93ms
step:119/1480 train_time:15583ms step_avg:142.97ms
step:120/1480 train_time:15729ms step_avg:142.99ms
step:121/1480 train_time:15876ms step_avg:143.02ms
step:122/1480 train_time:16022ms step_avg:143.05ms
step:123/1480 train_time:16169ms step_avg:143.09ms
step:124/1480 train_time:16317ms step_avg:143.13ms
step:125/1480 train_time:16465ms step_avg:143.18ms
step:125/1480 val_loss:4.4193 train_time:16522ms step_avg:143.67ms
step:126/1480 train_time:16619ms step_avg:143.27ms
step:127/1480 train_time:16769ms step_avg:143.32ms
step:128/1480 train_time:16916ms step_avg:143.36ms
step:129/1480 train_time:17062ms step_avg:143.37ms
step:130/1480 train_time:17208ms step_avg:143.40ms
step:131/1480 train_time:17355ms step_avg:143.43ms
step:132/1480 train_time:17502ms step_avg:143.46ms
step:133/1480 train_time:17651ms step_avg:143.50ms
step:134/1480 train_time:17799ms step_avg:143.54ms
step:135/1480 train_time:17946ms step_avg:143.57ms
step:136/1480 train_time:18094ms step_avg:143.60ms
step:137/1480 train_time:18240ms step_avg:143.62ms
step:138/1480 train_time:18385ms step_avg:143.63ms
step:139/1480 train_time:18532ms step_avg:143.66ms
step:140/1480 train_time:18680ms step_avg:143.69ms
step:141/1480 train_time:18827ms step_avg:143.72ms
step:142/1480 train_time:18976ms step_avg:143.76ms
step:143/1480 train_time:19123ms step_avg:143.78ms
step:144/1480 train_time:19271ms step_avg:143.81ms
step:145/1480 train_time:19418ms step_avg:143.84ms
step:146/1480 train_time:19563ms step_avg:143.85ms
step:147/1480 train_time:19711ms step_avg:143.87ms
step:148/1480 train_time:19857ms step_avg:143.89ms
step:149/1480 train_time:20004ms step_avg:143.91ms
step:150/1480 train_time:20153ms step_avg:143.95ms
step:151/1480 train_time:20300ms step_avg:143.97ms
step:152/1480 train_time:20447ms step_avg:143.99ms
step:153/1480 train_time:20595ms step_avg:144.02ms
step:154/1480 train_time:20741ms step_avg:144.03ms
step:155/1480 train_time:20887ms step_avg:144.05ms
step:156/1480 train_time:21035ms step_avg:144.07ms
step:157/1480 train_time:21182ms step_avg:144.09ms
step:158/1480 train_time:21328ms step_avg:144.10ms
step:159/1480 train_time:21476ms step_avg:144.13ms
step:160/1480 train_time:21622ms step_avg:144.14ms
step:161/1480 train_time:21768ms step_avg:144.16ms
step:162/1480 train_time:21915ms step_avg:144.18ms
step:163/1480 train_time:22061ms step_avg:144.19ms
step:164/1480 train_time:22208ms step_avg:144.21ms
step:165/1480 train_time:22355ms step_avg:144.23ms
step:166/1480 train_time:22501ms step_avg:144.24ms
step:167/1480 train_time:22647ms step_avg:144.25ms
step:168/1480 train_time:22795ms step_avg:144.27ms
step:169/1480 train_time:22941ms step_avg:144.28ms
step:170/1480 train_time:23087ms step_avg:144.30ms
step:171/1480 train_time:23234ms step_avg:144.31ms
step:172/1480 train_time:23382ms step_avg:144.33ms
step:173/1480 train_time:23528ms step_avg:144.34ms
step:174/1480 train_time:23676ms step_avg:144.37ms
step:175/1480 train_time:23822ms step_avg:144.38ms
step:176/1480 train_time:23969ms step_avg:144.39ms
step:177/1480 train_time:24117ms step_avg:144.41ms
step:178/1480 train_time:24262ms step_avg:144.42ms
step:179/1480 train_time:24409ms step_avg:144.43ms
step:180/1480 train_time:24557ms step_avg:144.45ms
step:181/1480 train_time:24702ms step_avg:144.46ms
step:182/1480 train_time:24850ms step_avg:144.47ms
step:183/1480 train_time:24997ms step_avg:144.49ms
step:184/1480 train_time:25143ms step_avg:144.50ms
step:185/1480 train_time:25290ms step_avg:144.52ms
step:186/1480 train_time:25437ms step_avg:144.53ms
step:187/1480 train_time:25584ms step_avg:144.54ms
step:188/1480 train_time:25731ms step_avg:144.55ms
step:189/1480 train_time:25879ms step_avg:144.57ms
step:190/1480 train_time:26024ms step_avg:144.58ms
step:191/1480 train_time:26172ms step_avg:144.60ms
step:192/1480 train_time:26319ms step_avg:144.61ms
step:193/1480 train_time:26465ms step_avg:144.62ms
step:194/1480 train_time:26613ms step_avg:144.63ms
step:195/1480 train_time:26760ms step_avg:144.65ms
step:196/1480 train_time:26907ms step_avg:144.66ms
step:197/1480 train_time:27054ms step_avg:144.67ms
step:198/1480 train_time:27201ms step_avg:144.68ms
step:199/1480 train_time:27346ms step_avg:144.69ms
step:200/1480 train_time:27494ms step_avg:144.71ms
step:201/1480 train_time:27640ms step_avg:144.71ms
step:202/1480 train_time:27786ms step_avg:144.72ms
step:203/1480 train_time:27933ms step_avg:144.73ms
step:204/1480 train_time:28081ms step_avg:144.75ms
step:205/1480 train_time:28229ms step_avg:144.76ms
step:206/1480 train_time:28377ms step_avg:144.78ms
step:207/1480 train_time:28522ms step_avg:144.78ms
step:208/1480 train_time:28670ms step_avg:144.80ms
step:209/1480 train_time:28817ms step_avg:144.81ms
step:210/1480 train_time:28962ms step_avg:144.81ms
step:211/1480 train_time:29110ms step_avg:144.83ms
step:212/1480 train_time:29257ms step_avg:144.84ms
step:213/1480 train_time:29404ms step_avg:144.85ms
step:214/1480 train_time:29552ms step_avg:144.86ms
step:215/1480 train_time:29699ms step_avg:144.87ms
step:216/1480 train_time:29844ms step_avg:144.87ms
step:217/1480 train_time:29991ms step_avg:144.88ms
step:218/1480 train_time:30137ms step_avg:144.89ms
step:219/1480 train_time:30282ms step_avg:144.89ms
step:220/1480 train_time:30429ms step_avg:144.90ms
step:221/1480 train_time:30579ms step_avg:144.93ms
step:222/1480 train_time:30730ms step_avg:144.95ms
step:223/1480 train_time:30881ms step_avg:144.98ms
step:224/1480 train_time:31032ms step_avg:145.01ms
step:225/1480 train_time:31183ms step_avg:145.04ms
step:226/1480 train_time:31333ms step_avg:145.06ms
step:227/1480 train_time:31483ms step_avg:145.08ms
step:228/1480 train_time:31634ms step_avg:145.11ms
step:229/1480 train_time:31784ms step_avg:145.13ms
step:230/1480 train_time:31936ms step_avg:145.16ms
step:231/1480 train_time:32086ms step_avg:145.18ms
step:232/1480 train_time:32236ms step_avg:145.21ms
step:233/1480 train_time:32386ms step_avg:145.23ms
step:234/1480 train_time:32537ms step_avg:145.25ms
step:235/1480 train_time:32688ms step_avg:145.28ms
step:236/1480 train_time:32839ms step_avg:145.31ms
step:237/1480 train_time:32990ms step_avg:145.33ms
step:238/1480 train_time:33141ms step_avg:145.35ms
step:239/1480 train_time:33292ms step_avg:145.38ms
step:240/1480 train_time:33442ms step_avg:145.40ms
step:241/1480 train_time:33593ms step_avg:145.43ms
step:242/1480 train_time:33744ms step_avg:145.45ms
step:243/1480 train_time:33894ms step_avg:145.47ms
step:244/1480 train_time:34044ms step_avg:145.49ms
step:245/1480 train_time:34194ms step_avg:145.51ms
step:246/1480 train_time:34344ms step_avg:145.52ms
step:247/1480 train_time:34494ms step_avg:145.54ms
step:248/1480 train_time:34644ms step_avg:145.56ms
step:249/1480 train_time:34796ms step_avg:145.59ms
step:250/1480 train_time:34946ms step_avg:145.61ms
step:250/1480 val_loss:3.9997 train_time:35006ms step_avg:145.86ms
step:251/1480 train_time:35103ms step_avg:145.66ms
step:252/1480 train_time:35255ms step_avg:145.68ms
step:253/1480 train_time:35406ms step_avg:145.70ms
step:254/1480 train_time:35554ms step_avg:145.72ms
step:255/1480 train_time:35705ms step_avg:145.73ms
step:256/1480 train_time:35853ms step_avg:145.75ms
step:257/1480 train_time:36003ms step_avg:145.76ms
step:258/1480 train_time:36156ms step_avg:145.79ms
step:259/1480 train_time:36307ms step_avg:145.81ms
step:260/1480 train_time:36458ms step_avg:145.83ms
step:261/1480 train_time:36609ms step_avg:145.85ms
step:262/1480 train_time:36757ms step_avg:145.86ms
step:263/1480 train_time:36907ms step_avg:145.88ms
step:264/1480 train_time:37056ms step_avg:145.89ms
step:265/1480 train_time:37208ms step_avg:145.91ms
step:266/1480 train_time:37358ms step_avg:145.93ms
step:267/1480 train_time:37509ms step_avg:145.95ms
step:268/1480 train_time:37659ms step_avg:145.96ms
step:269/1480 train_time:37811ms step_avg:145.99ms
step:270/1480 train_time:37961ms step_avg:146.00ms
step:271/1480 train_time:38111ms step_avg:146.02ms
step:272/1480 train_time:38262ms step_avg:146.04ms
step:273/1480 train_time:38413ms step_avg:146.06ms
step:274/1480 train_time:38564ms step_avg:146.08ms
step:275/1480 train_time:38715ms step_avg:146.09ms
step:276/1480 train_time:38865ms step_avg:146.11ms
step:277/1480 train_time:39015ms step_avg:146.12ms
step:278/1480 train_time:39166ms step_avg:146.14ms
step:279/1480 train_time:39316ms step_avg:146.15ms
step:280/1480 train_time:39467ms step_avg:146.17ms
step:281/1480 train_time:39618ms step_avg:146.19ms
step:282/1480 train_time:39769ms step_avg:146.21ms
step:283/1480 train_time:39919ms step_avg:146.22ms
step:284/1480 train_time:40069ms step_avg:146.24ms
step:285/1480 train_time:40220ms step_avg:146.26ms
step:286/1480 train_time:40371ms step_avg:146.27ms
step:287/1480 train_time:40523ms step_avg:146.29ms
step:288/1480 train_time:40673ms step_avg:146.31ms
step:289/1480 train_time:40825ms step_avg:146.33ms
step:290/1480 train_time:40974ms step_avg:146.34ms
step:291/1480 train_time:41127ms step_avg:146.36ms
step:292/1480 train_time:41277ms step_avg:146.37ms
step:293/1480 train_time:41428ms step_avg:146.39ms
step:294/1480 train_time:41578ms step_avg:146.40ms
step:295/1480 train_time:41728ms step_avg:146.41ms
step:296/1480 train_time:41879ms step_avg:146.43ms
step:297/1480 train_time:42029ms step_avg:146.44ms
step:298/1480 train_time:42180ms step_avg:146.46ms
step:299/1480 train_time:42331ms step_avg:146.47ms
step:300/1480 train_time:42482ms step_avg:146.49ms
step:301/1480 train_time:42633ms step_avg:146.50ms
step:302/1480 train_time:42783ms step_avg:146.52ms
step:303/1480 train_time:42934ms step_avg:146.53ms
step:304/1480 train_time:43085ms step_avg:146.55ms
step:305/1480 train_time:43235ms step_avg:146.56ms
step:306/1480 train_time:43386ms step_avg:146.57ms
step:307/1480 train_time:43537ms step_avg:146.59ms
step:308/1480 train_time:43688ms step_avg:146.60ms
step:309/1480 train_time:43837ms step_avg:146.61ms
step:310/1480 train_time:43987ms step_avg:146.62ms
step:311/1480 train_time:44137ms step_avg:146.64ms
step:312/1480 train_time:44288ms step_avg:146.65ms
step:313/1480 train_time:44438ms step_avg:146.66ms
step:314/1480 train_time:44588ms step_avg:146.67ms
step:315/1480 train_time:44738ms step_avg:146.68ms
step:316/1480 train_time:44889ms step_avg:146.70ms
step:317/1480 train_time:45039ms step_avg:146.71ms
step:318/1480 train_time:45191ms step_avg:146.72ms
step:319/1480 train_time:45341ms step_avg:146.73ms
step:320/1480 train_time:45491ms step_avg:146.74ms
step:321/1480 train_time:45640ms step_avg:146.75ms
step:322/1480 train_time:45792ms step_avg:146.77ms
step:323/1480 train_time:45941ms step_avg:146.78ms
step:324/1480 train_time:46092ms step_avg:146.79ms
step:325/1480 train_time:46243ms step_avg:146.80ms
step:326/1480 train_time:46394ms step_avg:146.82ms
step:327/1480 train_time:46544ms step_avg:146.83ms
step:328/1480 train_time:46695ms step_avg:146.84ms
step:329/1480 train_time:46845ms step_avg:146.85ms
step:330/1480 train_time:46998ms step_avg:146.87ms
step:331/1480 train_time:47151ms step_avg:146.89ms
step:332/1480 train_time:47306ms step_avg:146.91ms
step:333/1480 train_time:47460ms step_avg:146.93ms
step:334/1480 train_time:47614ms step_avg:146.96ms
step:335/1480 train_time:47768ms step_avg:146.98ms
step:336/1480 train_time:47921ms step_avg:147.00ms
step:337/1480 train_time:48075ms step_avg:147.02ms
step:338/1480 train_time:48229ms step_avg:147.04ms
step:339/1480 train_time:48382ms step_avg:147.06ms
step:340/1480 train_time:48536ms step_avg:147.08ms
step:341/1480 train_time:48690ms step_avg:147.10ms
step:342/1480 train_time:48845ms step_avg:147.12ms
step:343/1480 train_time:49000ms step_avg:147.15ms
step:344/1480 train_time:49155ms step_avg:147.17ms
step:345/1480 train_time:49308ms step_avg:147.19ms
step:346/1480 train_time:49464ms step_avg:147.21ms
step:347/1480 train_time:49619ms step_avg:147.24ms
step:348/1480 train_time:49773ms step_avg:147.26ms
step:349/1480 train_time:49927ms step_avg:147.28ms
step:350/1480 train_time:50082ms step_avg:147.30ms
step:351/1480 train_time:50236ms step_avg:147.32ms
step:352/1480 train_time:50390ms step_avg:147.34ms
step:353/1480 train_time:50545ms step_avg:147.36ms
step:354/1480 train_time:50698ms step_avg:147.38ms
step:355/1480 train_time:50852ms step_avg:147.40ms
step:356/1480 train_time:51005ms step_avg:147.41ms
step:357/1480 train_time:51159ms step_avg:147.43ms
step:358/1480 train_time:51313ms step_avg:147.45ms
step:359/1480 train_time:51467ms step_avg:147.47ms
step:360/1480 train_time:51622ms step_avg:147.49ms
step:361/1480 train_time:51777ms step_avg:147.51ms
step:362/1480 train_time:51930ms step_avg:147.53ms
step:363/1480 train_time:52084ms step_avg:147.55ms
step:364/1480 train_time:52237ms step_avg:147.56ms
step:365/1480 train_time:52390ms step_avg:147.58ms
step:366/1480 train_time:52543ms step_avg:147.59ms
step:367/1480 train_time:52697ms step_avg:147.61ms
step:368/1480 train_time:52850ms step_avg:147.63ms
step:369/1480 train_time:53005ms step_avg:147.65ms
step:370/1480 train_time:53160ms step_avg:147.67ms
step:371/1480 train_time:53315ms step_avg:147.69ms
step:372/1480 train_time:53468ms step_avg:147.70ms
step:373/1480 train_time:53623ms step_avg:147.72ms
step:374/1480 train_time:53776ms step_avg:147.74ms
step:375/1480 train_time:53929ms step_avg:147.75ms
step:375/1480 val_loss:3.8106 train_time:53989ms step_avg:147.91ms
step:376/1480 train_time:54087ms step_avg:147.78ms
step:377/1480 train_time:54242ms step_avg:147.80ms
step:378/1480 train_time:54395ms step_avg:147.81ms
step:379/1480 train_time:54547ms step_avg:147.83ms
step:380/1480 train_time:54700ms step_avg:147.84ms
step:381/1480 train_time:54851ms step_avg:147.85ms
step:382/1480 train_time:55006ms step_avg:147.87ms
step:383/1480 train_time:55163ms step_avg:147.89ms
step:384/1480 train_time:55318ms step_avg:147.91ms
step:385/1480 train_time:55471ms step_avg:147.92ms
step:386/1480 train_time:55625ms step_avg:147.94ms
step:387/1480 train_time:55777ms step_avg:147.95ms
step:388/1480 train_time:55930ms step_avg:147.96ms
step:389/1480 train_time:56084ms step_avg:147.98ms
step:390/1480 train_time:56239ms step_avg:148.00ms
step:391/1480 train_time:56392ms step_avg:148.01ms
step:392/1480 train_time:56545ms step_avg:148.02ms
step:393/1480 train_time:56700ms step_avg:148.04ms
step:394/1480 train_time:56853ms step_avg:148.05ms
step:395/1480 train_time:57006ms step_avg:148.07ms
step:396/1480 train_time:57159ms step_avg:148.08ms
step:397/1480 train_time:57313ms step_avg:148.09ms
step:398/1480 train_time:57466ms step_avg:148.11ms
step:399/1480 train_time:57622ms step_avg:148.13ms
step:400/1480 train_time:57776ms step_avg:148.14ms
step:401/1480 train_time:57929ms step_avg:148.16ms
step:402/1480 train_time:58082ms step_avg:148.17ms
step:403/1480 train_time:58237ms step_avg:148.18ms
step:404/1480 train_time:58390ms step_avg:148.20ms
step:405/1480 train_time:58544ms step_avg:148.21ms
step:406/1480 train_time:58701ms step_avg:148.23ms
step:407/1480 train_time:58855ms step_avg:148.25ms
step:408/1480 train_time:59008ms step_avg:148.26ms
step:409/1480 train_time:59162ms step_avg:148.27ms
step:410/1480 train_time:59317ms step_avg:148.29ms
step:411/1480 train_time:59470ms step_avg:148.30ms
step:412/1480 train_time:59625ms step_avg:148.32ms
step:413/1480 train_time:59780ms step_avg:148.34ms
step:414/1480 train_time:59932ms step_avg:148.35ms
step:415/1480 train_time:60086ms step_avg:148.36ms
step:416/1480 train_time:60240ms step_avg:148.37ms
step:417/1480 train_time:60394ms step_avg:148.39ms
step:418/1480 train_time:60548ms step_avg:148.40ms
step:419/1480 train_time:60703ms step_avg:148.42ms
step:420/1480 train_time:60856ms step_avg:148.43ms
step:421/1480 train_time:61009ms step_avg:148.44ms
step:422/1480 train_time:61162ms step_avg:148.45ms
step:423/1480 train_time:61315ms step_avg:148.46ms
step:424/1480 train_time:61468ms step_avg:148.47ms
step:425/1480 train_time:61623ms step_avg:148.49ms
step:426/1480 train_time:61778ms step_avg:148.50ms
step:427/1480 train_time:61932ms step_avg:148.52ms
step:428/1480 train_time:62087ms step_avg:148.53ms
step:429/1480 train_time:62240ms step_avg:148.54ms
step:430/1480 train_time:62394ms step_avg:148.56ms
step:431/1480 train_time:62547ms step_avg:148.57ms
step:432/1480 train_time:62701ms step_avg:148.58ms
step:433/1480 train_time:62854ms step_avg:148.59ms
step:434/1480 train_time:63011ms step_avg:148.61ms
step:435/1480 train_time:63162ms step_avg:148.62ms
step:436/1480 train_time:63316ms step_avg:148.63ms
step:437/1480 train_time:63469ms step_avg:148.64ms
step:438/1480 train_time:63623ms step_avg:148.65ms
step:439/1480 train_time:63777ms step_avg:148.67ms
step:440/1480 train_time:63932ms step_avg:148.68ms
step:441/1480 train_time:64089ms step_avg:148.70ms
step:442/1480 train_time:64247ms step_avg:148.72ms
step:443/1480 train_time:64405ms step_avg:148.74ms
step:444/1480 train_time:64562ms step_avg:148.76ms
step:445/1480 train_time:64718ms step_avg:148.78ms
step:446/1480 train_time:64873ms step_avg:148.79ms
step:447/1480 train_time:65029ms step_avg:148.81ms
step:448/1480 train_time:65187ms step_avg:148.83ms
step:449/1480 train_time:65346ms step_avg:148.85ms
step:450/1480 train_time:65504ms step_avg:148.87ms
step:451/1480 train_time:65663ms step_avg:148.90ms
step:452/1480 train_time:65820ms step_avg:148.91ms
step:453/1480 train_time:65975ms step_avg:148.93ms
step:454/1480 train_time:66131ms step_avg:148.94ms
step:455/1480 train_time:66287ms step_avg:148.96ms
step:456/1480 train_time:66444ms step_avg:148.98ms
step:457/1480 train_time:66602ms step_avg:149.00ms
step:458/1480 train_time:66759ms step_avg:149.01ms
step:459/1480 train_time:66918ms step_avg:149.04ms
step:460/1480 train_time:67075ms step_avg:149.06ms
step:461/1480 train_time:67234ms step_avg:149.08ms
step:462/1480 train_time:67391ms step_avg:149.09ms
step:463/1480 train_time:67548ms step_avg:149.11ms
step:464/1480 train_time:67705ms step_avg:149.13ms
step:465/1480 train_time:67863ms step_avg:149.15ms
step:466/1480 train_time:68022ms step_avg:149.17ms
step:467/1480 train_time:68180ms step_avg:149.19ms
step:468/1480 train_time:68336ms step_avg:149.20ms
step:469/1480 train_time:68491ms step_avg:149.22ms
step:470/1480 train_time:68647ms step_avg:149.23ms
step:471/1480 train_time:68805ms step_avg:149.25ms
step:472/1480 train_time:68964ms step_avg:149.27ms
step:473/1480 train_time:69123ms step_avg:149.29ms
step:474/1480 train_time:69279ms step_avg:149.31ms
step:475/1480 train_time:69435ms step_avg:149.32ms
step:476/1480 train_time:69591ms step_avg:149.34ms
step:477/1480 train_time:69748ms step_avg:149.35ms
step:478/1480 train_time:69905ms step_avg:149.37ms
step:479/1480 train_time:70063ms step_avg:149.39ms
step:480/1480 train_time:70223ms step_avg:149.41ms
step:481/1480 train_time:70380ms step_avg:149.43ms
step:482/1480 train_time:70536ms step_avg:149.44ms
step:483/1480 train_time:70691ms step_avg:149.45ms
step:484/1480 train_time:70848ms step_avg:149.47ms
step:485/1480 train_time:71005ms step_avg:149.48ms
step:486/1480 train_time:71163ms step_avg:149.50ms
step:487/1480 train_time:71321ms step_avg:149.52ms
step:488/1480 train_time:71478ms step_avg:149.53ms
step:489/1480 train_time:71634ms step_avg:149.55ms
step:490/1480 train_time:71791ms step_avg:149.56ms
step:491/1480 train_time:71947ms step_avg:149.58ms
step:492/1480 train_time:72103ms step_avg:149.59ms
step:493/1480 train_time:72260ms step_avg:149.61ms
step:494/1480 train_time:72416ms step_avg:149.62ms
step:495/1480 train_time:72573ms step_avg:149.64ms
step:496/1480 train_time:72731ms step_avg:149.65ms
step:497/1480 train_time:72888ms step_avg:149.67ms
step:498/1480 train_time:73046ms step_avg:149.68ms
step:499/1480 train_time:73204ms step_avg:149.70ms
step:500/1480 train_time:73362ms step_avg:149.72ms
step:500/1480 val_loss:3.6873 train_time:73424ms step_avg:149.85ms
step:501/1480 train_time:73523ms step_avg:149.74ms
step:502/1480 train_time:73682ms step_avg:149.76ms
step:503/1480 train_time:73838ms step_avg:149.77ms
step:504/1480 train_time:73994ms step_avg:149.79ms
step:505/1480 train_time:74148ms step_avg:149.79ms
step:506/1480 train_time:74304ms step_avg:149.81ms
step:507/1480 train_time:74462ms step_avg:149.82ms
step:508/1480 train_time:74621ms step_avg:149.84ms
step:509/1480 train_time:74779ms step_avg:149.86ms
step:510/1480 train_time:74937ms step_avg:149.87ms
step:511/1480 train_time:75094ms step_avg:149.89ms
step:512/1480 train_time:75251ms step_avg:149.90ms
step:513/1480 train_time:75406ms step_avg:149.91ms
step:514/1480 train_time:75563ms step_avg:149.93ms
step:515/1480 train_time:75721ms step_avg:149.94ms
step:516/1480 train_time:75881ms step_avg:149.96ms
step:517/1480 train_time:76039ms step_avg:149.98ms
step:518/1480 train_time:76197ms step_avg:149.99ms
step:519/1480 train_time:76353ms step_avg:150.01ms
step:520/1480 train_time:76510ms step_avg:150.02ms
step:521/1480 train_time:76666ms step_avg:150.03ms
step:522/1480 train_time:76823ms step_avg:150.04ms
step:523/1480 train_time:76981ms step_avg:150.06ms
step:524/1480 train_time:77137ms step_avg:150.07ms
step:525/1480 train_time:77295ms step_avg:150.09ms
step:526/1480 train_time:77452ms step_avg:150.10ms
step:527/1480 train_time:77607ms step_avg:150.11ms
step:528/1480 train_time:77764ms step_avg:150.12ms
step:529/1480 train_time:77921ms step_avg:150.14ms
step:530/1480 train_time:78078ms step_avg:150.15ms
step:531/1480 train_time:78236ms step_avg:150.16ms
step:532/1480 train_time:78392ms step_avg:150.18ms
step:533/1480 train_time:78548ms step_avg:150.19ms
step:534/1480 train_time:78705ms step_avg:150.20ms
step:535/1480 train_time:78863ms step_avg:150.22ms
step:536/1480 train_time:79021ms step_avg:150.23ms
step:537/1480 train_time:79179ms step_avg:150.24ms
step:538/1480 train_time:79336ms step_avg:150.26ms
step:539/1480 train_time:79493ms step_avg:150.27ms
step:540/1480 train_time:79649ms step_avg:150.28ms
step:541/1480 train_time:79806ms step_avg:150.29ms
step:542/1480 train_time:79964ms step_avg:150.31ms
step:543/1480 train_time:80122ms step_avg:150.32ms
step:544/1480 train_time:80279ms step_avg:150.34ms
step:545/1480 train_time:80435ms step_avg:150.35ms
step:546/1480 train_time:80591ms step_avg:150.36ms
step:547/1480 train_time:80746ms step_avg:150.37ms
step:548/1480 train_time:80905ms step_avg:150.38ms
step:549/1480 train_time:81062ms step_avg:150.39ms
step:550/1480 train_time:81221ms step_avg:150.41ms
step:551/1480 train_time:81381ms step_avg:150.43ms
step:552/1480 train_time:81541ms step_avg:150.44ms
step:553/1480 train_time:81701ms step_avg:150.46ms
step:554/1480 train_time:81860ms step_avg:150.48ms
step:555/1480 train_time:82021ms step_avg:150.50ms
step:556/1480 train_time:82181ms step_avg:150.51ms
step:557/1480 train_time:82342ms step_avg:150.53ms
step:558/1480 train_time:82503ms step_avg:150.55ms
step:559/1480 train_time:82663ms step_avg:150.57ms
step:560/1480 train_time:82823ms step_avg:150.59ms
step:561/1480 train_time:82981ms step_avg:150.60ms
step:562/1480 train_time:83142ms step_avg:150.62ms
step:563/1480 train_time:83302ms step_avg:150.64ms
step:564/1480 train_time:83461ms step_avg:150.65ms
step:565/1480 train_time:83621ms step_avg:150.67ms
step:566/1480 train_time:83782ms step_avg:150.69ms
step:567/1480 train_time:83942ms step_avg:150.70ms
step:568/1480 train_time:84102ms step_avg:150.72ms
step:569/1480 train_time:84262ms step_avg:150.74ms
step:570/1480 train_time:84420ms step_avg:150.75ms
step:571/1480 train_time:84580ms step_avg:150.77ms
step:572/1480 train_time:84740ms step_avg:150.78ms
step:573/1480 train_time:84901ms step_avg:150.80ms
step:574/1480 train_time:85061ms step_avg:150.82ms
step:575/1480 train_time:85223ms step_avg:150.84ms
step:576/1480 train_time:85384ms step_avg:150.86ms
step:577/1480 train_time:85544ms step_avg:150.87ms
step:578/1480 train_time:85703ms step_avg:150.89ms
step:579/1480 train_time:85863ms step_avg:150.90ms
step:580/1480 train_time:86024ms step_avg:150.92ms
step:581/1480 train_time:86185ms step_avg:150.94ms
step:582/1480 train_time:86344ms step_avg:150.95ms
step:583/1480 train_time:86505ms step_avg:150.97ms
step:584/1480 train_time:86664ms step_avg:150.98ms
step:585/1480 train_time:86823ms step_avg:151.00ms
step:586/1480 train_time:86984ms step_avg:151.01ms
step:587/1480 train_time:87143ms step_avg:151.03ms
step:588/1480 train_time:87304ms step_avg:151.04ms
step:589/1480 train_time:87465ms step_avg:151.06ms
step:590/1480 train_time:87626ms step_avg:151.08ms
step:591/1480 train_time:87784ms step_avg:151.09ms
step:592/1480 train_time:87943ms step_avg:151.11ms
step:593/1480 train_time:88105ms step_avg:151.12ms
step:594/1480 train_time:88264ms step_avg:151.14ms
step:595/1480 train_time:88427ms step_avg:151.16ms
step:596/1480 train_time:88588ms step_avg:151.17ms
step:597/1480 train_time:88747ms step_avg:151.19ms
step:598/1480 train_time:88905ms step_avg:151.20ms
step:599/1480 train_time:89064ms step_avg:151.21ms
step:600/1480 train_time:89224ms step_avg:151.23ms
step:601/1480 train_time:89385ms step_avg:151.24ms
step:602/1480 train_time:89544ms step_avg:151.26ms
step:603/1480 train_time:89705ms step_avg:151.27ms
step:604/1480 train_time:89864ms step_avg:151.29ms
step:605/1480 train_time:90022ms step_avg:151.30ms
step:606/1480 train_time:90185ms step_avg:151.32ms
step:607/1480 train_time:90346ms step_avg:151.33ms
step:608/1480 train_time:90506ms step_avg:151.35ms
step:609/1480 train_time:90667ms step_avg:151.36ms
step:610/1480 train_time:90825ms step_avg:151.38ms
step:611/1480 train_time:90985ms step_avg:151.39ms
step:612/1480 train_time:91144ms step_avg:151.40ms
step:613/1480 train_time:91305ms step_avg:151.42ms
step:614/1480 train_time:91465ms step_avg:151.43ms
step:615/1480 train_time:91624ms step_avg:151.44ms
step:616/1480 train_time:91784ms step_avg:151.46ms
step:617/1480 train_time:91943ms step_avg:151.47ms
step:618/1480 train_time:92103ms step_avg:151.49ms
step:619/1480 train_time:92264ms step_avg:151.50ms
step:620/1480 train_time:92424ms step_avg:151.52ms
step:621/1480 train_time:92585ms step_avg:151.53ms
step:622/1480 train_time:92745ms step_avg:151.54ms
step:623/1480 train_time:92905ms step_avg:151.56ms
step:624/1480 train_time:93064ms step_avg:151.57ms
step:625/1480 train_time:93224ms step_avg:151.58ms
step:625/1480 val_loss:3.6065 train_time:93289ms step_avg:151.69ms
step:626/1480 train_time:93388ms step_avg:151.60ms
step:627/1480 train_time:93548ms step_avg:151.62ms
step:628/1480 train_time:93706ms step_avg:151.63ms
step:629/1480 train_time:93863ms step_avg:151.64ms
step:630/1480 train_time:94020ms step_avg:151.64ms
step:631/1480 train_time:94178ms step_avg:151.66ms
step:632/1480 train_time:94337ms step_avg:151.67ms
step:633/1480 train_time:94497ms step_avg:151.68ms
step:634/1480 train_time:94657ms step_avg:151.69ms
step:635/1480 train_time:94815ms step_avg:151.70ms
step:636/1480 train_time:94975ms step_avg:151.72ms
step:637/1480 train_time:95134ms step_avg:151.73ms
step:638/1480 train_time:95293ms step_avg:151.74ms
step:639/1480 train_time:95452ms step_avg:151.75ms
step:640/1480 train_time:95612ms step_avg:151.77ms
step:641/1480 train_time:95773ms step_avg:151.78ms
step:642/1480 train_time:95932ms step_avg:151.79ms
step:643/1480 train_time:96092ms step_avg:151.80ms
step:644/1480 train_time:96252ms step_avg:151.82ms
step:645/1480 train_time:96414ms step_avg:151.83ms
step:646/1480 train_time:96572ms step_avg:151.84ms
step:647/1480 train_time:96732ms step_avg:151.86ms
step:648/1480 train_time:96893ms step_avg:151.87ms
step:649/1480 train_time:97052ms step_avg:151.88ms
step:650/1480 train_time:97213ms step_avg:151.90ms
step:651/1480 train_time:97373ms step_avg:151.91ms
step:652/1480 train_time:97533ms step_avg:151.92ms
step:653/1480 train_time:97692ms step_avg:151.93ms
step:654/1480 train_time:97853ms step_avg:151.95ms
step:655/1480 train_time:98012ms step_avg:151.96ms
step:656/1480 train_time:98172ms step_avg:151.97ms
step:657/1480 train_time:98332ms step_avg:151.98ms
step:658/1480 train_time:98492ms step_avg:151.99ms
step:659/1480 train_time:98654ms step_avg:152.01ms
step:660/1480 train_time:98816ms step_avg:152.02ms
step:661/1480 train_time:98979ms step_avg:152.04ms
step:662/1480 train_time:99139ms step_avg:152.05ms
step:663/1480 train_time:99299ms step_avg:152.07ms
step:664/1480 train_time:99460ms step_avg:152.08ms
step:665/1480 train_time:99620ms step_avg:152.09ms
step:666/1480 train_time:99780ms step_avg:152.10ms
step:667/1480 train_time:99943ms step_avg:152.12ms
step:668/1480 train_time:100105ms step_avg:152.13ms
step:669/1480 train_time:100269ms step_avg:152.15ms
step:670/1480 train_time:100430ms step_avg:152.17ms
step:671/1480 train_time:100591ms step_avg:152.18ms
step:672/1480 train_time:100753ms step_avg:152.19ms
step:673/1480 train_time:100915ms step_avg:152.21ms
step:674/1480 train_time:101077ms step_avg:152.22ms
step:675/1480 train_time:101239ms step_avg:152.24ms
step:676/1480 train_time:101401ms step_avg:152.25ms
step:677/1480 train_time:101561ms step_avg:152.27ms
step:678/1480 train_time:101720ms step_avg:152.28ms
step:679/1480 train_time:101882ms step_avg:152.29ms
step:680/1480 train_time:102045ms step_avg:152.31ms
step:681/1480 train_time:102207ms step_avg:152.32ms
step:682/1480 train_time:102371ms step_avg:152.34ms
step:683/1480 train_time:102534ms step_avg:152.35ms
step:684/1480 train_time:102695ms step_avg:152.37ms
step:685/1480 train_time:102857ms step_avg:152.38ms
step:686/1480 train_time:103017ms step_avg:152.39ms
step:687/1480 train_time:103177ms step_avg:152.40ms
step:688/1480 train_time:103340ms step_avg:152.42ms
step:689/1480 train_time:103503ms step_avg:152.43ms
step:690/1480 train_time:103668ms step_avg:152.45ms
step:691/1480 train_time:103832ms step_avg:152.47ms
step:692/1480 train_time:103994ms step_avg:152.48ms
step:693/1480 train_time:104155ms step_avg:152.50ms
step:694/1480 train_time:104316ms step_avg:152.51ms
step:695/1480 train_time:104476ms step_avg:152.52ms
step:696/1480 train_time:104637ms step_avg:152.53ms
step:697/1480 train_time:104799ms step_avg:152.55ms
step:698/1480 train_time:104960ms step_avg:152.56ms
step:699/1480 train_time:105123ms step_avg:152.57ms
step:700/1480 train_time:105286ms step_avg:152.59ms
step:701/1480 train_time:105448ms step_avg:152.60ms
step:702/1480 train_time:105609ms step_avg:152.61ms
step:703/1480 train_time:105770ms step_avg:152.63ms
step:704/1480 train_time:105933ms step_avg:152.64ms
step:705/1480 train_time:106096ms step_avg:152.66ms
step:706/1480 train_time:106260ms step_avg:152.67ms
step:707/1480 train_time:106422ms step_avg:152.69ms
step:708/1480 train_time:106584ms step_avg:152.70ms
step:709/1480 train_time:106747ms step_avg:152.71ms
step:710/1480 train_time:106907ms step_avg:152.72ms
step:711/1480 train_time:107071ms step_avg:152.74ms
step:712/1480 train_time:107237ms step_avg:152.76ms
step:713/1480 train_time:107399ms step_avg:152.77ms
step:714/1480 train_time:107559ms step_avg:152.78ms
step:715/1480 train_time:107718ms step_avg:152.79ms
step:716/1480 train_time:107877ms step_avg:152.80ms
step:717/1480 train_time:108039ms step_avg:152.81ms
step:718/1480 train_time:108198ms step_avg:152.82ms
step:719/1480 train_time:108358ms step_avg:152.83ms
step:720/1480 train_time:108522ms step_avg:152.85ms
step:721/1480 train_time:108682ms step_avg:152.86ms
step:722/1480 train_time:108841ms step_avg:152.87ms
step:723/1480 train_time:109000ms step_avg:152.88ms
step:724/1480 train_time:109163ms step_avg:152.89ms
step:725/1480 train_time:109327ms step_avg:152.90ms
step:726/1480 train_time:109491ms step_avg:152.92ms
step:727/1480 train_time:109655ms step_avg:152.94ms
step:728/1480 train_time:109815ms step_avg:152.95ms
step:729/1480 train_time:109976ms step_avg:152.96ms
step:730/1480 train_time:110138ms step_avg:152.97ms
step:731/1480 train_time:110298ms step_avg:152.98ms
step:732/1480 train_time:110457ms step_avg:152.99ms
step:733/1480 train_time:110617ms step_avg:153.00ms
step:734/1480 train_time:110779ms step_avg:153.01ms
step:735/1480 train_time:110940ms step_avg:153.02ms
step:736/1480 train_time:111100ms step_avg:153.03ms
step:737/1480 train_time:111261ms step_avg:153.04ms
step:738/1480 train_time:111421ms step_avg:153.05ms
step:739/1480 train_time:111580ms step_avg:153.06ms
step:740/1480 train_time:111745ms step_avg:153.08ms
step:741/1480 train_time:111908ms step_avg:153.09ms
step:742/1480 train_time:112071ms step_avg:153.10ms
step:743/1480 train_time:112233ms step_avg:153.11ms
step:744/1480 train_time:112397ms step_avg:153.13ms
step:745/1480 train_time:112561ms step_avg:153.14ms
step:746/1480 train_time:112720ms step_avg:153.15ms
step:747/1480 train_time:112882ms step_avg:153.16ms
step:748/1480 train_time:113049ms step_avg:153.18ms
step:749/1480 train_time:113212ms step_avg:153.20ms
step:750/1480 train_time:113374ms step_avg:153.21ms
step:750/1480 val_loss:3.5498 train_time:113437ms step_avg:153.29ms
step:751/1480 train_time:113537ms step_avg:153.22ms
step:752/1480 train_time:113701ms step_avg:153.24ms
step:753/1480 train_time:113863ms step_avg:153.25ms
step:754/1480 train_time:114025ms step_avg:153.26ms
step:755/1480 train_time:114186ms step_avg:153.27ms
step:756/1480 train_time:114348ms step_avg:153.28ms
step:757/1480 train_time:114511ms step_avg:153.30ms
step:758/1480 train_time:114671ms step_avg:153.30ms
step:759/1480 train_time:114833ms step_avg:153.32ms
step:760/1480 train_time:114993ms step_avg:153.32ms
step:761/1480 train_time:115155ms step_avg:153.34ms
step:762/1480 train_time:115315ms step_avg:153.34ms
step:763/1480 train_time:115477ms step_avg:153.36ms
step:764/1480 train_time:115638ms step_avg:153.37ms
step:765/1480 train_time:115799ms step_avg:153.38ms
step:766/1480 train_time:115963ms step_avg:153.39ms
step:767/1480 train_time:116127ms step_avg:153.40ms
step:768/1480 train_time:116289ms step_avg:153.42ms
step:769/1480 train_time:116452ms step_avg:153.43ms
step:770/1480 train_time:116615ms step_avg:153.44ms
step:771/1480 train_time:116778ms step_avg:153.45ms
step:772/1480 train_time:116939ms step_avg:153.46ms
step:773/1480 train_time:117103ms step_avg:153.48ms
step:774/1480 train_time:117267ms step_avg:153.49ms
step:775/1480 train_time:117431ms step_avg:153.50ms
step:776/1480 train_time:117595ms step_avg:153.52ms
step:777/1480 train_time:117761ms step_avg:153.53ms
step:778/1480 train_time:117925ms step_avg:153.55ms
step:779/1480 train_time:118087ms step_avg:153.56ms
step:780/1480 train_time:118251ms step_avg:153.57ms
step:781/1480 train_time:118414ms step_avg:153.59ms
step:782/1480 train_time:118576ms step_avg:153.60ms
step:783/1480 train_time:118737ms step_avg:153.60ms
step:784/1480 train_time:118902ms step_avg:153.62ms
step:785/1480 train_time:119066ms step_avg:153.63ms
step:786/1480 train_time:119232ms step_avg:153.65ms
step:787/1480 train_time:119395ms step_avg:153.66ms
step:788/1480 train_time:119560ms step_avg:153.68ms
step:789/1480 train_time:119723ms step_avg:153.69ms
step:790/1480 train_time:119888ms step_avg:153.70ms
step:791/1480 train_time:120054ms step_avg:153.72ms
step:792/1480 train_time:120220ms step_avg:153.73ms
step:793/1480 train_time:120383ms step_avg:153.75ms
step:794/1480 train_time:120547ms step_avg:153.76ms
step:795/1480 train_time:120711ms step_avg:153.77ms
step:796/1480 train_time:120876ms step_avg:153.79ms
step:797/1480 train_time:121040ms step_avg:153.80ms
step:798/1480 train_time:121205ms step_avg:153.81ms
step:799/1480 train_time:121371ms step_avg:153.83ms
step:800/1480 train_time:121534ms step_avg:153.84ms
step:801/1480 train_time:121696ms step_avg:153.85ms
step:802/1480 train_time:121865ms step_avg:153.87ms
step:803/1480 train_time:122029ms step_avg:153.88ms
step:804/1480 train_time:122191ms step_avg:153.89ms
step:805/1480 train_time:122356ms step_avg:153.91ms
step:806/1480 train_time:122517ms step_avg:153.92ms
step:807/1480 train_time:122679ms step_avg:153.93ms
step:808/1480 train_time:122846ms step_avg:153.94ms
step:809/1480 train_time:123009ms step_avg:153.95ms
step:810/1480 train_time:123170ms step_avg:153.96ms
step:811/1480 train_time:123332ms step_avg:153.97ms
step:812/1480 train_time:123496ms step_avg:153.98ms
step:813/1480 train_time:123655ms step_avg:153.99ms
step:814/1480 train_time:123819ms step_avg:154.00ms
step:815/1480 train_time:123980ms step_avg:154.01ms
step:816/1480 train_time:124146ms step_avg:154.03ms
step:817/1480 train_time:124311ms step_avg:154.04ms
step:818/1480 train_time:124472ms step_avg:154.05ms
step:819/1480 train_time:124636ms step_avg:154.06ms
step:820/1480 train_time:124800ms step_avg:154.07ms
step:821/1480 train_time:124962ms step_avg:154.08ms
step:822/1480 train_time:125126ms step_avg:154.10ms
step:823/1480 train_time:125289ms step_avg:154.11ms
step:824/1480 train_time:125451ms step_avg:154.12ms
step:825/1480 train_time:125615ms step_avg:154.13ms
step:826/1480 train_time:125782ms step_avg:154.14ms
step:827/1480 train_time:125947ms step_avg:154.16ms
step:828/1480 train_time:126110ms step_avg:154.17ms
step:829/1480 train_time:126273ms step_avg:154.18ms
step:830/1480 train_time:126436ms step_avg:154.19ms
step:831/1480 train_time:126602ms step_avg:154.20ms
step:832/1480 train_time:126766ms step_avg:154.22ms
step:833/1480 train_time:126931ms step_avg:154.23ms
step:834/1480 train_time:127094ms step_avg:154.24ms
step:835/1480 train_time:127257ms step_avg:154.25ms
step:836/1480 train_time:127424ms step_avg:154.27ms
step:837/1480 train_time:127586ms step_avg:154.28ms
step:838/1480 train_time:127752ms step_avg:154.29ms
step:839/1480 train_time:127914ms step_avg:154.30ms
step:840/1480 train_time:128075ms step_avg:154.31ms
step:841/1480 train_time:128234ms step_avg:154.31ms
step:842/1480 train_time:128399ms step_avg:154.33ms
step:843/1480 train_time:128561ms step_avg:154.34ms
step:844/1480 train_time:128725ms step_avg:154.35ms
step:845/1480 train_time:128890ms step_avg:154.36ms
step:846/1480 train_time:129053ms step_avg:154.37ms
step:847/1480 train_time:129217ms step_avg:154.38ms
step:848/1480 train_time:129379ms step_avg:154.39ms
step:849/1480 train_time:129542ms step_avg:154.40ms
step:850/1480 train_time:129706ms step_avg:154.41ms
step:851/1480 train_time:129869ms step_avg:154.42ms
step:852/1480 train_time:130032ms step_avg:154.43ms
step:853/1480 train_time:130193ms step_avg:154.44ms
step:854/1480 train_time:130359ms step_avg:154.45ms
step:855/1480 train_time:130523ms step_avg:154.47ms
step:856/1480 train_time:130686ms step_avg:154.48ms
step:857/1480 train_time:130851ms step_avg:154.49ms
step:858/1480 train_time:131017ms step_avg:154.50ms
step:859/1480 train_time:131182ms step_avg:154.51ms
step:860/1480 train_time:131344ms step_avg:154.52ms
step:861/1480 train_time:131510ms step_avg:154.54ms
step:862/1480 train_time:131678ms step_avg:154.55ms
step:863/1480 train_time:131847ms step_avg:154.57ms
step:864/1480 train_time:132011ms step_avg:154.58ms
step:865/1480 train_time:132172ms step_avg:154.59ms
step:866/1480 train_time:132337ms step_avg:154.60ms
step:867/1480 train_time:132501ms step_avg:154.61ms
step:868/1480 train_time:132664ms step_avg:154.62ms
step:869/1480 train_time:132827ms step_avg:154.63ms
step:870/1480 train_time:132991ms step_avg:154.64ms
step:871/1480 train_time:133154ms step_avg:154.65ms
step:872/1480 train_time:133317ms step_avg:154.66ms
step:873/1480 train_time:133479ms step_avg:154.67ms
step:874/1480 train_time:133647ms step_avg:154.68ms
step:875/1480 train_time:133812ms step_avg:154.70ms
step:875/1480 val_loss:3.5044 train_time:133876ms step_avg:154.77ms
step:876/1480 train_time:133977ms step_avg:154.71ms
step:877/1480 train_time:134141ms step_avg:154.72ms
step:878/1480 train_time:134303ms step_avg:154.73ms
step:879/1480 train_time:134467ms step_avg:154.74ms
step:880/1480 train_time:134631ms step_avg:154.75ms
step:881/1480 train_time:134794ms step_avg:154.76ms
step:882/1480 train_time:134959ms step_avg:154.77ms
step:883/1480 train_time:135125ms step_avg:154.78ms
step:884/1480 train_time:135294ms step_avg:154.80ms
step:885/1480 train_time:135459ms step_avg:154.81ms
step:886/1480 train_time:135626ms step_avg:154.82ms
step:887/1480 train_time:135795ms step_avg:154.84ms
step:888/1480 train_time:135968ms step_avg:154.86ms
step:889/1480 train_time:136136ms step_avg:154.88ms
step:890/1480 train_time:136298ms step_avg:154.88ms
step:891/1480 train_time:136463ms step_avg:154.90ms
step:892/1480 train_time:136628ms step_avg:154.91ms
step:893/1480 train_time:136792ms step_avg:154.92ms
step:894/1480 train_time:136959ms step_avg:154.93ms
step:895/1480 train_time:137125ms step_avg:154.94ms
step:896/1480 train_time:137291ms step_avg:154.96ms
step:897/1480 train_time:137458ms step_avg:154.97ms
step:898/1480 train_time:137626ms step_avg:154.98ms
step:899/1480 train_time:137791ms step_avg:155.00ms
step:900/1480 train_time:137955ms step_avg:155.01ms
step:901/1480 train_time:138118ms step_avg:155.01ms
step:902/1480 train_time:138281ms step_avg:155.02ms
step:903/1480 train_time:138453ms step_avg:155.04ms
step:904/1480 train_time:138618ms step_avg:155.05ms
step:905/1480 train_time:138779ms step_avg:155.06ms
step:906/1480 train_time:138945ms step_avg:155.07ms
step:907/1480 train_time:139113ms step_avg:155.09ms
step:908/1480 train_time:139277ms step_avg:155.10ms
step:909/1480 train_time:139441ms step_avg:155.11ms
step:910/1480 train_time:139612ms step_avg:155.12ms
step:911/1480 train_time:139777ms step_avg:155.14ms
step:912/1480 train_time:139942ms step_avg:155.15ms
step:913/1480 train_time:140111ms step_avg:155.16ms
step:914/1480 train_time:140277ms step_avg:155.17ms
step:915/1480 train_time:140447ms step_avg:155.19ms
step:916/1480 train_time:140611ms step_avg:155.20ms
step:917/1480 train_time:140775ms step_avg:155.21ms
step:918/1480 train_time:140942ms step_avg:155.22ms
step:919/1480 train_time:141112ms step_avg:155.24ms
step:920/1480 train_time:141278ms step_avg:155.25ms
step:921/1480 train_time:141443ms step_avg:155.26ms
step:922/1480 train_time:141611ms step_avg:155.28ms
step:923/1480 train_time:141774ms step_avg:155.28ms
step:924/1480 train_time:141939ms step_avg:155.29ms
step:925/1480 train_time:142104ms step_avg:155.31ms
step:926/1480 train_time:142268ms step_avg:155.31ms
step:927/1480 train_time:142433ms step_avg:155.32ms
step:928/1480 train_time:142598ms step_avg:155.34ms
step:929/1480 train_time:142763ms step_avg:155.35ms
step:930/1480 train_time:142929ms step_avg:155.36ms
step:931/1480 train_time:143094ms step_avg:155.37ms
step:932/1480 train_time:143260ms step_avg:155.38ms
step:933/1480 train_time:143428ms step_avg:155.39ms
step:934/1480 train_time:143595ms step_avg:155.41ms
step:935/1480 train_time:143766ms step_avg:155.42ms
step:936/1480 train_time:143934ms step_avg:155.44ms
step:937/1480 train_time:144103ms step_avg:155.45ms
step:938/1480 train_time:144267ms step_avg:155.46ms
step:939/1480 train_time:144437ms step_avg:155.48ms
step:940/1480 train_time:144603ms step_avg:155.49ms
step:941/1480 train_time:144766ms step_avg:155.50ms
step:942/1480 train_time:144932ms step_avg:155.51ms
step:943/1480 train_time:145103ms step_avg:155.52ms
step:944/1480 train_time:145275ms step_avg:155.54ms
step:945/1480 train_time:145437ms step_avg:155.55ms
step:946/1480 train_time:145606ms step_avg:155.56ms
step:947/1480 train_time:145774ms step_avg:155.58ms
step:948/1480 train_time:145940ms step_avg:155.59ms
step:949/1480 train_time:146104ms step_avg:155.60ms
step:950/1480 train_time:146268ms step_avg:155.60ms
step:951/1480 train_time:146436ms step_avg:155.62ms
step:952/1480 train_time:146600ms step_avg:155.63ms
step:953/1480 train_time:146770ms step_avg:155.64ms
step:954/1480 train_time:146937ms step_avg:155.65ms
step:955/1480 train_time:147100ms step_avg:155.66ms
step:956/1480 train_time:147266ms step_avg:155.67ms
step:957/1480 train_time:147434ms step_avg:155.69ms
step:958/1480 train_time:147603ms step_avg:155.70ms
step:959/1480 train_time:147767ms step_avg:155.71ms
step:960/1480 train_time:147935ms step_avg:155.72ms
step:961/1480 train_time:148100ms step_avg:155.73ms
step:962/1480 train_time:148264ms step_avg:155.74ms
step:963/1480 train_time:148431ms step_avg:155.75ms
step:964/1480 train_time:148600ms step_avg:155.76ms
step:965/1480 train_time:148763ms step_avg:155.77ms
step:966/1480 train_time:148929ms step_avg:155.78ms
step:967/1480 train_time:149092ms step_avg:155.79ms
step:968/1480 train_time:149257ms step_avg:155.80ms
step:969/1480 train_time:149424ms step_avg:155.81ms
step:970/1480 train_time:149587ms step_avg:155.82ms
step:971/1480 train_time:149753ms step_avg:155.83ms
step:972/1480 train_time:149917ms step_avg:155.84ms
step:973/1480 train_time:150081ms step_avg:155.85ms
step:974/1480 train_time:150253ms step_avg:155.86ms
step:975/1480 train_time:150418ms step_avg:155.87ms
step:976/1480 train_time:150582ms step_avg:155.88ms
step:977/1480 train_time:150747ms step_avg:155.89ms
step:978/1480 train_time:150912ms step_avg:155.90ms
step:979/1480 train_time:151078ms step_avg:155.91ms
step:980/1480 train_time:151244ms step_avg:155.92ms
step:981/1480 train_time:151413ms step_avg:155.94ms
step:982/1480 train_time:151577ms step_avg:155.94ms
step:983/1480 train_time:151740ms step_avg:155.95ms
step:984/1480 train_time:151904ms step_avg:155.96ms
step:985/1480 train_time:152073ms step_avg:155.97ms
step:986/1480 train_time:152239ms step_avg:155.98ms
step:987/1480 train_time:152401ms step_avg:155.99ms
step:988/1480 train_time:152571ms step_avg:156.00ms
step:989/1480 train_time:152737ms step_avg:156.01ms
step:990/1480 train_time:152905ms step_avg:156.03ms
step:991/1480 train_time:153074ms step_avg:156.04ms
step:992/1480 train_time:153247ms step_avg:156.06ms
step:993/1480 train_time:153422ms step_avg:156.08ms
step:994/1480 train_time:153587ms step_avg:156.08ms
step:995/1480 train_time:153751ms step_avg:156.09ms
step:996/1480 train_time:153913ms step_avg:156.10ms
step:997/1480 train_time:154078ms step_avg:156.11ms
step:998/1480 train_time:154242ms step_avg:156.12ms
step:999/1480 train_time:154409ms step_avg:156.13ms
step:1000/1480 train_time:154578ms step_avg:156.14ms
step:1000/1480 val_loss:3.4408 train_time:154645ms step_avg:156.21ms
step:1001/1480 train_time:154746ms step_avg:156.15ms
step:1002/1480 train_time:154913ms step_avg:156.16ms
step:1003/1480 train_time:155087ms step_avg:156.18ms
step:1004/1480 train_time:155256ms step_avg:156.19ms
step:1005/1480 train_time:155426ms step_avg:156.21ms
step:1006/1480 train_time:155592ms step_avg:156.22ms
step:1007/1480 train_time:155758ms step_avg:156.23ms
step:1008/1480 train_time:155926ms step_avg:156.24ms
step:1009/1480 train_time:156102ms step_avg:156.26ms
step:1010/1480 train_time:156268ms step_avg:156.27ms
step:1011/1480 train_time:156432ms step_avg:156.28ms
step:1012/1480 train_time:156597ms step_avg:156.28ms
step:1013/1480 train_time:156767ms step_avg:156.30ms
step:1014/1480 train_time:156933ms step_avg:156.31ms
step:1015/1480 train_time:157105ms step_avg:156.32ms
step:1016/1480 train_time:157271ms step_avg:156.33ms
step:1017/1480 train_time:157442ms step_avg:156.35ms
step:1018/1480 train_time:157611ms step_avg:156.36ms
step:1019/1480 train_time:157782ms step_avg:156.37ms
step:1020/1480 train_time:157950ms step_avg:156.39ms
step:1021/1480 train_time:158115ms step_avg:156.39ms
step:1022/1480 train_time:158283ms step_avg:156.41ms
step:1023/1480 train_time:158448ms step_avg:156.41ms
step:1024/1480 train_time:158615ms step_avg:156.42ms
step:1025/1480 train_time:158787ms step_avg:156.44ms
step:1026/1480 train_time:158952ms step_avg:156.45ms
step:1027/1480 train_time:159118ms step_avg:156.46ms
step:1028/1480 train_time:159291ms step_avg:156.47ms
step:1029/1480 train_time:159465ms step_avg:156.49ms
step:1030/1480 train_time:159631ms step_avg:156.50ms
step:1031/1480 train_time:159795ms step_avg:156.51ms
step:1032/1480 train_time:159968ms step_avg:156.52ms
step:1033/1480 train_time:160133ms step_avg:156.53ms
step:1034/1480 train_time:160303ms step_avg:156.55ms
step:1035/1480 train_time:160471ms step_avg:156.56ms
step:1036/1480 train_time:160638ms step_avg:156.57ms
step:1037/1480 train_time:160806ms step_avg:156.58ms
step:1038/1480 train_time:160975ms step_avg:156.59ms
step:1039/1480 train_time:161146ms step_avg:156.60ms
step:1040/1480 train_time:161311ms step_avg:156.61ms
step:1041/1480 train_time:161479ms step_avg:156.62ms
step:1042/1480 train_time:161644ms step_avg:156.63ms
step:1043/1480 train_time:161810ms step_avg:156.64ms
step:1044/1480 train_time:161974ms step_avg:156.65ms
step:1045/1480 train_time:162146ms step_avg:156.66ms
step:1046/1480 train_time:162314ms step_avg:156.67ms
step:1047/1480 train_time:162482ms step_avg:156.68ms
step:1048/1480 train_time:162648ms step_avg:156.69ms
step:1049/1480 train_time:162813ms step_avg:156.70ms
step:1050/1480 train_time:162984ms step_avg:156.72ms
step:1051/1480 train_time:163152ms step_avg:156.73ms
step:1052/1480 train_time:163320ms step_avg:156.74ms
step:1053/1480 train_time:163486ms step_avg:156.75ms
step:1054/1480 train_time:163653ms step_avg:156.76ms
step:1055/1480 train_time:163819ms step_avg:156.76ms
step:1056/1480 train_time:163985ms step_avg:156.77ms
step:1057/1480 train_time:164151ms step_avg:156.78ms
step:1058/1480 train_time:164323ms step_avg:156.80ms
step:1059/1480 train_time:164498ms step_avg:156.81ms
step:1060/1480 train_time:164667ms step_avg:156.83ms
step:1061/1480 train_time:164830ms step_avg:156.83ms
step:1062/1480 train_time:164996ms step_avg:156.84ms
step:1063/1480 train_time:165162ms step_avg:156.85ms
step:1064/1480 train_time:165326ms step_avg:156.86ms
step:1065/1480 train_time:165493ms step_avg:156.87ms
step:1066/1480 train_time:165661ms step_avg:156.88ms
step:1067/1480 train_time:165829ms step_avg:156.89ms
step:1068/1480 train_time:165995ms step_avg:156.89ms
step:1069/1480 train_time:166166ms step_avg:156.91ms
step:1070/1480 train_time:166331ms step_avg:156.92ms
step:1071/1480 train_time:166507ms step_avg:156.93ms
step:1072/1480 train_time:166673ms step_avg:156.94ms
step:1073/1480 train_time:166837ms step_avg:156.95ms
step:1074/1480 train_time:167004ms step_avg:156.96ms
step:1075/1480 train_time:167173ms step_avg:156.97ms
step:1076/1480 train_time:167343ms step_avg:156.98ms
step:1077/1480 train_time:167510ms step_avg:156.99ms
step:1078/1480 train_time:167686ms step_avg:157.01ms
step:1079/1480 train_time:167860ms step_avg:157.02ms
step:1080/1480 train_time:168031ms step_avg:157.04ms
step:1081/1480 train_time:168199ms step_avg:157.05ms
step:1082/1480 train_time:168366ms step_avg:157.06ms
step:1083/1480 train_time:168532ms step_avg:157.07ms
step:1084/1480 train_time:168700ms step_avg:157.08ms
step:1085/1480 train_time:168868ms step_avg:157.09ms
step:1086/1480 train_time:169038ms step_avg:157.10ms
step:1087/1480 train_time:169205ms step_avg:157.11ms
step:1088/1480 train_time:169374ms step_avg:157.12ms
step:1089/1480 train_time:169546ms step_avg:157.13ms
step:1090/1480 train_time:169718ms step_avg:157.15ms
step:1091/1480 train_time:169887ms step_avg:157.16ms
step:1092/1480 train_time:170053ms step_avg:157.17ms
step:1093/1480 train_time:170222ms step_avg:157.18ms
step:1094/1480 train_time:170389ms step_avg:157.18ms
step:1095/1480 train_time:170552ms step_avg:157.19ms
step:1096/1480 train_time:170722ms step_avg:157.20ms
step:1097/1480 train_time:170890ms step_avg:157.21ms
step:1098/1480 train_time:171062ms step_avg:157.23ms
step:1099/1480 train_time:171234ms step_avg:157.24ms
step:1100/1480 train_time:171407ms step_avg:157.25ms
step:1101/1480 train_time:171577ms step_avg:157.27ms
step:1102/1480 train_time:171748ms step_avg:157.28ms
step:1103/1480 train_time:171924ms step_avg:157.30ms
step:1104/1480 train_time:172092ms step_avg:157.31ms
step:1105/1480 train_time:172264ms step_avg:157.32ms
step:1106/1480 train_time:172432ms step_avg:157.33ms
step:1107/1480 train_time:172602ms step_avg:157.34ms
step:1108/1480 train_time:172767ms step_avg:157.35ms
step:1109/1480 train_time:172932ms step_avg:157.35ms
step:1110/1480 train_time:173099ms step_avg:157.36ms
step:1111/1480 train_time:173266ms step_avg:157.37ms
step:1112/1480 train_time:173434ms step_avg:157.38ms
step:1113/1480 train_time:173615ms step_avg:157.40ms
step:1114/1480 train_time:173790ms step_avg:157.42ms
step:1115/1480 train_time:173964ms step_avg:157.43ms
step:1116/1480 train_time:174130ms step_avg:157.44ms
step:1117/1480 train_time:174304ms step_avg:157.46ms
step:1118/1480 train_time:174478ms step_avg:157.47ms
step:1119/1480 train_time:174645ms step_avg:157.48ms
step:1120/1480 train_time:174814ms step_avg:157.49ms
step:1121/1480 train_time:174986ms step_avg:157.50ms
step:1122/1480 train_time:175151ms step_avg:157.51ms
step:1123/1480 train_time:175319ms step_avg:157.52ms
step:1124/1480 train_time:175488ms step_avg:157.53ms
step:1125/1480 train_time:175654ms step_avg:157.54ms
step:1125/1480 val_loss:3.3858 train_time:175722ms step_avg:157.60ms
step:1126/1480 train_time:175827ms step_avg:157.55ms
step:1127/1480 train_time:175996ms step_avg:157.56ms
step:1128/1480 train_time:176168ms step_avg:157.57ms
step:1129/1480 train_time:176343ms step_avg:157.59ms
step:1130/1480 train_time:176512ms step_avg:157.60ms
step:1131/1480 train_time:176689ms step_avg:157.62ms
step:1132/1480 train_time:176854ms step_avg:157.62ms
step:1133/1480 train_time:177028ms step_avg:157.64ms
step:1134/1480 train_time:177199ms step_avg:157.65ms
step:1135/1480 train_time:177368ms step_avg:157.66ms
step:1136/1480 train_time:177536ms step_avg:157.67ms
step:1137/1480 train_time:177706ms step_avg:157.68ms
step:1138/1480 train_time:177876ms step_avg:157.69ms
step:1139/1480 train_time:178046ms step_avg:157.70ms
step:1140/1480 train_time:178215ms step_avg:157.71ms
step:1141/1480 train_time:178388ms step_avg:157.73ms
step:1142/1480 train_time:178555ms step_avg:157.73ms
step:1143/1480 train_time:178727ms step_avg:157.75ms
step:1144/1480 train_time:178895ms step_avg:157.76ms
step:1145/1480 train_time:179059ms step_avg:157.76ms
step:1146/1480 train_time:179230ms step_avg:157.77ms
step:1147/1480 train_time:179398ms step_avg:157.78ms
step:1148/1480 train_time:179567ms step_avg:157.79ms
step:1149/1480 train_time:179737ms step_avg:157.80ms
step:1150/1480 train_time:179907ms step_avg:157.81ms
step:1151/1480 train_time:180077ms step_avg:157.82ms
step:1152/1480 train_time:180251ms step_avg:157.84ms
step:1153/1480 train_time:180425ms step_avg:157.85ms
step:1154/1480 train_time:180592ms step_avg:157.86ms
step:1155/1480 train_time:180762ms step_avg:157.87ms
step:1156/1480 train_time:180943ms step_avg:157.89ms
step:1157/1480 train_time:181112ms step_avg:157.90ms
step:1158/1480 train_time:181279ms step_avg:157.91ms
step:1159/1480 train_time:181447ms step_avg:157.92ms
step:1160/1480 train_time:181612ms step_avg:157.92ms
step:1161/1480 train_time:181781ms step_avg:157.93ms
step:1162/1480 train_time:181950ms step_avg:157.94ms
step:1163/1480 train_time:182120ms step_avg:157.95ms
step:1164/1480 train_time:182289ms step_avg:157.96ms
step:1165/1480 train_time:182454ms step_avg:157.97ms
step:1166/1480 train_time:182625ms step_avg:157.98ms
step:1167/1480 train_time:182793ms step_avg:157.99ms
step:1168/1480 train_time:182961ms step_avg:158.00ms
step:1169/1480 train_time:183131ms step_avg:158.01ms
step:1170/1480 train_time:183299ms step_avg:158.02ms
step:1171/1480 train_time:183467ms step_avg:158.02ms
step:1172/1480 train_time:183634ms step_avg:158.03ms
step:1173/1480 train_time:183805ms step_avg:158.04ms
step:1174/1480 train_time:183988ms step_avg:158.06ms
step:1175/1480 train_time:184158ms step_avg:158.08ms
step:1176/1480 train_time:184331ms step_avg:158.09ms
step:1177/1480 train_time:184508ms step_avg:158.10ms
step:1178/1480 train_time:184675ms step_avg:158.11ms
step:1179/1480 train_time:184840ms step_avg:158.12ms
step:1180/1480 train_time:185020ms step_avg:158.14ms
step:1181/1480 train_time:185190ms step_avg:158.15ms
step:1182/1480 train_time:185357ms step_avg:158.15ms
step:1183/1480 train_time:185528ms step_avg:158.16ms
step:1184/1480 train_time:185694ms step_avg:158.17ms
step:1185/1480 train_time:185866ms step_avg:158.18ms
step:1186/1480 train_time:186038ms step_avg:158.20ms
step:1187/1480 train_time:186222ms step_avg:158.22ms
step:1188/1480 train_time:186389ms step_avg:158.23ms
step:1189/1480 train_time:186560ms step_avg:158.24ms
step:1190/1480 train_time:186728ms step_avg:158.24ms
step:1191/1480 train_time:186897ms step_avg:158.25ms
step:1192/1480 train_time:187063ms step_avg:158.26ms
step:1193/1480 train_time:187230ms step_avg:158.27ms
step:1194/1480 train_time:187399ms step_avg:158.28ms
step:1195/1480 train_time:187573ms step_avg:158.29ms
step:1196/1480 train_time:187756ms step_avg:158.31ms
step:1197/1480 train_time:187929ms step_avg:158.32ms
step:1198/1480 train_time:188110ms step_avg:158.34ms
step:1199/1480 train_time:188282ms step_avg:158.35ms
step:1200/1480 train_time:188451ms step_avg:158.36ms
step:1201/1480 train_time:188619ms step_avg:158.37ms
step:1202/1480 train_time:188801ms step_avg:158.39ms
step:1203/1480 train_time:188976ms step_avg:158.40ms
step:1204/1480 train_time:189150ms step_avg:158.42ms
step:1205/1480 train_time:189317ms step_avg:158.42ms
step:1206/1480 train_time:189486ms step_avg:158.43ms
step:1207/1480 train_time:189656ms step_avg:158.44ms
step:1208/1480 train_time:189824ms step_avg:158.45ms
step:1209/1480 train_time:189997ms step_avg:158.46ms
step:1210/1480 train_time:190173ms step_avg:158.48ms
step:1211/1480 train_time:190349ms step_avg:158.49ms
step:1212/1480 train_time:190520ms step_avg:158.50ms
step:1213/1480 train_time:190693ms step_avg:158.51ms
step:1214/1480 train_time:190871ms step_avg:158.53ms
step:1215/1480 train_time:191045ms step_avg:158.54ms
step:1216/1480 train_time:191213ms step_avg:158.55ms
step:1217/1480 train_time:191386ms step_avg:158.56ms
step:1218/1480 train_time:191556ms step_avg:158.57ms
step:1219/1480 train_time:191736ms step_avg:158.59ms
step:1220/1480 train_time:191905ms step_avg:158.60ms
step:1221/1480 train_time:192073ms step_avg:158.61ms
step:1222/1480 train_time:192242ms step_avg:158.62ms
step:1223/1480 train_time:192412ms step_avg:158.63ms
step:1224/1480 train_time:192591ms step_avg:158.64ms
step:1225/1480 train_time:192764ms step_avg:158.65ms
step:1226/1480 train_time:192937ms step_avg:158.67ms
step:1227/1480 train_time:193110ms step_avg:158.68ms
step:1228/1480 train_time:193279ms step_avg:158.69ms
step:1229/1480 train_time:193452ms step_avg:158.70ms
step:1230/1480 train_time:193634ms step_avg:158.72ms
step:1231/1480 train_time:193810ms step_avg:158.73ms
step:1232/1480 train_time:193985ms step_avg:158.74ms
step:1233/1480 train_time:194154ms step_avg:158.75ms
step:1234/1480 train_time:194325ms step_avg:158.76ms
step:1235/1480 train_time:194498ms step_avg:158.77ms
step:1236/1480 train_time:194668ms step_avg:158.78ms
step:1237/1480 train_time:194839ms step_avg:158.79ms
step:1238/1480 train_time:195027ms step_avg:158.82ms
step:1239/1480 train_time:195197ms step_avg:158.83ms
step:1240/1480 train_time:195368ms step_avg:158.84ms
step:1241/1480 train_time:195541ms step_avg:158.85ms
step:1242/1480 train_time:195710ms step_avg:158.86ms
step:1243/1480 train_time:195885ms step_avg:158.87ms
step:1244/1480 train_time:196051ms step_avg:158.87ms
step:1245/1480 train_time:196219ms step_avg:158.88ms
step:1246/1480 train_time:196390ms step_avg:158.89ms
step:1247/1480 train_time:196559ms step_avg:158.90ms
step:1248/1480 train_time:196730ms step_avg:158.91ms
step:1249/1480 train_time:196897ms step_avg:158.92ms
step:1250/1480 train_time:197068ms step_avg:158.93ms
step:1250/1480 val_loss:3.3362 train_time:197139ms step_avg:158.98ms
step:1251/1480 train_time:197249ms step_avg:158.94ms
step:1252/1480 train_time:197419ms step_avg:158.95ms
step:1253/1480 train_time:197587ms step_avg:158.96ms
step:1254/1480 train_time:197758ms step_avg:158.97ms
step:1255/1480 train_time:197945ms step_avg:158.99ms
step:1256/1480 train_time:198117ms step_avg:159.00ms
step:1257/1480 train_time:198287ms step_avg:159.01ms
step:1258/1480 train_time:198461ms step_avg:159.02ms
step:1259/1480 train_time:198632ms step_avg:159.03ms
step:1260/1480 train_time:198801ms step_avg:159.04ms
step:1261/1480 train_time:198973ms step_avg:159.05ms
step:1262/1480 train_time:199147ms step_avg:159.06ms
step:1263/1480 train_time:199319ms step_avg:159.07ms
step:1264/1480 train_time:199487ms step_avg:159.08ms
step:1265/1480 train_time:199654ms step_avg:159.09ms
step:1266/1480 train_time:199828ms step_avg:159.10ms
step:1267/1480 train_time:199998ms step_avg:159.11ms
step:1268/1480 train_time:200169ms step_avg:159.12ms
step:1269/1480 train_time:200345ms step_avg:159.13ms
step:1270/1480 train_time:200515ms step_avg:159.14ms
step:1271/1480 train_time:200685ms step_avg:159.15ms
step:1272/1480 train_time:200851ms step_avg:159.15ms
step:1273/1480 train_time:201024ms step_avg:159.16ms
step:1274/1480 train_time:201196ms step_avg:159.17ms
step:1275/1480 train_time:201364ms step_avg:159.18ms
step:1276/1480 train_time:201530ms step_avg:159.19ms
step:1277/1480 train_time:201702ms step_avg:159.20ms
step:1278/1480 train_time:201871ms step_avg:159.20ms
step:1279/1480 train_time:202044ms step_avg:159.21ms
step:1280/1480 train_time:202223ms step_avg:159.23ms
step:1281/1480 train_time:202390ms step_avg:159.24ms
step:1282/1480 train_time:202556ms step_avg:159.24ms
step:1283/1480 train_time:202727ms step_avg:159.25ms
step:1284/1480 train_time:202896ms step_avg:159.26ms
step:1285/1480 train_time:203065ms step_avg:159.27ms
step:1286/1480 train_time:203234ms step_avg:159.27ms
step:1287/1480 train_time:203406ms step_avg:159.28ms
step:1288/1480 train_time:203578ms step_avg:159.29ms
step:1289/1480 train_time:203762ms step_avg:159.31ms
step:1290/1480 train_time:203942ms step_avg:159.33ms
step:1291/1480 train_time:204113ms step_avg:159.34ms
step:1292/1480 train_time:204287ms step_avg:159.35ms
step:1293/1480 train_time:204463ms step_avg:159.36ms
step:1294/1480 train_time:204635ms step_avg:159.37ms
step:1295/1480 train_time:204806ms step_avg:159.38ms
step:1296/1480 train_time:204979ms step_avg:159.39ms
step:1297/1480 train_time:205150ms step_avg:159.40ms
step:1298/1480 train_time:205321ms step_avg:159.41ms
step:1299/1480 train_time:205492ms step_avg:159.42ms
step:1300/1480 train_time:205660ms step_avg:159.43ms
step:1301/1480 train_time:205828ms step_avg:159.43ms
step:1302/1480 train_time:206002ms step_avg:159.44ms
step:1303/1480 train_time:206177ms step_avg:159.46ms
step:1304/1480 train_time:206351ms step_avg:159.47ms
step:1305/1480 train_time:206520ms step_avg:159.47ms
step:1306/1480 train_time:206693ms step_avg:159.49ms
step:1307/1480 train_time:206862ms step_avg:159.49ms
step:1308/1480 train_time:207031ms step_avg:159.50ms
step:1309/1480 train_time:207205ms step_avg:159.51ms
step:1310/1480 train_time:207373ms step_avg:159.52ms
step:1311/1480 train_time:207542ms step_avg:159.52ms
step:1312/1480 train_time:207712ms step_avg:159.53ms
step:1313/1480 train_time:207882ms step_avg:159.54ms
step:1314/1480 train_time:208053ms step_avg:159.55ms
step:1315/1480 train_time:208224ms step_avg:159.56ms
step:1316/1480 train_time:208390ms step_avg:159.56ms
step:1317/1480 train_time:208564ms step_avg:159.57ms
step:1318/1480 train_time:208747ms step_avg:159.59ms
step:1319/1480 train_time:208923ms step_avg:159.61ms
step:1320/1480 train_time:209100ms step_avg:159.62ms
step:1321/1480 train_time:209272ms step_avg:159.63ms
step:1322/1480 train_time:209454ms step_avg:159.64ms
step:1323/1480 train_time:209628ms step_avg:159.66ms
step:1324/1480 train_time:209803ms step_avg:159.67ms
step:1325/1480 train_time:209984ms step_avg:159.68ms
step:1326/1480 train_time:210159ms step_avg:159.70ms
step:1327/1480 train_time:210331ms step_avg:159.70ms
step:1328/1480 train_time:210502ms step_avg:159.71ms
step:1329/1480 train_time:210698ms step_avg:159.74ms
step:1330/1480 train_time:210875ms step_avg:159.75ms
step:1331/1480 train_time:211046ms step_avg:159.76ms
step:1332/1480 train_time:211220ms step_avg:159.77ms
step:1333/1480 train_time:211396ms step_avg:159.79ms
step:1334/1480 train_time:211568ms step_avg:159.79ms
step:1335/1480 train_time:211736ms step_avg:159.80ms
step:1336/1480 train_time:211922ms step_avg:159.82ms
step:1337/1480 train_time:212095ms step_avg:159.83ms
step:1338/1480 train_time:212269ms step_avg:159.84ms
step:1339/1480 train_time:212444ms step_avg:159.85ms
step:1340/1480 train_time:212617ms step_avg:159.86ms
step:1341/1480 train_time:212786ms step_avg:159.87ms
step:1342/1480 train_time:212961ms step_avg:159.88ms
step:1343/1480 train_time:213132ms step_avg:159.89ms
step:1344/1480 train_time:213303ms step_avg:159.90ms
step:1345/1480 train_time:213482ms step_avg:159.91ms
step:1346/1480 train_time:213650ms step_avg:159.92ms
step:1347/1480 train_time:213820ms step_avg:159.93ms
step:1348/1480 train_time:213989ms step_avg:159.93ms
step:1349/1480 train_time:214159ms step_avg:159.94ms
step:1350/1480 train_time:214334ms step_avg:159.95ms
step:1351/1480 train_time:214505ms step_avg:159.96ms
step:1352/1480 train_time:214676ms step_avg:159.97ms
step:1353/1480 train_time:214851ms step_avg:159.98ms
step:1354/1480 train_time:215023ms step_avg:159.99ms
step:1355/1480 train_time:215189ms step_avg:159.99ms
step:1356/1480 train_time:215363ms step_avg:160.00ms
step:1357/1480 train_time:215535ms step_avg:160.01ms
step:1358/1480 train_time:215707ms step_avg:160.02ms
step:1359/1480 train_time:215880ms step_avg:160.03ms
step:1360/1480 train_time:216053ms step_avg:160.04ms
step:1361/1480 train_time:216231ms step_avg:160.05ms
step:1362/1480 train_time:216407ms step_avg:160.06ms
step:1363/1480 train_time:216586ms step_avg:160.08ms
step:1364/1480 train_time:216754ms step_avg:160.08ms
step:1365/1480 train_time:216922ms step_avg:160.09ms
step:1366/1480 train_time:217094ms step_avg:160.10ms
step:1367/1480 train_time:217266ms step_avg:160.11ms
step:1368/1480 train_time:217438ms step_avg:160.12ms
step:1369/1480 train_time:217618ms step_avg:160.13ms
step:1370/1480 train_time:217794ms step_avg:160.14ms
step:1371/1480 train_time:217967ms step_avg:160.15ms
step:1372/1480 train_time:218146ms step_avg:160.17ms
step:1373/1480 train_time:218315ms step_avg:160.17ms
step:1374/1480 train_time:218491ms step_avg:160.18ms
step:1375/1480 train_time:218663ms step_avg:160.19ms
step:1375/1480 val_loss:3.2977 train_time:218731ms step_avg:160.24ms
step:1376/1480 train_time:218838ms step_avg:160.20ms
step:1377/1480 train_time:219010ms step_avg:160.21ms
step:1378/1480 train_time:219179ms step_avg:160.22ms
step:1379/1480 train_time:219356ms step_avg:160.23ms
step:1380/1480 train_time:219530ms step_avg:160.24ms
step:1381/1480 train_time:219709ms step_avg:160.25ms
step:1382/1480 train_time:219880ms step_avg:160.26ms
step:1383/1480 train_time:220053ms step_avg:160.27ms
step:1384/1480 train_time:220231ms step_avg:160.28ms
step:1385/1480 train_time:220397ms step_avg:160.29ms
step:1386/1480 train_time:220569ms step_avg:160.30ms
step:1387/1480 train_time:220740ms step_avg:160.30ms
step:1388/1480 train_time:220908ms step_avg:160.31ms
step:1389/1480 train_time:221081ms step_avg:160.32ms
step:1390/1480 train_time:221248ms step_avg:160.32ms
step:1391/1480 train_time:221418ms step_avg:160.33ms
step:1392/1480 train_time:221589ms step_avg:160.34ms
step:1393/1480 train_time:221760ms step_avg:160.35ms
step:1394/1480 train_time:221932ms step_avg:160.36ms
step:1395/1480 train_time:222100ms step_avg:160.36ms
step:1396/1480 train_time:222268ms step_avg:160.37ms
step:1397/1480 train_time:222436ms step_avg:160.37ms
step:1398/1480 train_time:222602ms step_avg:160.38ms
step:1399/1480 train_time:222772ms step_avg:160.38ms
step:1400/1480 train_time:222951ms step_avg:160.40ms
step:1401/1480 train_time:223117ms step_avg:160.40ms
step:1402/1480 train_time:223288ms step_avg:160.41ms
step:1403/1480 train_time:223464ms step_avg:160.42ms
step:1404/1480 train_time:223634ms step_avg:160.43ms
step:1405/1480 train_time:223805ms step_avg:160.43ms
step:1406/1480 train_time:223980ms step_avg:160.44ms
step:1407/1480 train_time:224150ms step_avg:160.45ms
step:1408/1480 train_time:224320ms step_avg:160.46ms
step:1409/1480 train_time:224502ms step_avg:160.47ms
step:1410/1480 train_time:224673ms step_avg:160.48ms
step:1411/1480 train_time:224841ms step_avg:160.49ms
step:1412/1480 train_time:225011ms step_avg:160.49ms
step:1413/1480 train_time:225180ms step_avg:160.50ms
step:1414/1480 train_time:225351ms step_avg:160.51ms
step:1415/1480 train_time:225526ms step_avg:160.52ms
step:1416/1480 train_time:225713ms step_avg:160.54ms
step:1417/1480 train_time:225887ms step_avg:160.55ms
step:1418/1480 train_time:226059ms step_avg:160.55ms
step:1419/1480 train_time:226233ms step_avg:160.56ms
step:1420/1480 train_time:226408ms step_avg:160.57ms
step:1421/1480 train_time:226580ms step_avg:160.58ms
step:1422/1480 train_time:226754ms step_avg:160.59ms
step:1423/1480 train_time:226923ms step_avg:160.60ms
step:1424/1480 train_time:227102ms step_avg:160.61ms
step:1425/1480 train_time:227282ms step_avg:160.62ms
step:1426/1480 train_time:227455ms step_avg:160.63ms
step:1427/1480 train_time:227629ms step_avg:160.64ms
step:1428/1480 train_time:227799ms step_avg:160.65ms
step:1429/1480 train_time:227966ms step_avg:160.65ms
step:1430/1480 train_time:228142ms step_avg:160.66ms
step:1431/1480 train_time:228317ms step_avg:160.67ms
step:1432/1480 train_time:228494ms step_avg:160.68ms
step:1433/1480 train_time:228672ms step_avg:160.70ms
step:1434/1480 train_time:228854ms step_avg:160.71ms
step:1435/1480 train_time:229029ms step_avg:160.72ms
step:1436/1480 train_time:229203ms step_avg:160.73ms
step:1437/1480 train_time:229374ms step_avg:160.74ms
step:1438/1480 train_time:229542ms step_avg:160.74ms
step:1439/1480 train_time:229716ms step_avg:160.75ms
step:1440/1480 train_time:229884ms step_avg:160.76ms
step:1441/1480 train_time:230056ms step_avg:160.77ms
step:1442/1480 train_time:230233ms step_avg:160.78ms
step:1443/1480 train_time:230422ms step_avg:160.80ms
step:1444/1480 train_time:230593ms step_avg:160.80ms
step:1445/1480 train_time:230764ms step_avg:160.81ms
step:1446/1480 train_time:230940ms step_avg:160.82ms
step:1447/1480 train_time:231117ms step_avg:160.83ms
step:1448/1480 train_time:231289ms step_avg:160.84ms
step:1449/1480 train_time:231462ms step_avg:160.85ms
step:1450/1480 train_time:231635ms step_avg:160.86ms
step:1451/1480 train_time:231805ms step_avg:160.86ms
step:1452/1480 train_time:231979ms step_avg:160.87ms
step:1453/1480 train_time:232149ms step_avg:160.88ms
step:1454/1480 train_time:232320ms step_avg:160.89ms
step:1455/1480 train_time:232499ms step_avg:160.90ms
step:1456/1480 train_time:232672ms step_avg:160.91ms
step:1457/1480 train_time:232844ms step_avg:160.91ms
step:1458/1480 train_time:233016ms step_avg:160.92ms
step:1459/1480 train_time:233191ms step_avg:160.93ms
step:1460/1480 train_time:233362ms step_avg:160.94ms
step:1461/1480 train_time:233538ms step_avg:160.95ms
step:1462/1480 train_time:233708ms step_avg:160.96ms
step:1463/1480 train_time:233885ms step_avg:160.97ms
step:1464/1480 train_time:234060ms step_avg:160.98ms
step:1465/1480 train_time:234233ms step_avg:160.99ms
step:1466/1480 train_time:234403ms step_avg:160.99ms
step:1467/1480 train_time:234578ms step_avg:161.00ms
step:1468/1480 train_time:234749ms step_avg:161.01ms
step:1469/1480 train_time:234922ms step_avg:161.02ms
step:1470/1480 train_time:235100ms step_avg:161.03ms
step:1471/1480 train_time:235286ms step_avg:161.04ms
step:1472/1480 train_time:235466ms step_avg:161.06ms
step:1473/1480 train_time:235637ms step_avg:161.06ms
step:1474/1480 train_time:235815ms step_avg:161.08ms
step:1475/1480 train_time:235994ms step_avg:161.09ms
step:1476/1480 train_time:236167ms step_avg:161.10ms
step:1477/1480 train_time:236352ms step_avg:161.11ms
step:1478/1480 train_time:236535ms step_avg:161.13ms
step:1479/1480 train_time:236707ms step_avg:161.14ms
step:1480/1480 train_time:236880ms step_avg:161.14ms
step:1480/1480 val_loss:3.2788 train_time:236952ms step_avg:161.19ms