26fa5797-44d0-4a63-9e57-f435f2f59aad.txt

import os
import sys
with open(sys.argv[0]) as f:
    code = f.read() # read the code of this file ASAP, for logging
import uuid
import time
from dataclasses import dataclass
from pathlib import Path

import torch
from torch import nn
import torch.nn.functional as F
import torch.distributed as dist
import torch._inductor.config as config
from torch.nn.parallel import DistributedDataParallel as DDP
# Use of FlexAttention contributed by @KoszarskyB
from torch.nn.attention.flex_attention import BlockMask, flex_attention

# -----------------------------------------------------------------------------
# Muon optimizer

def zeropower_via_svd(G, steps=None):
    U, S, V = G.svd()
    return U @ V.T

@torch.compile
def zeropower_via_newtonschulz5(G, steps=10, eps=1e-7):
    """
    Newton-Schulz iteration to compute the zeroth power / orthogonalization of G. We opt to use a
    quintic iteration whose coefficients are selected to maximize the slope at zero. For the purpose
    of minimizing steps, it turns out to be empirically effective to keep increasing the slope at
    zero even beyond the point where the iteration no longer converges all the way to one everywhere
    on the interval. This iteration therefore does not produce UV^T but rather something like US'V^T
    where S' is diagonal with S_{ii}' ~ Uniform(0.5, 1.5), which turns out not to hurt model
    performance at all relative to UV^T, where USV^T = G is the SVD.
    """
    assert len(G.shape) == 2
    a, b, c = (3.4445, -4.7750,  2.0315)
    X = G.bfloat16()
    X /= (X.norm() + eps) # ensure top singular value <= 1
    if G.size(0) > G.size(1):
        X = X.T
    for _ in range(steps):
        A = X @ X.T
        B = b * A + c * A @ A # adapted from suggestion by @jxbz, @leloykun, and @YouJiacheng
        X = a * X + B @ X
    if G.size(0) > G.size(1):
        X = X.T
    return X

zeropower_backends = dict(svd=zeropower_via_svd, newtonschulz5=zeropower_via_newtonschulz5)

class Muon(torch.optim.Optimizer):
    """
    Muon - MomentUm Orthogonalized by Newton-schulz

    Muon internally runs standard SGD-momentum, and then performs an orthogonalization post-
    processing step, in which each 2D parameter's update is replaced with the nearest orthogonal
    matrix. To efficiently orthogonalize each update, we use a Newton-Schulz iteration, which has
    the advantage that it can be stably run in bfloat16 on the GPU.

    Some warnings:
    - This optimizer assumes that all parameters passed in are 2D.
    - It should not be used for the embedding layer, the final fully connected layer, or any {0,1}-D
    parameters; those should all be optimized by a standard method (e.g., AdamW).
    - To use it with 4D convolutional filters, it works well to just flatten their last 3 dimensions.
    - We believe it is unlikely to work well for training with small batch size.
    - We believe it may not work well for finetuning pretrained models, but we haven't tested this.
    - We have not yet tried this optimizer for training scenarios larger than NanoGPT (124M).

    Arguments:
        lr: The learning rate used by the internal SGD.
        momentum: The momentum used by the internal SGD.
        nesterov: Whether to use Nesterov-style momentum in the internal SGD. (recommended)
        backend: The chosen backend for the orthogonalization step. (recommended: 'newtonschulz5')
        backend_steps: The number of iteration steps to use in the backend, if it is iterative.
    """
    def __init__(self, params, lr=0.02, momentum=0.95, nesterov=True,
                 backend='newtonschulz5', backend_steps=5):
        self.num_process = int(os.environ['WORLD_SIZE'])
        self.rank = int(os.environ["RANK"])
        defaults = dict(lr=lr, momentum=momentum, nesterov=nesterov, backend=backend, backend_steps=backend_steps)
        params: "list[torch.Tensor]" = list(params)
        assert all(isinstance(p, torch.Tensor) for p in params)
        sizes = {p.numel() for p in params}
        param_groups = [
            {
                "params": [p for p in params if p.numel() == size],
                "update_buffer": [
                    torch.empty(size, device="cuda", dtype=torch.bfloat16)
                    for _ in range(self.num_process)
                ],
            }
            for size in sizes
        ]
        super().__init__(param_groups, defaults)

    def step(self):
        for group in self.param_groups:
            lr: float = group["lr"]
            momentum: float = group["momentum"]
            nesterov: bool = group["nesterov"]
            zeropower_backend = zeropower_backends[group["backend"]]
            backend_steps: int = group["backend_steps"]
            update_buffers: "list[torch.Tensor]" = group["update_buffer"]
            # generate weight updates in distributed fashion
            params: "list[torch.Tensor]" = group["params"]
            assert len(params) % self.num_process == 0
            handle = None
            params_world = None
            def update_prev():
                if params_world is None:
                    return
                assert handle is not None
                handle.wait()
                for p_world, g_world in zip(params_world, update_buffers):
                    p_world.data.add_(
                        g_world.view_as(p_world),
                        alpha=-lr * max(1, p_world.size(0) / p_world.size(1)) ** 0.5,
                    )
            for base_i in range(len(params))[::self.num_process]:
                p = params[base_i + self.rank]
                g = p.grad
                assert g is not None
                state = self.state[p] 
                if "momentum_buffer" not in state:
                    state["momentum_buffer"] = torch.zeros_like(g)
                buf: torch.Tensor = state["momentum_buffer"]
                buf.lerp_(g, 1 - momentum)
                g = g.lerp_(buf, momentum) if nesterov else buf
                g = zeropower_backend(g, steps=backend_steps).flatten()
                update_prev()
                handle = dist.all_gather(update_buffers, g, async_op=True)
                params_world = params[base_i : base_i + self.num_process]
            update_prev()


# -----------------------------------------------------------------------------
# PyTorch nn.Module definitions for the GPT-2 model

def norm(x):
    return F.rms_norm(x, (x.size(-1),))

class CastedLinear(nn.Linear):

    def __init__(self, in_features, out_features):
        super().__init__(in_features, out_features, bias=False)

    def forward(self, x):
        return F.linear(x, self.weight.to(x.dtype))

class Rotary(torch.nn.Module):

    def __init__(self, dim, base=10000):
        super().__init__()
        self.register_buffer('inv_freq', (1 / base) ** (torch.arange(0, dim, 2) / dim))
        self.seq_len_cached = None
        self.cos_cached = None
        self.sin_cached = None

    def forward(self, x):
        seq_len = x.shape[1]
        if seq_len != self.seq_len_cached:
            t = torch.arange(seq_len, device=x.device)
            freqs = torch.outer(t, self.inv_freq)
            self.seq_len_cached = seq_len
            self.cos_cached = freqs.cos()
            self.sin_cached = freqs.sin()
        cos, sin = self.cos_cached[None, :, None, :], self.sin_cached[None, :, None, :]
        # apply_rotary_emb(x, cos, sin)
        x1, x2 = x.chunk(2, dim=3)
        y1 = x1 * cos + x2 * sin
        y2 = x1 * (-sin) + x2 * cos
        return torch.cat((y1, y2), 3).type_as(x)

class CausalSelfAttention(nn.Module):

    def __init__(self, dim, n_head):
        super().__init__()
        assert dim % n_head == 0
        self.n_head = n_head
        self.c_q = CastedLinear(dim, dim)
        self.c_k = CastedLinear(dim, dim)
        self.c_v = CastedLinear(dim, dim)
        # value residual lambda
        self.lambdas = nn.Parameter(torch.tensor([0.5, 0.5])) # @Grad62304977
        # rotary embeddings
        self.rotary = Rotary(dim // n_head) # dim // n_head = head_dim
        # output projection
        self.c_proj = CastedLinear(dim, dim)
        self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977

    def forward(self, x: torch.Tensor, vi: torch.Tensor, block_mask: BlockMask) -> torch.Tensor:
        B, T = x.size(0), x.size(1) # batch size, sequence length
        assert B == 1, "Must use batch size = 1 for FlexAttention"
        q: torch.Tensor = self.c_q(x).view(B, T, self.n_head, -1)
        k: torch.Tensor = self.c_k(x).view(B, T, self.n_head, -1)
        v: torch.Tensor = self.c_v(x).view(B, T, self.n_head, -1)
        v = self.lambdas[0] * v + self.lambdas[1] * vi.view_as(v) # @Grad62304977
        q, k = norm(q), norm(k) # QK norm suggested by @Grad62304977
        q, k = self.rotary(q), self.rotary(k)
        y = flex_attention(q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2), block_mask=block_mask)
        y = y.transpose(1, 2).contiguous().view_as(x) # re-assemble all head outputs side by side
        y = self.c_proj(y)
        return y

class MLP(nn.Module):

    def __init__(self, dim: int):
        super().__init__()
        self.c_fc   = CastedLinear(dim, 4 * dim)
        self.c_proj = CastedLinear(4 * dim, dim)
        self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        x = self.c_fc(x)
        x = F.relu(x).square() # https://arxiv.org/abs/2109.08668v2; ~1-2% better than GELU; suggested by @SKYLINEZ007 and @Grad62304977
        x = self.c_proj(x)
        return x

class Block(nn.Module):

    def __init__(self, config):
        super().__init__()
        self.attn = CausalSelfAttention(config.n_embd, config.n_head)
        self.mlp = MLP(config.n_embd)
        self.lambdas = nn.Parameter(torch.tensor([1., 0.]))

    def forward(self, x: torch.Tensor, vi: torch.Tensor, x0: torch.Tensor, block_mask: BlockMask) -> torch.Tensor:
        x = self.lambdas[0] * x + self.lambdas[1] * x0
        x = x + self.attn(norm(x), vi, block_mask)
        x = x + self.mlp(norm(x))
        return x

# -----------------------------------------------------------------------------
# The main GPT-2 model

@dataclass
class GPTConfig:
    vocab_size : int = 50304
    n_layer : int = 12
    n_head : int = 6 # head dim 128 suggested by @Grad62304977
    n_embd : int = 768
    lm_head_softcap : int = 30

class GPT(nn.Module):

    def __init__(self, config: GPTConfig):
        super().__init__()
        self.n_layer = config.n_layer
        self.lm_head_softcap = config.lm_head_softcap

        # U-net design by @brendanh0gan
        self.num_encoder_layers = config.n_layer // 2 # Half of the layers for encoder
        self.num_decoder_layers = config.n_layer - self.num_encoder_layers # Remaining for decoder
        # Add learnable skip connection weights for decoder layers
        self.skip_weights = nn.Parameter(torch.ones(self.num_decoder_layers))

        self.transformer = nn.ModuleDict(dict(
            wte = nn.Embedding(config.vocab_size, config.n_embd),
            # token value embeddings by @KoszarskyB - inspired by @Grad62304977's value residual learning
            # U-net structure on token value embeddings by @leloykun
            vte = nn.Embedding(config.vocab_size, config.n_embd*self.num_encoder_layers),
            h = nn.ModuleList([Block(config) for _ in range(config.n_layer)]),
        ))
        self.lm_head = CastedLinear(config.n_embd, config.vocab_size)
        self.lm_head.weight.data.zero_() # @Grad62304977

    def forward(self, idx: torch.Tensor, target: torch.Tensor, sliding_window: torch.Tensor) -> torch.Tensor:
        BLOCK_SIZE = 128
        assert idx.ndim == 1
        docs = (idx == 50256).cumsum(0)
        docs_low = docs.reshape(-1, BLOCK_SIZE)[:, 0].contiguous()
        docs_high = docs.reshape(-1, BLOCK_SIZE)[:, -1].contiguous()
        def document_sliding_window_causal(b, h, q_idx, kv_idx):
            causal_mask = q_idx >= kv_idx
            document_mask = docs[q_idx] == docs[kv_idx]
            window_mask = q_idx - kv_idx < sliding_window
            return causal_mask & document_mask & window_mask

        S = len(idx)
        def create_sliding_window_causal_mask(S: int, sliding_window: torch.Tensor):
            kv_idx = block_idx = torch.arange(S // BLOCK_SIZE, dtype=torch.int32, device="cuda")
            q_idx = block_idx[:, None]
            causal_mask = q_idx >= kv_idx
            document_mask = (docs_low[q_idx] <= docs_high[kv_idx]) & (docs_low[kv_idx] <= docs_high[q_idx])
            window_mask = q_idx - kv_idx < ((sliding_window + BLOCK_SIZE - 1) // BLOCK_SIZE)
            dense_mask = causal_mask & document_mask & window_mask
            dense_mask = dense_mask.to(torch.int32)
            num_blocks = dense_mask.sum(dim=-1).to(torch.int32)
            indices = torch.argsort(dense_mask, dim=-1, descending=True, stable=True).to(torch.int32)
            num_blocks = num_blocks[None, None, :].contiguous()
            indices = indices[None, None, :].contiguous()
            return BlockMask.from_kv_blocks(num_blocks, indices, BLOCK_SIZE=BLOCK_SIZE, mask_mod=document_sliding_window_causal)
        block_mask = create_sliding_window_causal_mask(S, sliding_window)

        # forward the GPT model itself
        x = self.transformer.wte(idx[None]) # token embeddings of shape (b, t, n_embd)
        x = norm(x) # @Grad62304977
        x0 = x
        vi = self.transformer.vte(idx[None]).chunk(self.num_encoder_layers, dim=-1)

        # Store outputs for U-Net skip connections
        skip_connections = []
        # Encoder pass - process only the first half of the blocks
        for i in range(self.num_encoder_layers):
            x = self.transformer.h[i](x, vi[i], x0, block_mask)
            skip_connections.append(x)
        # Decoder pass - process the remaining blocks with weighted skip connections
        for i in range(self.num_decoder_layers):
            x = x + self.skip_weights[i] * skip_connections.pop()
            # U-net structure on token value embeddings by @leloykun
            x = self.transformer.h[self.num_encoder_layers + i](x, vi[self.num_encoder_layers-1-i], x0, block_mask)

        x = norm(x)
        logits = self.lm_head(x)
        logits = self.lm_head_softcap * torch.tanh(logits / self.lm_head_softcap) # @Grad62304977
        logits = logits.float()
        loss = F.cross_entropy(logits.view(-1, logits.size(-1)), target.view(-1))
        return loss

# -----------------------------------------------------------------------------
# Our own simple Distributed Data Loader

def _peek_data_shard(file: Path):
    # only reads the header, returns header data
    # header is 256 int32
    header = torch.from_file(f"{file}", False, 256, dtype=torch.int32)
    assert header[0] == 20240520, "magic number mismatch in the data .bin file"
    assert header[1] == 1, "unsupported version"
    return int(header[2]) # number of tokens (claimed)

def _load_data_shard(file: Path, ntok: int):
    with file.open("rb") as f:
        tokens = torch.empty(ntok, dtype=torch.uint16, pin_memory=True)
        f.seek(256 * 4)
        nbytes = f.readinto(tokens.numpy())
        assert nbytes == 2 * ntok, "number of tokens read does not match header?"
    return tokens

class DistributedDataLoader:
    def __init__(self, filename_pattern, T, process_rank, num_processes):
        self.process_rank = process_rank
        self.num_processes = num_processes
        self.T = T

        # glob files that match the pattern
        self.files = sorted(Path.cwd().glob(filename_pattern))
        assert len(self.files) > 0, f"did not find any files that match the pattern {filename_pattern}"

        # load and validate all data shards, count number of tokens in total
        self.ntoks = [_peek_data_shard(file) for file in self.files]
        assert min(self.ntoks) >= num_processes * T + 1
        self.ntok_total = sum(self.ntoks)

        self.reset()

    def reset(self):
        self.current_shard = -1
        self.advance()

    def advance(self): # advance to next data shard
        self.current_shard = (self.current_shard + 1) % len(self.files)
        self.current_position = self.process_rank * self.T
        self.tokens = _load_data_shard(self.files[self.current_shard], self.ntoks[self.current_shard])

    def next_batch(self):
        batch_size = self.T * self.num_processes
        buf = self.tokens[self.current_position:self.current_position+self.T+1]
        # host side async is sufficient;
        # no performance improvement was observed when introducing a separate stream.
        x = buf[:-1].to(device="cuda", dtype=torch.int32, non_blocking=True) # inputs
        y = buf[1:].to(device="cuda", dtype=torch.int64, non_blocking=True) # targets
        # advance current position and load next shard if necessary
        self.current_position += batch_size
        if self.current_position + batch_size + 1 >= len(self.tokens):
            self.advance()
        return x, y

# -----------------------------------------------------------------------------
# int main

@dataclass
class Hyperparameters:
    # data hyperparams
    input_bin : str = 'data/fineweb10B/fineweb_train_*.bin' # input .bin to train on
    input_val_bin : str = 'data/fineweb10B/fineweb_val_*.bin' # input .bin to eval validation loss on
    # optimization hyperparams
    batch_size : int = 8 # batch size, in sequences, across all devices
    sequence_length : int = 64*1024 # sequence length, in tokens
    num_iterations : int = 1480 # number of iterations to run
    warmup_iters : int = 0
    cooldown_iters : int = 600 # number of iterations of linear warmup/cooldown for triangular or trapezoidal schedule
    weight_decay : float = 0
    # evaluation and logging hyperparams
    val_loss_every : int = 125 # every how many steps to evaluate val loss? 0 for only at the end
    val_tokens : int = 10485760 # how many tokens of validation data? it's important to keep this fixed for consistent comparisons
    save_every : int = 0 # every how many steps to save the checkpoint? 0 for only at the end
args = Hyperparameters()

# set up DDP (distributed data parallel). torchrun sets this env variable
assert torch.cuda.is_available()
dist.init_process_group(backend='nccl')
ddp_rank = int(os.environ['RANK'])
ddp_local_rank = int(os.environ['LOCAL_RANK'])
ddp_world_size = int(os.environ['WORLD_SIZE'])
device = f'cuda:{ddp_local_rank}'
torch.cuda.set_device(device)
print(f"using device: {device}")
master_process = (ddp_rank == 0) # this process will do logging, checkpointing etc.

# begin logging
logfile = None
if master_process:
    run_id = str(uuid.uuid4())
    logdir = 'logs/%s/' % run_id
    # os.makedirs(logdir, exist_ok=True)
    logfile = 'logs/%s.txt' % run_id
    # create the log file
    with open(logfile, "w") as f:
        # begin the log by printing this file (the Python code)
        f.write(code)
        f.write('='*100 + '\n')
def print0(s, logonly=False):
    if master_process:
        with open(logfile, "a") as f:
            if not logonly:
                print(s)
            f.write(s+'\n')
# log information about the hardware/software environment this is running on
# and print the full `nvidia-smi` to file
print0(f"Running pytorch {torch.version.__version__} compiled for CUDA {torch.version.cuda}\nnvidia-smi:")
import subprocess
result = subprocess.run(['nvidia-smi'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
print0(f'{result.stdout}', logonly=True)
print0('='*100, logonly=True)

# convenience variables
T = args.sequence_length
# calculate the number of steps to take in the val loop.
assert args.val_tokens % (T * ddp_world_size) == 0
val_steps = args.val_tokens // (T * ddp_world_size)
# calculate the steps of gradient accumulation required to attain the desired global batch size.
assert args.batch_size % (ddp_world_size) == 0
train_accumulation_steps = args.batch_size // ddp_world_size
assert train_accumulation_steps == 1

# load tokens
train_loader = DistributedDataLoader(args.input_bin, T, ddp_rank, ddp_world_size)
val_loader = DistributedDataLoader(args.input_val_bin, T, ddp_rank, ddp_world_size)
print0(f"Training DataLoader: total number of tokens: {train_loader.ntok_total} across {len(train_loader.files)} files")
print0(f"Validation DataLoader: total number of tokens: {val_loader.ntok_total} across {len(val_loader.files)} files")
print0('='*100, logonly=True)
x, y = train_loader.next_batch()

# there are only 50257 unique GPT-2 tokens; we extend to nearest multiple of 128 for efficiency. suggested to me by @Grad62304977.
# this originates from Karpathy's experiments.
num_vocab = 50304
model = GPT(GPTConfig(vocab_size=num_vocab, n_layer=12, n_head=6, n_embd=768))
model = model.cuda().bfloat16()
for m in model.modules():
    if isinstance(m, CastedLinear):
        m.float()
if hasattr(config, "coordinate_descent_tuning"):
    config.coordinate_descent_tuning = True # suggested by @Chillee
model = torch.compile(model)
# here we wrap model into DDP container
model = DDP(model, device_ids=[ddp_local_rank])
raw_model = model.module # always contains the "raw" unwrapped model

# init the optimizer(s)
optimizer1 = torch.optim.Adam([raw_model.transformer.wte.weight, raw_model.transformer.vte.weight], lr=0.6, betas=(0.8, 0.95), fused=True)
optimizer2 = torch.optim.Adam([raw_model.lm_head.weight], lr=0.008, betas=(0.8, 0.95), fused=True)
params = list(raw_model.transformer.h.parameters())
matrix_params = [p for p in params if p.ndim == 2]
scalar_params = [p for p in params if p.ndim < 2] + [raw_model.skip_weights]
optimizer3 = Muon(matrix_params, lr=0.05, momentum=0.95)
optimizer4 = torch.optim.Adam(scalar_params, lr=0.04, betas=(0.8, 0.95), fused=True)
optimizers = [optimizer1, optimizer2, optimizer3, optimizer4]
# learning rate decay scheduler (linear warmup and cooldown)
def get_lr(it):
    assert it <= args.num_iterations
    # 1) linear warmup for warmup_iters steps
    if it < args.warmup_iters:
        return (it+1) / args.warmup_iters
    # 2) constant lr for a while
    elif it < args.num_iterations - args.cooldown_iters:
        return 1.0
    # 3) linear cooldown
    else:
        decay_ratio = (args.num_iterations - it) / args.cooldown_iters
        return decay_ratio
schedulers = [torch.optim.lr_scheduler.LambdaLR(opt, get_lr) for opt in optimizers]

sliding_window_size = torch.tensor(64, dtype=torch.int32, device="cuda")
sw_size_prev = 64
# Start training loop
training_time_ms = 0
# start the clock
torch.cuda.synchronize()
t0 = time.perf_counter()
# begin training
for step in range(args.num_iterations + 1):
    last_step = (step == args.num_iterations)
    # This effectively ignores timing first 10 steps, which are slower for weird reasons.
    # Alternately, and slightly more correctly in terms of benchmarking, we could do 10
    # steps with dummy data first, and then re-initialize the model and reset the loader.
    if step == 10:
        training_time_ms = 0
        t0 = time.perf_counter()
    timed_steps = float('nan') if step <= 11 else (step - 10) + 1 # <= 11 to avoid bug in val

    # Set the sliding window size for the current step, in chunks of 64. By @fernbear.bsky.social
    sw_size =  64 * int((64 + (1792 - 64) * step / args.num_iterations) // 64)
    if sw_size != sw_size_prev:
        sliding_window_size.copy_(sw_size, non_blocking=True)
        sw_size_prev = sw_size

    # once in a while evaluate the validation dataset
    if (last_step or (args.val_loss_every > 0 and step % args.val_loss_every == 0)):
        # stop the clock
        torch.cuda.synchronize()
        training_time_ms += 1000 * (time.perf_counter() - t0)
        # run validation batches
        model.eval()
        val_loader.reset()
        val_loss = 0.0
        for _ in range(val_steps):
            with torch.no_grad():
                x_val, y_val = val_loader.next_batch()
                val_loss += model(x_val, y_val, sliding_window=sliding_window_size)
        dist.all_reduce(val_loss, op=dist.ReduceOp.AVG)
        val_loss /= val_steps
        # log val loss to console and to logfile
        print0(f'step:{step}/{args.num_iterations} val_loss:{val_loss:.4f} train_time:{training_time_ms:.0f}ms step_avg:{training_time_ms/(timed_steps-1):.2f}ms')
        # start the clock again
        torch.cuda.synchronize()
        t0 = time.perf_counter()

    if master_process and (last_step or (args.save_every > 0 and step % args.save_every == 0)):
        # stop the clock
        torch.cuda.synchronize()
        training_time_ms += 1000 * (time.perf_counter() - t0)
        # save the state of the training process
        log = dict(step=step, code=code, model=raw_model.state_dict(), optimizers=[opt.state_dict() for opt in optimizers])
        # torch.save(log, 'logs/%s/state_step%06d.pt' % (run_id, step))
        # start the clock again
        torch.cuda.synchronize()
        t0 = time.perf_counter()

    # bit confusing: we want to make sure to eval on 0th iteration
    # but also after the very last iteration. so we loop for step <= num_iterations
    # instead of just < num_iterations (one extra due to <=), only to do
    # the validation/sampling one last time, and then we break right here as we're done.
    if last_step:
        break

    # --------------- TRAINING SECTION BEGIN -----------------
    model.train()
    loss = model(x, y, sliding_window=sliding_window_size)
    loss.backward()
    del loss
    # advance the dataset for the next batch
    x, y = train_loader.next_batch()
    # momentum warmup for Muon
    frac = min(step/300, 1)
    for group in optimizer3.param_groups:
        group['momentum'] = (1 - frac) * 0.85 + frac * 0.95
    # step the optimizers and schedulers
    for opt, sched in zip(optimizers, schedulers):
        opt.step()
        sched.step()
    # null the gradients
    model.zero_grad(set_to_none=True)
    # --------------- TRAINING SECTION END -------------------
    # everything that follows now is just diagnostics, prints, logging, etc.
    approx_time = training_time_ms + 1000 * (time.perf_counter() - t0)
    print0(f"step:{step+1}/{args.num_iterations} train_time:{approx_time:.0f}ms step_avg:{approx_time/timed_steps:.2f}ms")

if master_process:
    print(f"peak memory consumption: {torch.cuda.max_memory_allocated() // 1024 // 1024} MiB")

# -------------------------------------------------------------------------
# clean up nice
dist.destroy_process_group()
====================================================================================================
Running pytorch 2.6.0.dev20241203+cu124 compiled for CUDA 12.4
nvidia-smi:
Sun Dec  8 07:55:35 2024       
+---------------------------------------------------------------------------------------+
| NVIDIA-SMI 535.129.03             Driver Version: 535.129.03   CUDA Version: 12.6     |
|-----------------------------------------+----------------------+----------------------+
| GPU  Name                 Persistence-M | Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp   Perf          Pwr:Usage/Cap |         Memory-Usage | GPU-Util  Compute M. |
|                                         |                      |               MIG M. |
|=========================================+======================+======================|
|   0  NVIDIA H100 80GB HBM3          On  | 00000000:65:02.0 Off |                    0 |
| N/A   36C    P0              74W / 700W |      7MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   1  NVIDIA H100 80GB HBM3          On  | 00000000:67:02.0 Off |                    0 |
| N/A   45C    P0             130W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   2  NVIDIA H100 80GB HBM3          On  | 00000000:69:02.0 Off |                    0 |
| N/A   45C    P0             105W / 700W |     27MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   3  NVIDIA H100 80GB HBM3          On  | 00000000:6B:02.0 Off |                    0 |
| N/A   39C    P0              97W / 700W |     27MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   4  NVIDIA H100 80GB HBM3          On  | 00000000:6F:02.0 Off |                    0 |
| N/A   39C    P0             117W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   5  NVIDIA H100 80GB HBM3          On  | 00000000:71:02.0 Off |                    0 |
| N/A   45C    P0             122W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   6  NVIDIA H100 80GB HBM3          On  | 00000000:73:02.0 Off |                    0 |
| N/A   46C    P0             112W / 700W |     37MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   7  NVIDIA H100 80GB HBM3          On  | 00000000:75:02.0 Off |                    0 |
| N/A   38C    P0             124W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
                                                                                         
+---------------------------------------------------------------------------------------+
| Processes:                                                                            |
|  GPU   GI   CI        PID   Type   Process name                            GPU Memory |
|        ID   ID                                                             Usage      |
|=======================================================================================|
+---------------------------------------------------------------------------------------+

====================================================================================================
Training DataLoader: total number of tokens: 3200000000 across 32 files
Validation DataLoader: total number of tokens: 100000000 across 1 files
====================================================================================================
step:0/1480 val_loss:10.8258 train_time:0ms step_avg:nanms
step:1/1480 train_time:23203ms step_avg:nanms
step:2/1480 train_time:23290ms step_avg:nanms
step:3/1480 train_time:23429ms step_avg:nanms
step:4/1480 train_time:23570ms step_avg:nanms
step:5/1480 train_time:23712ms step_avg:nanms
step:6/1480 train_time:23853ms step_avg:nanms
step:7/1480 train_time:23994ms step_avg:nanms
step:8/1480 train_time:24137ms step_avg:nanms
step:9/1480 train_time:24281ms step_avg:nanms
step:10/1480 train_time:24427ms step_avg:nanms
step:11/1480 train_time:143ms step_avg:nanms
step:12/1480 train_time:286ms step_avg:nanms
step:13/1480 train_time:428ms step_avg:142.64ms
step:14/1480 train_time:568ms step_avg:142.08ms
step:15/1480 train_time:710ms step_avg:142.04ms
step:16/1480 train_time:853ms step_avg:142.21ms
step:17/1480 train_time:997ms step_avg:142.44ms
step:18/1480 train_time:1140ms step_avg:142.44ms
step:19/1480 train_time:1284ms step_avg:142.69ms
step:20/1480 train_time:1426ms step_avg:142.64ms
step:21/1480 train_time:1568ms step_avg:142.54ms
step:22/1480 train_time:1710ms step_avg:142.54ms
step:23/1480 train_time:1852ms step_avg:142.48ms
step:24/1480 train_time:1993ms step_avg:142.38ms
step:25/1480 train_time:2135ms step_avg:142.34ms
step:26/1480 train_time:2280ms step_avg:142.49ms
step:27/1480 train_time:2424ms step_avg:142.57ms
step:28/1480 train_time:2567ms step_avg:142.59ms
step:29/1480 train_time:2709ms step_avg:142.58ms
step:30/1480 train_time:2852ms step_avg:142.61ms
step:31/1480 train_time:2996ms step_avg:142.65ms
step:32/1480 train_time:3138ms step_avg:142.61ms
step:33/1480 train_time:3281ms step_avg:142.64ms
step:34/1480 train_time:3425ms step_avg:142.70ms
step:35/1480 train_time:3567ms step_avg:142.68ms
step:36/1480 train_time:3710ms step_avg:142.68ms
step:37/1480 train_time:3851ms step_avg:142.63ms
step:38/1480 train_time:3994ms step_avg:142.64ms
step:39/1480 train_time:4137ms step_avg:142.65ms
step:40/1480 train_time:4283ms step_avg:142.75ms
step:41/1480 train_time:4426ms step_avg:142.78ms
step:42/1480 train_time:4568ms step_avg:142.76ms
step:43/1480 train_time:4711ms step_avg:142.76ms
step:44/1480 train_time:4852ms step_avg:142.71ms
step:45/1480 train_time:4993ms step_avg:142.66ms
step:46/1480 train_time:5135ms step_avg:142.65ms
step:47/1480 train_time:5280ms step_avg:142.71ms
step:48/1480 train_time:5424ms step_avg:142.73ms
step:49/1480 train_time:5566ms step_avg:142.73ms
step:50/1480 train_time:5710ms step_avg:142.74ms
step:51/1480 train_time:5851ms step_avg:142.70ms
step:52/1480 train_time:5992ms step_avg:142.67ms
step:53/1480 train_time:6134ms step_avg:142.64ms
step:54/1480 train_time:6276ms step_avg:142.64ms
step:55/1480 train_time:6420ms step_avg:142.67ms
step:56/1480 train_time:6564ms step_avg:142.70ms
step:57/1480 train_time:6707ms step_avg:142.71ms
step:58/1480 train_time:6849ms step_avg:142.69ms
step:59/1480 train_time:6990ms step_avg:142.66ms
step:60/1480 train_time:7132ms step_avg:142.64ms
step:61/1480 train_time:7273ms step_avg:142.62ms
step:62/1480 train_time:7416ms step_avg:142.61ms
step:63/1480 train_time:7558ms step_avg:142.60ms
step:64/1480 train_time:7702ms step_avg:142.63ms
step:65/1480 train_time:7844ms step_avg:142.62ms
step:66/1480 train_time:7987ms step_avg:142.62ms
step:67/1480 train_time:8129ms step_avg:142.61ms
step:68/1480 train_time:8271ms step_avg:142.61ms
step:69/1480 train_time:8415ms step_avg:142.63ms
step:70/1480 train_time:8558ms step_avg:142.63ms
step:71/1480 train_time:8701ms step_avg:142.64ms
step:72/1480 train_time:8844ms step_avg:142.65ms
step:73/1480 train_time:8987ms step_avg:142.66ms
step:74/1480 train_time:9129ms step_avg:142.64ms
step:75/1480 train_time:9269ms step_avg:142.61ms
step:76/1480 train_time:9413ms step_avg:142.63ms
step:77/1480 train_time:9556ms step_avg:142.63ms
step:78/1480 train_time:9700ms step_avg:142.64ms
step:79/1480 train_time:9842ms step_avg:142.63ms
step:80/1480 train_time:9986ms step_avg:142.66ms
step:81/1480 train_time:10128ms step_avg:142.65ms
step:82/1480 train_time:10269ms step_avg:142.62ms
step:83/1480 train_time:10413ms step_avg:142.64ms
step:84/1480 train_time:10557ms step_avg:142.66ms
step:85/1480 train_time:10702ms step_avg:142.69ms
step:86/1480 train_time:10846ms step_avg:142.71ms
step:87/1480 train_time:10988ms step_avg:142.70ms
step:88/1480 train_time:11130ms step_avg:142.69ms
step:89/1480 train_time:11271ms step_avg:142.67ms
step:90/1480 train_time:11412ms step_avg:142.66ms
step:91/1480 train_time:11554ms step_avg:142.64ms
step:92/1480 train_time:11697ms step_avg:142.65ms
step:93/1480 train_time:11842ms step_avg:142.67ms
step:94/1480 train_time:11986ms step_avg:142.69ms
step:95/1480 train_time:12129ms step_avg:142.69ms
step:96/1480 train_time:12270ms step_avg:142.68ms
step:97/1480 train_time:12412ms step_avg:142.67ms
step:98/1480 train_time:12554ms step_avg:142.66ms
step:99/1480 train_time:12696ms step_avg:142.65ms
step:100/1480 train_time:12839ms step_avg:142.66ms
step:101/1480 train_time:12983ms step_avg:142.67ms
step:102/1480 train_time:13126ms step_avg:142.68ms
step:103/1480 train_time:13268ms step_avg:142.66ms
step:104/1480 train_time:13411ms step_avg:142.67ms
step:105/1480 train_time:13554ms step_avg:142.67ms
step:106/1480 train_time:13695ms step_avg:142.65ms
step:107/1480 train_time:13838ms step_avg:142.66ms
step:108/1480 train_time:13982ms step_avg:142.67ms
step:109/1480 train_time:14125ms step_avg:142.68ms
step:110/1480 train_time:14267ms step_avg:142.67ms
step:111/1480 train_time:14412ms step_avg:142.69ms
step:112/1480 train_time:14558ms step_avg:142.73ms
step:113/1480 train_time:14706ms step_avg:142.78ms
step:114/1480 train_time:14854ms step_avg:142.83ms
step:115/1480 train_time:15001ms step_avg:142.87ms
step:116/1480 train_time:15149ms step_avg:142.91ms
step:117/1480 train_time:15295ms step_avg:142.94ms
step:118/1480 train_time:15442ms step_avg:142.98ms
step:119/1480 train_time:15590ms step_avg:143.02ms
step:120/1480 train_time:15735ms step_avg:143.05ms
step:121/1480 train_time:15883ms step_avg:143.09ms
step:122/1480 train_time:16031ms step_avg:143.13ms
step:123/1480 train_time:16176ms step_avg:143.15ms
step:124/1480 train_time:16326ms step_avg:143.21ms
step:125/1480 train_time:16473ms step_avg:143.24ms
step:125/1480 val_loss:4.4058 train_time:16530ms step_avg:143.74ms
step:126/1480 train_time:16626ms step_avg:143.33ms
step:127/1480 train_time:16776ms step_avg:143.39ms
step:128/1480 train_time:16922ms step_avg:143.41ms
step:129/1480 train_time:17068ms step_avg:143.43ms
step:130/1480 train_time:17215ms step_avg:143.46ms
step:131/1480 train_time:17360ms step_avg:143.47ms
step:132/1480 train_time:17508ms step_avg:143.51ms
step:133/1480 train_time:17658ms step_avg:143.56ms
step:134/1480 train_time:17806ms step_avg:143.59ms
step:135/1480 train_time:17953ms step_avg:143.63ms
step:136/1480 train_time:18099ms step_avg:143.64ms
step:137/1480 train_time:18245ms step_avg:143.66ms
step:138/1480 train_time:18392ms step_avg:143.69ms
step:139/1480 train_time:18539ms step_avg:143.71ms
step:140/1480 train_time:18685ms step_avg:143.73ms
step:141/1480 train_time:18834ms step_avg:143.77ms
step:142/1480 train_time:18979ms step_avg:143.78ms
step:143/1480 train_time:19125ms step_avg:143.80ms
step:144/1480 train_time:19271ms step_avg:143.81ms
step:145/1480 train_time:19418ms step_avg:143.83ms
step:146/1480 train_time:19562ms step_avg:143.84ms
step:147/1480 train_time:19711ms step_avg:143.87ms
step:148/1480 train_time:19859ms step_avg:143.90ms
step:149/1480 train_time:20005ms step_avg:143.92ms
step:150/1480 train_time:20153ms step_avg:143.95ms
step:151/1480 train_time:20299ms step_avg:143.97ms
step:152/1480 train_time:20447ms step_avg:143.99ms
step:153/1480 train_time:20595ms step_avg:144.02ms
step:154/1480 train_time:20742ms step_avg:144.04ms
step:155/1480 train_time:20890ms step_avg:144.07ms
step:156/1480 train_time:21037ms step_avg:144.09ms
step:157/1480 train_time:21183ms step_avg:144.10ms
step:158/1480 train_time:21330ms step_avg:144.12ms
step:159/1480 train_time:21477ms step_avg:144.14ms
step:160/1480 train_time:21622ms step_avg:144.15ms
step:161/1480 train_time:21769ms step_avg:144.17ms
step:162/1480 train_time:21918ms step_avg:144.20ms
step:163/1480 train_time:22063ms step_avg:144.20ms
step:164/1480 train_time:22212ms step_avg:144.23ms
step:165/1480 train_time:22358ms step_avg:144.25ms
step:166/1480 train_time:22504ms step_avg:144.26ms
step:167/1480 train_time:22652ms step_avg:144.28ms
step:168/1480 train_time:22798ms step_avg:144.29ms
step:169/1480 train_time:22945ms step_avg:144.31ms
step:170/1480 train_time:23091ms step_avg:144.32ms
step:171/1480 train_time:23239ms step_avg:144.34ms
step:172/1480 train_time:23384ms step_avg:144.34ms
step:173/1480 train_time:23531ms step_avg:144.36ms
step:174/1480 train_time:23678ms step_avg:144.38ms
step:175/1480 train_time:23824ms step_avg:144.39ms
step:176/1480 train_time:23972ms step_avg:144.41ms
step:177/1480 train_time:24118ms step_avg:144.42ms
step:178/1480 train_time:24263ms step_avg:144.42ms
step:179/1480 train_time:24411ms step_avg:144.44ms
step:180/1480 train_time:24557ms step_avg:144.46ms
step:181/1480 train_time:24703ms step_avg:144.46ms
step:182/1480 train_time:24851ms step_avg:144.48ms
step:183/1480 train_time:24997ms step_avg:144.49ms
step:184/1480 train_time:25143ms step_avg:144.50ms
step:185/1480 train_time:25290ms step_avg:144.51ms
step:186/1480 train_time:25437ms step_avg:144.53ms
step:187/1480 train_time:25583ms step_avg:144.54ms
step:188/1480 train_time:25730ms step_avg:144.55ms
step:189/1480 train_time:25877ms step_avg:144.57ms
step:190/1480 train_time:26024ms step_avg:144.58ms
step:191/1480 train_time:26170ms step_avg:144.58ms
step:192/1480 train_time:26318ms step_avg:144.60ms
step:193/1480 train_time:26462ms step_avg:144.60ms
step:194/1480 train_time:26609ms step_avg:144.62ms
step:195/1480 train_time:26757ms step_avg:144.63ms
step:196/1480 train_time:26902ms step_avg:144.63ms
step:197/1480 train_time:27049ms step_avg:144.65ms
step:198/1480 train_time:27196ms step_avg:144.66ms
step:199/1480 train_time:27342ms step_avg:144.66ms
step:200/1480 train_time:27490ms step_avg:144.68ms
step:201/1480 train_time:27637ms step_avg:144.69ms
step:202/1480 train_time:27783ms step_avg:144.70ms
step:203/1480 train_time:27930ms step_avg:144.72ms
step:204/1480 train_time:28077ms step_avg:144.73ms
step:205/1480 train_time:28223ms step_avg:144.74ms
step:206/1480 train_time:28371ms step_avg:144.75ms
step:207/1480 train_time:28519ms step_avg:144.77ms
step:208/1480 train_time:28664ms step_avg:144.77ms
step:209/1480 train_time:28811ms step_avg:144.78ms
step:210/1480 train_time:28958ms step_avg:144.79ms
step:211/1480 train_time:29103ms step_avg:144.79ms
step:212/1480 train_time:29250ms step_avg:144.80ms
step:213/1480 train_time:29397ms step_avg:144.81ms
step:214/1480 train_time:29545ms step_avg:144.83ms
step:215/1480 train_time:29692ms step_avg:144.84ms
step:216/1480 train_time:29839ms step_avg:144.85ms
step:217/1480 train_time:29985ms step_avg:144.85ms
step:218/1480 train_time:30132ms step_avg:144.86ms
step:219/1480 train_time:30278ms step_avg:144.87ms
step:220/1480 train_time:30424ms step_avg:144.88ms
step:221/1480 train_time:30574ms step_avg:144.90ms
step:222/1480 train_time:30724ms step_avg:144.92ms
step:223/1480 train_time:30874ms step_avg:144.95ms
step:224/1480 train_time:31025ms step_avg:144.98ms
step:225/1480 train_time:31175ms step_avg:145.00ms
step:226/1480 train_time:31325ms step_avg:145.02ms
step:227/1480 train_time:31476ms step_avg:145.05ms
step:228/1480 train_time:31626ms step_avg:145.07ms
step:229/1480 train_time:31777ms step_avg:145.10ms
step:230/1480 train_time:31928ms step_avg:145.13ms
step:231/1480 train_time:32078ms step_avg:145.15ms
step:232/1480 train_time:32228ms step_avg:145.17ms
step:233/1480 train_time:32378ms step_avg:145.19ms
step:234/1480 train_time:32528ms step_avg:145.21ms
step:235/1480 train_time:32679ms step_avg:145.24ms
step:236/1480 train_time:32833ms step_avg:145.28ms
step:237/1480 train_time:32980ms step_avg:145.29ms
step:238/1480 train_time:33130ms step_avg:145.31ms
step:239/1480 train_time:33280ms step_avg:145.33ms
step:240/1480 train_time:33430ms step_avg:145.35ms
step:241/1480 train_time:33580ms step_avg:145.37ms
step:242/1480 train_time:33731ms step_avg:145.39ms
step:243/1480 train_time:33881ms step_avg:145.41ms
step:244/1480 train_time:34032ms step_avg:145.44ms
step:245/1480 train_time:34182ms step_avg:145.46ms
step:246/1480 train_time:34334ms step_avg:145.48ms
step:247/1480 train_time:34484ms step_avg:145.50ms
step:248/1480 train_time:34635ms step_avg:145.52ms
step:249/1480 train_time:34785ms step_avg:145.54ms
step:250/1480 train_time:34935ms step_avg:145.56ms
step:250/1480 val_loss:3.9913 train_time:34993ms step_avg:145.80ms
step:251/1480 train_time:35091ms step_avg:145.60ms
step:252/1480 train_time:35242ms step_avg:145.63ms
step:253/1480 train_time:35391ms step_avg:145.64ms
step:254/1480 train_time:35540ms step_avg:145.65ms
step:255/1480 train_time:35689ms step_avg:145.67ms
step:256/1480 train_time:35837ms step_avg:145.68ms
step:257/1480 train_time:35988ms step_avg:145.70ms
step:258/1480 train_time:36141ms step_avg:145.73ms
step:259/1480 train_time:36293ms step_avg:145.76ms
step:260/1480 train_time:36445ms step_avg:145.78ms
step:261/1480 train_time:36595ms step_avg:145.80ms
step:262/1480 train_time:36748ms step_avg:145.83ms
step:263/1480 train_time:36895ms step_avg:145.83ms
step:264/1480 train_time:37046ms step_avg:145.85ms
step:265/1480 train_time:37196ms step_avg:145.87ms
step:266/1480 train_time:37348ms step_avg:145.89ms
step:267/1480 train_time:37498ms step_avg:145.91ms
step:268/1480 train_time:37649ms step_avg:145.93ms
step:269/1480 train_time:37798ms step_avg:145.94ms
step:270/1480 train_time:37948ms step_avg:145.96ms
step:271/1480 train_time:38097ms step_avg:145.97ms
step:272/1480 train_time:38249ms step_avg:145.99ms
step:273/1480 train_time:38397ms step_avg:146.00ms
step:274/1480 train_time:38548ms step_avg:146.01ms
step:275/1480 train_time:38698ms step_avg:146.03ms
step:276/1480 train_time:38849ms step_avg:146.05ms
step:277/1480 train_time:38999ms step_avg:146.06ms
step:278/1480 train_time:39150ms step_avg:146.08ms
step:279/1480 train_time:39299ms step_avg:146.09ms
step:280/1480 train_time:39450ms step_avg:146.11ms
step:281/1480 train_time:39600ms step_avg:146.13ms
step:282/1480 train_time:39752ms step_avg:146.15ms
step:283/1480 train_time:39903ms step_avg:146.17ms
step:284/1480 train_time:40055ms step_avg:146.19ms
step:285/1480 train_time:40205ms step_avg:146.20ms
step:286/1480 train_time:40355ms step_avg:146.21ms
step:287/1480 train_time:40505ms step_avg:146.23ms
step:288/1480 train_time:40655ms step_avg:146.24ms
step:289/1480 train_time:40806ms step_avg:146.26ms
step:290/1480 train_time:40955ms step_avg:146.27ms
step:291/1480 train_time:41106ms step_avg:146.28ms
step:292/1480 train_time:41255ms step_avg:146.30ms
step:293/1480 train_time:41406ms step_avg:146.31ms
step:294/1480 train_time:41556ms step_avg:146.32ms
step:295/1480 train_time:41708ms step_avg:146.34ms
step:296/1480 train_time:41857ms step_avg:146.35ms
step:297/1480 train_time:42007ms step_avg:146.37ms
step:298/1480 train_time:42157ms step_avg:146.38ms
step:299/1480 train_time:42308ms step_avg:146.39ms
step:300/1480 train_time:42458ms step_avg:146.41ms
step:301/1480 train_time:42609ms step_avg:146.42ms
step:302/1480 train_time:42760ms step_avg:146.44ms
step:303/1480 train_time:42910ms step_avg:146.45ms
step:304/1480 train_time:43060ms step_avg:146.46ms
step:305/1480 train_time:43210ms step_avg:146.47ms
step:306/1480 train_time:43361ms step_avg:146.49ms
step:307/1480 train_time:43510ms step_avg:146.50ms
step:308/1480 train_time:43661ms step_avg:146.52ms
step:309/1480 train_time:43812ms step_avg:146.53ms
step:310/1480 train_time:43963ms step_avg:146.54ms
step:311/1480 train_time:44113ms step_avg:146.55ms
step:312/1480 train_time:44265ms step_avg:146.57ms
step:313/1480 train_time:44415ms step_avg:146.58ms
step:314/1480 train_time:44566ms step_avg:146.60ms
step:315/1480 train_time:44715ms step_avg:146.61ms
step:316/1480 train_time:44867ms step_avg:146.62ms
step:317/1480 train_time:45016ms step_avg:146.63ms
step:318/1480 train_time:45168ms step_avg:146.65ms
step:319/1480 train_time:45318ms step_avg:146.66ms
step:320/1480 train_time:45470ms step_avg:146.68ms
step:321/1480 train_time:45619ms step_avg:146.69ms
step:322/1480 train_time:45770ms step_avg:146.70ms
step:323/1480 train_time:45920ms step_avg:146.71ms
step:324/1480 train_time:46070ms step_avg:146.72ms
step:325/1480 train_time:46220ms step_avg:146.73ms
step:326/1480 train_time:46371ms step_avg:146.74ms
step:327/1480 train_time:46522ms step_avg:146.76ms
step:328/1480 train_time:46672ms step_avg:146.77ms
step:329/1480 train_time:46824ms step_avg:146.78ms
step:330/1480 train_time:46976ms step_avg:146.80ms
step:331/1480 train_time:47129ms step_avg:146.82ms
step:332/1480 train_time:47283ms step_avg:146.84ms
step:333/1480 train_time:47436ms step_avg:146.86ms
step:334/1480 train_time:47589ms step_avg:146.88ms
step:335/1480 train_time:47743ms step_avg:146.90ms
step:336/1480 train_time:47896ms step_avg:146.92ms
step:337/1480 train_time:48051ms step_avg:146.95ms
step:338/1480 train_time:48205ms step_avg:146.97ms
step:339/1480 train_time:48358ms step_avg:146.99ms
step:340/1480 train_time:48511ms step_avg:147.00ms
step:341/1480 train_time:48665ms step_avg:147.02ms
step:342/1480 train_time:48818ms step_avg:147.04ms
step:343/1480 train_time:48973ms step_avg:147.06ms
step:344/1480 train_time:49127ms step_avg:147.09ms
step:345/1480 train_time:49281ms step_avg:147.11ms
step:346/1480 train_time:49436ms step_avg:147.13ms
step:347/1480 train_time:49590ms step_avg:147.15ms
step:348/1480 train_time:49744ms step_avg:147.17ms
step:349/1480 train_time:49897ms step_avg:147.19ms
step:350/1480 train_time:50051ms step_avg:147.21ms
step:351/1480 train_time:50205ms step_avg:147.23ms
step:352/1480 train_time:50359ms step_avg:147.25ms
step:353/1480 train_time:50513ms step_avg:147.27ms
step:354/1480 train_time:50666ms step_avg:147.29ms
step:355/1480 train_time:50821ms step_avg:147.31ms
step:356/1480 train_time:50974ms step_avg:147.32ms
step:357/1480 train_time:51129ms step_avg:147.34ms
step:358/1480 train_time:51282ms step_avg:147.36ms
step:359/1480 train_time:51437ms step_avg:147.38ms
step:360/1480 train_time:51591ms step_avg:147.40ms
step:361/1480 train_time:51745ms step_avg:147.42ms
step:362/1480 train_time:51900ms step_avg:147.44ms
step:363/1480 train_time:52054ms step_avg:147.46ms
step:364/1480 train_time:52208ms step_avg:147.48ms
step:365/1480 train_time:52361ms step_avg:147.50ms
step:366/1480 train_time:52514ms step_avg:147.51ms
step:367/1480 train_time:52668ms step_avg:147.53ms
step:368/1480 train_time:52821ms step_avg:147.54ms
step:369/1480 train_time:52974ms step_avg:147.56ms
step:370/1480 train_time:53128ms step_avg:147.58ms
step:371/1480 train_time:53280ms step_avg:147.59ms
step:372/1480 train_time:53434ms step_avg:147.61ms
step:373/1480 train_time:53588ms step_avg:147.63ms
step:374/1480 train_time:53741ms step_avg:147.64ms
step:375/1480 train_time:53895ms step_avg:147.66ms
step:375/1480 val_loss:3.8108 train_time:53955ms step_avg:147.82ms
step:376/1480 train_time:54053ms step_avg:147.68ms
step:377/1480 train_time:54206ms step_avg:147.70ms
step:378/1480 train_time:54360ms step_avg:147.72ms
step:379/1480 train_time:54512ms step_avg:147.73ms
step:380/1480 train_time:54664ms step_avg:147.74ms
step:381/1480 train_time:54817ms step_avg:147.76ms
step:382/1480 train_time:54971ms step_avg:147.77ms
step:383/1480 train_time:55126ms step_avg:147.79ms
step:384/1480 train_time:55279ms step_avg:147.81ms
step:385/1480 train_time:55433ms step_avg:147.82ms
step:386/1480 train_time:55586ms step_avg:147.84ms
step:387/1480 train_time:55740ms step_avg:147.85ms
step:388/1480 train_time:55893ms step_avg:147.86ms
step:389/1480 train_time:56047ms step_avg:147.88ms
step:390/1480 train_time:56202ms step_avg:147.90ms
step:391/1480 train_time:56358ms step_avg:147.92ms
step:392/1480 train_time:56511ms step_avg:147.93ms
step:393/1480 train_time:56666ms step_avg:147.95ms
step:394/1480 train_time:56820ms step_avg:147.97ms
step:395/1480 train_time:56973ms step_avg:147.98ms
step:396/1480 train_time:57126ms step_avg:148.00ms
step:397/1480 train_time:57280ms step_avg:148.01ms
step:398/1480 train_time:57434ms step_avg:148.03ms
step:399/1480 train_time:57588ms step_avg:148.04ms
step:400/1480 train_time:57742ms step_avg:148.06ms
step:401/1480 train_time:57896ms step_avg:148.07ms
step:402/1480 train_time:58049ms step_avg:148.09ms
step:403/1480 train_time:58202ms step_avg:148.10ms
step:404/1480 train_time:58357ms step_avg:148.11ms
step:405/1480 train_time:58511ms step_avg:148.13ms
step:406/1480 train_time:58665ms step_avg:148.14ms
step:407/1480 train_time:58818ms step_avg:148.16ms
step:408/1480 train_time:58971ms step_avg:148.17ms
step:409/1480 train_time:59125ms step_avg:148.18ms
step:410/1480 train_time:59278ms step_avg:148.20ms
step:411/1480 train_time:59433ms step_avg:148.21ms
step:412/1480 train_time:59587ms step_avg:148.23ms
step:413/1480 train_time:59740ms step_avg:148.24ms
step:414/1480 train_time:59894ms step_avg:148.25ms
step:415/1480 train_time:60049ms step_avg:148.27ms
step:416/1480 train_time:60201ms step_avg:148.28ms
step:417/1480 train_time:60355ms step_avg:148.29ms
step:418/1480 train_time:60509ms step_avg:148.31ms
step:419/1480 train_time:60662ms step_avg:148.32ms
step:420/1480 train_time:60815ms step_avg:148.33ms
step:421/1480 train_time:60968ms step_avg:148.34ms
step:422/1480 train_time:61122ms step_avg:148.35ms
step:423/1480 train_time:61274ms step_avg:148.36ms
step:424/1480 train_time:61428ms step_avg:148.38ms
step:425/1480 train_time:61582ms step_avg:148.39ms
step:426/1480 train_time:61736ms step_avg:148.40ms
step:427/1480 train_time:61890ms step_avg:148.42ms
step:428/1480 train_time:62043ms step_avg:148.43ms
step:429/1480 train_time:62195ms step_avg:148.44ms
step:430/1480 train_time:62349ms step_avg:148.45ms
step:431/1480 train_time:62502ms step_avg:148.46ms
step:432/1480 train_time:62656ms step_avg:148.47ms
step:433/1480 train_time:62809ms step_avg:148.48ms
step:434/1480 train_time:62963ms step_avg:148.50ms
step:435/1480 train_time:63117ms step_avg:148.51ms
step:436/1480 train_time:63271ms step_avg:148.52ms
step:437/1480 train_time:63425ms step_avg:148.54ms
step:438/1480 train_time:63578ms step_avg:148.55ms
step:439/1480 train_time:63733ms step_avg:148.56ms
step:440/1480 train_time:63888ms step_avg:148.58ms
step:441/1480 train_time:64044ms step_avg:148.59ms
step:442/1480 train_time:64201ms step_avg:148.61ms
step:443/1480 train_time:64358ms step_avg:148.63ms
step:444/1480 train_time:64515ms step_avg:148.65ms
step:445/1480 train_time:64669ms step_avg:148.67ms
step:446/1480 train_time:64825ms step_avg:148.68ms
step:447/1480 train_time:64981ms step_avg:148.70ms
step:448/1480 train_time:65139ms step_avg:148.72ms
step:449/1480 train_time:65296ms step_avg:148.74ms
step:450/1480 train_time:65455ms step_avg:148.76ms
step:451/1480 train_time:65612ms step_avg:148.78ms
step:452/1480 train_time:65767ms step_avg:148.80ms
step:453/1480 train_time:65924ms step_avg:148.81ms
step:454/1480 train_time:66079ms step_avg:148.83ms
step:455/1480 train_time:66237ms step_avg:148.85ms
step:456/1480 train_time:66394ms step_avg:148.86ms
step:457/1480 train_time:66551ms step_avg:148.88ms
step:458/1480 train_time:66707ms step_avg:148.90ms
step:459/1480 train_time:66865ms step_avg:148.92ms
step:460/1480 train_time:67022ms step_avg:148.94ms
step:461/1480 train_time:67182ms step_avg:148.96ms
step:462/1480 train_time:67339ms step_avg:148.98ms
step:463/1480 train_time:67496ms step_avg:149.00ms
step:464/1480 train_time:67653ms step_avg:149.02ms
step:465/1480 train_time:67809ms step_avg:149.03ms
step:466/1480 train_time:67966ms step_avg:149.05ms
step:467/1480 train_time:68123ms step_avg:149.07ms
step:468/1480 train_time:68279ms step_avg:149.08ms
step:469/1480 train_time:68434ms step_avg:149.09ms
step:470/1480 train_time:68590ms step_avg:149.11ms
step:471/1480 train_time:68746ms step_avg:149.12ms
step:472/1480 train_time:68902ms step_avg:149.14ms
step:473/1480 train_time:69060ms step_avg:149.16ms
step:474/1480 train_time:69219ms step_avg:149.18ms
step:475/1480 train_time:69376ms step_avg:149.20ms
step:476/1480 train_time:69532ms step_avg:149.21ms
step:477/1480 train_time:69689ms step_avg:149.23ms
step:478/1480 train_time:69845ms step_avg:149.24ms
step:479/1480 train_time:70001ms step_avg:149.26ms
step:480/1480 train_time:70160ms step_avg:149.28ms
step:481/1480 train_time:70318ms step_avg:149.29ms
step:482/1480 train_time:70474ms step_avg:149.31ms
step:483/1480 train_time:70631ms step_avg:149.33ms
step:484/1480 train_time:70788ms step_avg:149.34ms
step:485/1480 train_time:70945ms step_avg:149.36ms
step:486/1480 train_time:71102ms step_avg:149.37ms
step:487/1480 train_time:71260ms step_avg:149.39ms
step:488/1480 train_time:71419ms step_avg:149.41ms
step:489/1480 train_time:71575ms step_avg:149.43ms
step:490/1480 train_time:71730ms step_avg:149.44ms
step:491/1480 train_time:71886ms step_avg:149.45ms
step:492/1480 train_time:72042ms step_avg:149.46ms
step:493/1480 train_time:72199ms step_avg:149.48ms
step:494/1480 train_time:72358ms step_avg:149.50ms
step:495/1480 train_time:72515ms step_avg:149.52ms
step:496/1480 train_time:72674ms step_avg:149.53ms
step:497/1480 train_time:72830ms step_avg:149.55ms
step:498/1480 train_time:72986ms step_avg:149.56ms
step:499/1480 train_time:73142ms step_avg:149.58ms
step:500/1480 train_time:73298ms step_avg:149.59ms
step:500/1480 val_loss:3.6887 train_time:73360ms step_avg:149.71ms
step:501/1480 train_time:73458ms step_avg:149.61ms
step:502/1480 train_time:73617ms step_avg:149.63ms
step:503/1480 train_time:73774ms step_avg:149.64ms
step:504/1480 train_time:73930ms step_avg:149.66ms
step:505/1480 train_time:74086ms step_avg:149.67ms
step:506/1480 train_time:74244ms step_avg:149.69ms
step:507/1480 train_time:74400ms step_avg:149.70ms
step:508/1480 train_time:74558ms step_avg:149.71ms
step:509/1480 train_time:74715ms step_avg:149.73ms
step:510/1480 train_time:74871ms step_avg:149.74ms
step:511/1480 train_time:75029ms step_avg:149.76ms
step:512/1480 train_time:75185ms step_avg:149.77ms
step:513/1480 train_time:75344ms step_avg:149.79ms
step:514/1480 train_time:75502ms step_avg:149.80ms
step:515/1480 train_time:75659ms step_avg:149.82ms
step:516/1480 train_time:75818ms step_avg:149.84ms
step:517/1480 train_time:75974ms step_avg:149.85ms
step:518/1480 train_time:76130ms step_avg:149.86ms
step:519/1480 train_time:76287ms step_avg:149.88ms
step:520/1480 train_time:76446ms step_avg:149.89ms
step:521/1480 train_time:76604ms step_avg:149.91ms
step:522/1480 train_time:76762ms step_avg:149.93ms
step:523/1480 train_time:76919ms step_avg:149.94ms
step:524/1480 train_time:77076ms step_avg:149.95ms
step:525/1480 train_time:77232ms step_avg:149.96ms
step:526/1480 train_time:77388ms step_avg:149.98ms
step:527/1480 train_time:77545ms step_avg:149.99ms
step:528/1480 train_time:77702ms step_avg:150.00ms
step:529/1480 train_time:77860ms step_avg:150.02ms
step:530/1480 train_time:78017ms step_avg:150.03ms
step:531/1480 train_time:78174ms step_avg:150.05ms
step:532/1480 train_time:78330ms step_avg:150.06ms
step:533/1480 train_time:78486ms step_avg:150.07ms
step:534/1480 train_time:78642ms step_avg:150.08ms
step:535/1480 train_time:78798ms step_avg:150.09ms
step:536/1480 train_time:78956ms step_avg:150.11ms
step:537/1480 train_time:79114ms step_avg:150.12ms
step:538/1480 train_time:79271ms step_avg:150.13ms
step:539/1480 train_time:79430ms step_avg:150.15ms
step:540/1480 train_time:79586ms step_avg:150.16ms
step:541/1480 train_time:79744ms step_avg:150.18ms
step:542/1480 train_time:79900ms step_avg:150.19ms
step:543/1480 train_time:80055ms step_avg:150.20ms
step:544/1480 train_time:80212ms step_avg:150.21ms
step:545/1480 train_time:80369ms step_avg:150.22ms
step:546/1480 train_time:80526ms step_avg:150.24ms
step:547/1480 train_time:80683ms step_avg:150.25ms
step:548/1480 train_time:80842ms step_avg:150.26ms
step:549/1480 train_time:81000ms step_avg:150.28ms
step:550/1480 train_time:81157ms step_avg:150.29ms
step:551/1480 train_time:81315ms step_avg:150.31ms
step:552/1480 train_time:81473ms step_avg:150.32ms
step:553/1480 train_time:81632ms step_avg:150.34ms
step:554/1480 train_time:81791ms step_avg:150.35ms
step:555/1480 train_time:81950ms step_avg:150.37ms
step:556/1480 train_time:82108ms step_avg:150.38ms
step:557/1480 train_time:82267ms step_avg:150.40ms
step:558/1480 train_time:82426ms step_avg:150.41ms
step:559/1480 train_time:82586ms step_avg:150.43ms
step:560/1480 train_time:82748ms step_avg:150.45ms
step:561/1480 train_time:82908ms step_avg:150.47ms
step:562/1480 train_time:83068ms step_avg:150.49ms
step:563/1480 train_time:83227ms step_avg:150.50ms
step:564/1480 train_time:83387ms step_avg:150.52ms
step:565/1480 train_time:83547ms step_avg:150.53ms
step:566/1480 train_time:83707ms step_avg:150.55ms
step:567/1480 train_time:83867ms step_avg:150.57ms
step:568/1480 train_time:84026ms step_avg:150.58ms
step:569/1480 train_time:84185ms step_avg:150.60ms
step:570/1480 train_time:84345ms step_avg:150.62ms
step:571/1480 train_time:84505ms step_avg:150.63ms
step:572/1480 train_time:84665ms step_avg:150.65ms
step:573/1480 train_time:84826ms step_avg:150.67ms
step:574/1480 train_time:84986ms step_avg:150.68ms
step:575/1480 train_time:85148ms step_avg:150.70ms
step:576/1480 train_time:85308ms step_avg:150.72ms
step:577/1480 train_time:85467ms step_avg:150.74ms
step:578/1480 train_time:85626ms step_avg:150.75ms
step:579/1480 train_time:85785ms step_avg:150.76ms
step:580/1480 train_time:85944ms step_avg:150.78ms
step:581/1480 train_time:86106ms step_avg:150.80ms
step:582/1480 train_time:86265ms step_avg:150.81ms
step:583/1480 train_time:86426ms step_avg:150.83ms
step:584/1480 train_time:86586ms step_avg:150.85ms
step:585/1480 train_time:86746ms step_avg:150.86ms
step:586/1480 train_time:86907ms step_avg:150.88ms
step:587/1480 train_time:87067ms step_avg:150.90ms
step:588/1480 train_time:87226ms step_avg:150.91ms
step:589/1480 train_time:87386ms step_avg:150.93ms
step:590/1480 train_time:87548ms step_avg:150.94ms
step:591/1480 train_time:87706ms step_avg:150.96ms
step:592/1480 train_time:87866ms step_avg:150.97ms
step:593/1480 train_time:88026ms step_avg:150.99ms
step:594/1480 train_time:88187ms step_avg:151.00ms
step:595/1480 train_time:88349ms step_avg:151.02ms
step:596/1480 train_time:88511ms step_avg:151.04ms
step:597/1480 train_time:88670ms step_avg:151.06ms
step:598/1480 train_time:88828ms step_avg:151.07ms
step:599/1480 train_time:88987ms step_avg:151.08ms
step:600/1480 train_time:89147ms step_avg:151.10ms
step:601/1480 train_time:89306ms step_avg:151.11ms
step:602/1480 train_time:89465ms step_avg:151.12ms
step:603/1480 train_time:89626ms step_avg:151.14ms
step:604/1480 train_time:89786ms step_avg:151.15ms
step:605/1480 train_time:89946ms step_avg:151.17ms
step:606/1480 train_time:90108ms step_avg:151.19ms
step:607/1480 train_time:90269ms step_avg:151.20ms
step:608/1480 train_time:90428ms step_avg:151.22ms
step:609/1480 train_time:90587ms step_avg:151.23ms
step:610/1480 train_time:90747ms step_avg:151.24ms
step:611/1480 train_time:90906ms step_avg:151.26ms
step:612/1480 train_time:91067ms step_avg:151.27ms
step:613/1480 train_time:91228ms step_avg:151.29ms
step:614/1480 train_time:91388ms step_avg:151.30ms
step:615/1480 train_time:91547ms step_avg:151.32ms
step:616/1480 train_time:91706ms step_avg:151.33ms
step:617/1480 train_time:91865ms step_avg:151.34ms
step:618/1480 train_time:92024ms step_avg:151.36ms
step:619/1480 train_time:92183ms step_avg:151.37ms
step:620/1480 train_time:92344ms step_avg:151.38ms
step:621/1480 train_time:92504ms step_avg:151.40ms
step:622/1480 train_time:92665ms step_avg:151.41ms
step:623/1480 train_time:92827ms step_avg:151.43ms
step:624/1480 train_time:92986ms step_avg:151.44ms
step:625/1480 train_time:93146ms step_avg:151.46ms
step:625/1480 val_loss:3.6042 train_time:93209ms step_avg:151.56ms
step:626/1480 train_time:93309ms step_avg:151.48ms
step:627/1480 train_time:93470ms step_avg:151.49ms
step:628/1480 train_time:93628ms step_avg:151.50ms
step:629/1480 train_time:93788ms step_avg:151.51ms
step:630/1480 train_time:93946ms step_avg:151.53ms
step:631/1480 train_time:94105ms step_avg:151.54ms
step:632/1480 train_time:94264ms step_avg:151.55ms
step:633/1480 train_time:94424ms step_avg:151.56ms
step:634/1480 train_time:94585ms step_avg:151.58ms
step:635/1480 train_time:94745ms step_avg:151.59ms
step:636/1480 train_time:94904ms step_avg:151.60ms
step:637/1480 train_time:95064ms step_avg:151.62ms
step:638/1480 train_time:95222ms step_avg:151.63ms
step:639/1480 train_time:95381ms step_avg:151.64ms
step:640/1480 train_time:95539ms step_avg:151.65ms
step:641/1480 train_time:95699ms step_avg:151.66ms
step:642/1480 train_time:95857ms step_avg:151.67ms
step:643/1480 train_time:96015ms step_avg:151.68ms
step:644/1480 train_time:96173ms step_avg:151.69ms
step:645/1480 train_time:96333ms step_avg:151.71ms
step:646/1480 train_time:96493ms step_avg:151.72ms
step:647/1480 train_time:96651ms step_avg:151.73ms
step:648/1480 train_time:96812ms step_avg:151.74ms
step:649/1480 train_time:96972ms step_avg:151.76ms
step:650/1480 train_time:97131ms step_avg:151.77ms
step:651/1480 train_time:97292ms step_avg:151.78ms
step:652/1480 train_time:97451ms step_avg:151.79ms
step:653/1480 train_time:97611ms step_avg:151.81ms
step:654/1480 train_time:97770ms step_avg:151.82ms
step:655/1480 train_time:97930ms step_avg:151.83ms
step:656/1480 train_time:98090ms step_avg:151.84ms
step:657/1480 train_time:98251ms step_avg:151.86ms
step:658/1480 train_time:98411ms step_avg:151.87ms
step:659/1480 train_time:98573ms step_avg:151.88ms
step:660/1480 train_time:98734ms step_avg:151.90ms
step:661/1480 train_time:98897ms step_avg:151.92ms
step:662/1480 train_time:99056ms step_avg:151.93ms
step:663/1480 train_time:99216ms step_avg:151.94ms
step:664/1480 train_time:99378ms step_avg:151.95ms
step:665/1480 train_time:99539ms step_avg:151.97ms
step:666/1480 train_time:99699ms step_avg:151.98ms
step:667/1480 train_time:99860ms step_avg:151.99ms
step:668/1480 train_time:100024ms step_avg:152.01ms
step:669/1480 train_time:100188ms step_avg:152.03ms
step:670/1480 train_time:100348ms step_avg:152.04ms
step:671/1480 train_time:100511ms step_avg:152.06ms
step:672/1480 train_time:100672ms step_avg:152.07ms
step:673/1480 train_time:100834ms step_avg:152.09ms
step:674/1480 train_time:100996ms step_avg:152.10ms
step:675/1480 train_time:101158ms step_avg:152.12ms
step:676/1480 train_time:101320ms step_avg:152.13ms
step:677/1480 train_time:101481ms step_avg:152.15ms
step:678/1480 train_time:101643ms step_avg:152.16ms
step:679/1480 train_time:101806ms step_avg:152.18ms
step:680/1480 train_time:101971ms step_avg:152.19ms
step:681/1480 train_time:102131ms step_avg:152.21ms
step:682/1480 train_time:102293ms step_avg:152.22ms
step:683/1480 train_time:102455ms step_avg:152.24ms
step:684/1480 train_time:102616ms step_avg:152.25ms
step:685/1480 train_time:102778ms step_avg:152.26ms
step:686/1480 train_time:102938ms step_avg:152.28ms
step:687/1480 train_time:103099ms step_avg:152.29ms
step:688/1480 train_time:103261ms step_avg:152.30ms
step:689/1480 train_time:103424ms step_avg:152.32ms
step:690/1480 train_time:103588ms step_avg:152.34ms
step:691/1480 train_time:103750ms step_avg:152.35ms
step:692/1480 train_time:103913ms step_avg:152.36ms
step:693/1480 train_time:104075ms step_avg:152.38ms
step:694/1480 train_time:104236ms step_avg:152.39ms
step:695/1480 train_time:104396ms step_avg:152.40ms
step:696/1480 train_time:104557ms step_avg:152.42ms
step:697/1480 train_time:104720ms step_avg:152.43ms
step:698/1480 train_time:104881ms step_avg:152.44ms
step:699/1480 train_time:105045ms step_avg:152.46ms
step:700/1480 train_time:105208ms step_avg:152.48ms
step:701/1480 train_time:105370ms step_avg:152.49ms
step:702/1480 train_time:105530ms step_avg:152.50ms
step:703/1480 train_time:105691ms step_avg:152.51ms
step:704/1480 train_time:105852ms step_avg:152.53ms
step:705/1480 train_time:106015ms step_avg:152.54ms
step:706/1480 train_time:106178ms step_avg:152.56ms
step:707/1480 train_time:106340ms step_avg:152.57ms
step:708/1480 train_time:106502ms step_avg:152.58ms
step:709/1480 train_time:106665ms step_avg:152.60ms
step:710/1480 train_time:106826ms step_avg:152.61ms
step:711/1480 train_time:106988ms step_avg:152.62ms
step:712/1480 train_time:107154ms step_avg:152.64ms
step:713/1480 train_time:107317ms step_avg:152.66ms
step:714/1480 train_time:107478ms step_avg:152.67ms
step:715/1480 train_time:107637ms step_avg:152.68ms
step:716/1480 train_time:107797ms step_avg:152.69ms
step:717/1480 train_time:107958ms step_avg:152.70ms
step:718/1480 train_time:108118ms step_avg:152.71ms
step:719/1480 train_time:108278ms step_avg:152.72ms
step:720/1480 train_time:108440ms step_avg:152.73ms
step:721/1480 train_time:108602ms step_avg:152.75ms
step:722/1480 train_time:108767ms step_avg:152.76ms
step:723/1480 train_time:108930ms step_avg:152.78ms
step:724/1480 train_time:109093ms step_avg:152.79ms
step:725/1480 train_time:109254ms step_avg:152.80ms
step:726/1480 train_time:109418ms step_avg:152.82ms
step:727/1480 train_time:109582ms step_avg:152.83ms
step:728/1480 train_time:109742ms step_avg:152.84ms
step:729/1480 train_time:109904ms step_avg:152.86ms
step:730/1480 train_time:110068ms step_avg:152.87ms
step:731/1480 train_time:110230ms step_avg:152.89ms
step:732/1480 train_time:110392ms step_avg:152.90ms
step:733/1480 train_time:110552ms step_avg:152.91ms
step:734/1480 train_time:110714ms step_avg:152.92ms
step:735/1480 train_time:110875ms step_avg:152.93ms
step:736/1480 train_time:111036ms step_avg:152.94ms
step:737/1480 train_time:111196ms step_avg:152.95ms
step:738/1480 train_time:111357ms step_avg:152.96ms
step:739/1480 train_time:111517ms step_avg:152.97ms
step:740/1480 train_time:111682ms step_avg:152.99ms
step:741/1480 train_time:111846ms step_avg:153.00ms
step:742/1480 train_time:112010ms step_avg:153.02ms
step:743/1480 train_time:112173ms step_avg:153.03ms
step:744/1480 train_time:112336ms step_avg:153.05ms
step:745/1480 train_time:112500ms step_avg:153.06ms
step:746/1480 train_time:112659ms step_avg:153.07ms
step:747/1480 train_time:112822ms step_avg:153.08ms
step:748/1480 train_time:112991ms step_avg:153.10ms
step:749/1480 train_time:113154ms step_avg:153.12ms
step:750/1480 train_time:113313ms step_avg:153.13ms
step:750/1480 val_loss:3.5515 train_time:113377ms step_avg:153.21ms
step:751/1480 train_time:113477ms step_avg:153.14ms
step:752/1480 train_time:113638ms step_avg:153.15ms
step:753/1480 train_time:113798ms step_avg:153.16ms
step:754/1480 train_time:113960ms step_avg:153.17ms
step:755/1480 train_time:114121ms step_avg:153.18ms
step:756/1480 train_time:114283ms step_avg:153.19ms
step:757/1480 train_time:114452ms step_avg:153.22ms
step:758/1480 train_time:114611ms step_avg:153.22ms
step:759/1480 train_time:114773ms step_avg:153.24ms
step:760/1480 train_time:114934ms step_avg:153.25ms
step:761/1480 train_time:115095ms step_avg:153.26ms
step:762/1480 train_time:115257ms step_avg:153.27ms
step:763/1480 train_time:115419ms step_avg:153.28ms
step:764/1480 train_time:115581ms step_avg:153.29ms
step:765/1480 train_time:115743ms step_avg:153.30ms
step:766/1480 train_time:115907ms step_avg:153.32ms
step:767/1480 train_time:116070ms step_avg:153.33ms
step:768/1480 train_time:116232ms step_avg:153.34ms
step:769/1480 train_time:116394ms step_avg:153.35ms
step:770/1480 train_time:116556ms step_avg:153.36ms
step:771/1480 train_time:116718ms step_avg:153.37ms
step:772/1480 train_time:116879ms step_avg:153.39ms
step:773/1480 train_time:117043ms step_avg:153.40ms
step:774/1480 train_time:117207ms step_avg:153.41ms
step:775/1480 train_time:117371ms step_avg:153.43ms
step:776/1480 train_time:117536ms step_avg:153.44ms
step:777/1480 train_time:117700ms step_avg:153.46ms
step:778/1480 train_time:117864ms step_avg:153.47ms
step:779/1480 train_time:118027ms step_avg:153.48ms
step:780/1480 train_time:118192ms step_avg:153.50ms
step:781/1480 train_time:118354ms step_avg:153.51ms
step:782/1480 train_time:118517ms step_avg:153.52ms
step:783/1480 train_time:118677ms step_avg:153.53ms
step:784/1480 train_time:118841ms step_avg:153.54ms
step:785/1480 train_time:119003ms step_avg:153.55ms
step:786/1480 train_time:119171ms step_avg:153.57ms
step:787/1480 train_time:119334ms step_avg:153.58ms
step:788/1480 train_time:119497ms step_avg:153.59ms
step:789/1480 train_time:119658ms step_avg:153.60ms
step:790/1480 train_time:119824ms step_avg:153.62ms
step:791/1480 train_time:119992ms step_avg:153.64ms
step:792/1480 train_time:120156ms step_avg:153.65ms
step:793/1480 train_time:120317ms step_avg:153.66ms
step:794/1480 train_time:120480ms step_avg:153.67ms
step:795/1480 train_time:120648ms step_avg:153.69ms
step:796/1480 train_time:120814ms step_avg:153.71ms
step:797/1480 train_time:120977ms step_avg:153.72ms
step:798/1480 train_time:121140ms step_avg:153.73ms
step:799/1480 train_time:121307ms step_avg:153.75ms
step:800/1480 train_time:121471ms step_avg:153.76ms
step:801/1480 train_time:121633ms step_avg:153.77ms
step:802/1480 train_time:121800ms step_avg:153.79ms
step:803/1480 train_time:121962ms step_avg:153.80ms
step:804/1480 train_time:122124ms step_avg:153.81ms
step:805/1480 train_time:122290ms step_avg:153.82ms
step:806/1480 train_time:122452ms step_avg:153.83ms
step:807/1480 train_time:122613ms step_avg:153.84ms
step:808/1480 train_time:122776ms step_avg:153.85ms
step:809/1480 train_time:122938ms step_avg:153.87ms
step:810/1480 train_time:123099ms step_avg:153.87ms
step:811/1480 train_time:123263ms step_avg:153.89ms
step:812/1480 train_time:123428ms step_avg:153.90ms
step:813/1480 train_time:123591ms step_avg:153.91ms
step:814/1480 train_time:123753ms step_avg:153.92ms
step:815/1480 train_time:123914ms step_avg:153.93ms
step:816/1480 train_time:124079ms step_avg:153.94ms
step:817/1480 train_time:124242ms step_avg:153.96ms
step:818/1480 train_time:124404ms step_avg:153.97ms
step:819/1480 train_time:124569ms step_avg:153.98ms
step:820/1480 train_time:124733ms step_avg:153.99ms
step:821/1480 train_time:124894ms step_avg:154.00ms
step:822/1480 train_time:125059ms step_avg:154.01ms
step:823/1480 train_time:125221ms step_avg:154.02ms
step:824/1480 train_time:125383ms step_avg:154.03ms
step:825/1480 train_time:125550ms step_avg:154.05ms
step:826/1480 train_time:125715ms step_avg:154.06ms
step:827/1480 train_time:125879ms step_avg:154.08ms
step:828/1480 train_time:126044ms step_avg:154.09ms
step:829/1480 train_time:126209ms step_avg:154.10ms
step:830/1480 train_time:126375ms step_avg:154.12ms
step:831/1480 train_time:126537ms step_avg:154.13ms
step:832/1480 train_time:126701ms step_avg:154.14ms
step:833/1480 train_time:126866ms step_avg:154.15ms
step:834/1480 train_time:127032ms step_avg:154.17ms
step:835/1480 train_time:127195ms step_avg:154.18ms
step:836/1480 train_time:127361ms step_avg:154.19ms
step:837/1480 train_time:127523ms step_avg:154.20ms
step:838/1480 train_time:127689ms step_avg:154.21ms
step:839/1480 train_time:127851ms step_avg:154.22ms
step:840/1480 train_time:128012ms step_avg:154.23ms
step:841/1480 train_time:128173ms step_avg:154.24ms
step:842/1480 train_time:128336ms step_avg:154.25ms
step:843/1480 train_time:128497ms step_avg:154.26ms
step:844/1480 train_time:128659ms step_avg:154.27ms
step:845/1480 train_time:128822ms step_avg:154.28ms
step:846/1480 train_time:128989ms step_avg:154.29ms
step:847/1480 train_time:129153ms step_avg:154.30ms
step:848/1480 train_time:129314ms step_avg:154.31ms
step:849/1480 train_time:129477ms step_avg:154.32ms
step:850/1480 train_time:129640ms step_avg:154.33ms
step:851/1480 train_time:129804ms step_avg:154.35ms
step:852/1480 train_time:129968ms step_avg:154.36ms
step:853/1480 train_time:130131ms step_avg:154.37ms
step:854/1480 train_time:130295ms step_avg:154.38ms
step:855/1480 train_time:130458ms step_avg:154.39ms
step:856/1480 train_time:130620ms step_avg:154.40ms
step:857/1480 train_time:130785ms step_avg:154.41ms
step:858/1480 train_time:130952ms step_avg:154.42ms
step:859/1480 train_time:131115ms step_avg:154.44ms
step:860/1480 train_time:131276ms step_avg:154.44ms
step:861/1480 train_time:131442ms step_avg:154.46ms
step:862/1480 train_time:131612ms step_avg:154.47ms
step:863/1480 train_time:131778ms step_avg:154.49ms
step:864/1480 train_time:131942ms step_avg:154.50ms
step:865/1480 train_time:132104ms step_avg:154.51ms
step:866/1480 train_time:132273ms step_avg:154.53ms
step:867/1480 train_time:132436ms step_avg:154.53ms
step:868/1480 train_time:132597ms step_avg:154.54ms
step:869/1480 train_time:132759ms step_avg:154.55ms
step:870/1480 train_time:132926ms step_avg:154.57ms
step:871/1480 train_time:133090ms step_avg:154.58ms
step:872/1480 train_time:133254ms step_avg:154.59ms
step:873/1480 train_time:133415ms step_avg:154.59ms
step:874/1480 train_time:133582ms step_avg:154.61ms
step:875/1480 train_time:133746ms step_avg:154.62ms
step:875/1480 val_loss:3.5062 train_time:133812ms step_avg:154.70ms
step:876/1480 train_time:133912ms step_avg:154.63ms
step:877/1480 train_time:134079ms step_avg:154.65ms
step:878/1480 train_time:134241ms step_avg:154.66ms
step:879/1480 train_time:134404ms step_avg:154.67ms
step:880/1480 train_time:134567ms step_avg:154.67ms
step:881/1480 train_time:134729ms step_avg:154.68ms
step:882/1480 train_time:134895ms step_avg:154.70ms
step:883/1480 train_time:135060ms step_avg:154.71ms
step:884/1480 train_time:135227ms step_avg:154.72ms
step:885/1480 train_time:135395ms step_avg:154.74ms
step:886/1480 train_time:135559ms step_avg:154.75ms
step:887/1480 train_time:135726ms step_avg:154.76ms
step:888/1480 train_time:135900ms step_avg:154.78ms
step:889/1480 train_time:136068ms step_avg:154.80ms
step:890/1480 train_time:136231ms step_avg:154.81ms
step:891/1480 train_time:136399ms step_avg:154.82ms
step:892/1480 train_time:136563ms step_avg:154.83ms
step:893/1480 train_time:136724ms step_avg:154.84ms
step:894/1480 train_time:136893ms step_avg:154.86ms
step:895/1480 train_time:137057ms step_avg:154.87ms
step:896/1480 train_time:137223ms step_avg:154.88ms
step:897/1480 train_time:137390ms step_avg:154.89ms
step:898/1480 train_time:137557ms step_avg:154.91ms
step:899/1480 train_time:137720ms step_avg:154.92ms
step:900/1480 train_time:137883ms step_avg:154.92ms
step:901/1480 train_time:138047ms step_avg:154.93ms
step:902/1480 train_time:138213ms step_avg:154.95ms
step:903/1480 train_time:138384ms step_avg:154.97ms
step:904/1480 train_time:138550ms step_avg:154.98ms
step:905/1480 train_time:138713ms step_avg:154.99ms
step:906/1480 train_time:138881ms step_avg:155.00ms
step:907/1480 train_time:139048ms step_avg:155.01ms
step:908/1480 train_time:139214ms step_avg:155.03ms
step:909/1480 train_time:139379ms step_avg:155.04ms
step:910/1480 train_time:139550ms step_avg:155.06ms
step:911/1480 train_time:139716ms step_avg:155.07ms
step:912/1480 train_time:139881ms step_avg:155.08ms
step:913/1480 train_time:140047ms step_avg:155.09ms
step:914/1480 train_time:140215ms step_avg:155.10ms
step:915/1480 train_time:140383ms step_avg:155.12ms
step:916/1480 train_time:140547ms step_avg:155.13ms
step:917/1480 train_time:140713ms step_avg:155.14ms
step:918/1480 train_time:140882ms step_avg:155.16ms
step:919/1480 train_time:141050ms step_avg:155.17ms
step:920/1480 train_time:141217ms step_avg:155.18ms
step:921/1480 train_time:141381ms step_avg:155.19ms
step:922/1480 train_time:141548ms step_avg:155.21ms
step:923/1480 train_time:141712ms step_avg:155.22ms
step:924/1480 train_time:141877ms step_avg:155.23ms
step:925/1480 train_time:142042ms step_avg:155.24ms
step:926/1480 train_time:142205ms step_avg:155.25ms
step:927/1480 train_time:142371ms step_avg:155.26ms
step:928/1480 train_time:142536ms step_avg:155.27ms
step:929/1480 train_time:142701ms step_avg:155.28ms
step:930/1480 train_time:142866ms step_avg:155.29ms
step:931/1480 train_time:143029ms step_avg:155.30ms
step:932/1480 train_time:143195ms step_avg:155.31ms
step:933/1480 train_time:143362ms step_avg:155.32ms
step:934/1480 train_time:143530ms step_avg:155.34ms
step:935/1480 train_time:143704ms step_avg:155.36ms
step:936/1480 train_time:143871ms step_avg:155.37ms
step:937/1480 train_time:144041ms step_avg:155.38ms
step:938/1480 train_time:144204ms step_avg:155.39ms
step:939/1480 train_time:144373ms step_avg:155.41ms
step:940/1480 train_time:144540ms step_avg:155.42ms
step:941/1480 train_time:144704ms step_avg:155.43ms
step:942/1480 train_time:144869ms step_avg:155.44ms
step:943/1480 train_time:145038ms step_avg:155.45ms
step:944/1480 train_time:145210ms step_avg:155.47ms
step:945/1480 train_time:145375ms step_avg:155.48ms
step:946/1480 train_time:145543ms step_avg:155.49ms
step:947/1480 train_time:145711ms step_avg:155.51ms
step:948/1480 train_time:145877ms step_avg:155.52ms
step:949/1480 train_time:146042ms step_avg:155.53ms
step:950/1480 train_time:146205ms step_avg:155.54ms
step:951/1480 train_time:146376ms step_avg:155.55ms
step:952/1480 train_time:146541ms step_avg:155.56ms
step:953/1480 train_time:146708ms step_avg:155.58ms
step:954/1480 train_time:146878ms step_avg:155.59ms
step:955/1480 train_time:147041ms step_avg:155.60ms
step:956/1480 train_time:147205ms step_avg:155.61ms
step:957/1480 train_time:147374ms step_avg:155.62ms
step:958/1480 train_time:147543ms step_avg:155.64ms
step:959/1480 train_time:147707ms step_avg:155.65ms
step:960/1480 train_time:147876ms step_avg:155.66ms
step:961/1480 train_time:148041ms step_avg:155.67ms
step:962/1480 train_time:148205ms step_avg:155.68ms
step:963/1480 train_time:148371ms step_avg:155.69ms
step:964/1480 train_time:148539ms step_avg:155.70ms
step:965/1480 train_time:148702ms step_avg:155.71ms
step:966/1480 train_time:148865ms step_avg:155.72ms
step:967/1480 train_time:149030ms step_avg:155.73ms
step:968/1480 train_time:149196ms step_avg:155.74ms
step:969/1480 train_time:149360ms step_avg:155.75ms
step:970/1480 train_time:149523ms step_avg:155.75ms
step:971/1480 train_time:149689ms step_avg:155.76ms
step:972/1480 train_time:149854ms step_avg:155.77ms
step:973/1480 train_time:150019ms step_avg:155.78ms
step:974/1480 train_time:150187ms step_avg:155.80ms
step:975/1480 train_time:150352ms step_avg:155.80ms
step:976/1480 train_time:150517ms step_avg:155.81ms
step:977/1480 train_time:150681ms step_avg:155.82ms
step:978/1480 train_time:150847ms step_avg:155.83ms
step:979/1480 train_time:151012ms step_avg:155.84ms
step:980/1480 train_time:151178ms step_avg:155.85ms
step:981/1480 train_time:151345ms step_avg:155.86ms
step:982/1480 train_time:151509ms step_avg:155.87ms
step:983/1480 train_time:151676ms step_avg:155.88ms
step:984/1480 train_time:151840ms step_avg:155.89ms
step:985/1480 train_time:152009ms step_avg:155.91ms
step:986/1480 train_time:152174ms step_avg:155.92ms
step:987/1480 train_time:152339ms step_avg:155.93ms
step:988/1480 train_time:152505ms step_avg:155.94ms
step:989/1480 train_time:152671ms step_avg:155.95ms
step:990/1480 train_time:152841ms step_avg:155.96ms
step:991/1480 train_time:153008ms step_avg:155.97ms
step:992/1480 train_time:153183ms step_avg:155.99ms
step:993/1480 train_time:153359ms step_avg:156.01ms
step:994/1480 train_time:153524ms step_avg:156.02ms
step:995/1480 train_time:153689ms step_avg:156.03ms
step:996/1480 train_time:153851ms step_avg:156.04ms
step:997/1480 train_time:154018ms step_avg:156.05ms
step:998/1480 train_time:154181ms step_avg:156.05ms
step:999/1480 train_time:154346ms step_avg:156.06ms
step:1000/1480 train_time:154517ms step_avg:156.08ms
step:1000/1480 val_loss:3.4415 train_time:154584ms step_avg:156.15ms
step:1001/1480 train_time:154686ms step_avg:156.09ms
step:1002/1480 train_time:154850ms step_avg:156.10ms
step:1003/1480 train_time:155023ms step_avg:156.12ms
step:1004/1480 train_time:155190ms step_avg:156.13ms
step:1005/1480 train_time:155359ms step_avg:156.14ms
step:1006/1480 train_time:155527ms step_avg:156.15ms
step:1007/1480 train_time:155692ms step_avg:156.16ms
step:1008/1480 train_time:155859ms step_avg:156.17ms
step:1009/1480 train_time:156031ms step_avg:156.19ms
step:1010/1480 train_time:156197ms step_avg:156.20ms
step:1011/1480 train_time:156363ms step_avg:156.21ms
step:1012/1480 train_time:156527ms step_avg:156.22ms
step:1013/1480 train_time:156698ms step_avg:156.23ms
step:1014/1480 train_time:156865ms step_avg:156.24ms
step:1015/1480 train_time:157034ms step_avg:156.25ms
step:1016/1480 train_time:157204ms step_avg:156.27ms
step:1017/1480 train_time:157375ms step_avg:156.28ms
step:1018/1480 train_time:157544ms step_avg:156.29ms
step:1019/1480 train_time:157712ms step_avg:156.31ms
step:1020/1480 train_time:157883ms step_avg:156.32ms
step:1021/1480 train_time:158048ms step_avg:156.33ms
step:1022/1480 train_time:158213ms step_avg:156.34ms
step:1023/1480 train_time:158382ms step_avg:156.35ms
step:1024/1480 train_time:158547ms step_avg:156.36ms
step:1025/1480 train_time:158717ms step_avg:156.37ms
step:1026/1480 train_time:158883ms step_avg:156.38ms
step:1027/1480 train_time:159049ms step_avg:156.39ms
step:1028/1480 train_time:159222ms step_avg:156.41ms
step:1029/1480 train_time:159396ms step_avg:156.42ms
step:1030/1480 train_time:159564ms step_avg:156.44ms
step:1031/1480 train_time:159728ms step_avg:156.44ms
step:1032/1480 train_time:159902ms step_avg:156.46ms
step:1033/1480 train_time:160068ms step_avg:156.47ms
step:1034/1480 train_time:160237ms step_avg:156.48ms
step:1035/1480 train_time:160406ms step_avg:156.49ms
step:1036/1480 train_time:160570ms step_avg:156.50ms
step:1037/1480 train_time:160739ms step_avg:156.51ms
step:1038/1480 train_time:160908ms step_avg:156.52ms
step:1039/1480 train_time:161079ms step_avg:156.54ms
step:1040/1480 train_time:161246ms step_avg:156.55ms
step:1041/1480 train_time:161414ms step_avg:156.56ms
step:1042/1480 train_time:161578ms step_avg:156.57ms
step:1043/1480 train_time:161745ms step_avg:156.58ms
step:1044/1480 train_time:161908ms step_avg:156.58ms
step:1045/1480 train_time:162079ms step_avg:156.60ms
step:1046/1480 train_time:162248ms step_avg:156.61ms
step:1047/1480 train_time:162412ms step_avg:156.62ms
step:1048/1480 train_time:162580ms step_avg:156.63ms
step:1049/1480 train_time:162747ms step_avg:156.64ms
step:1050/1480 train_time:162915ms step_avg:156.65ms
step:1051/1480 train_time:163086ms step_avg:156.66ms
step:1052/1480 train_time:163253ms step_avg:156.67ms
step:1053/1480 train_time:163422ms step_avg:156.68ms
step:1054/1480 train_time:163589ms step_avg:156.69ms
step:1055/1480 train_time:163753ms step_avg:156.70ms
step:1056/1480 train_time:163918ms step_avg:156.71ms
step:1057/1480 train_time:164085ms step_avg:156.72ms
step:1058/1480 train_time:164253ms step_avg:156.73ms
step:1059/1480 train_time:164428ms step_avg:156.75ms
step:1060/1480 train_time:164598ms step_avg:156.76ms
step:1061/1480 train_time:164762ms step_avg:156.77ms
step:1062/1480 train_time:164928ms step_avg:156.78ms
step:1063/1480 train_time:165092ms step_avg:156.78ms
step:1064/1480 train_time:165255ms step_avg:156.79ms
step:1065/1480 train_time:165424ms step_avg:156.80ms
step:1066/1480 train_time:165591ms step_avg:156.81ms
step:1067/1480 train_time:165761ms step_avg:156.82ms
step:1068/1480 train_time:165927ms step_avg:156.83ms
step:1069/1480 train_time:166100ms step_avg:156.85ms
step:1070/1480 train_time:166266ms step_avg:156.86ms
step:1071/1480 train_time:166439ms step_avg:156.87ms
step:1072/1480 train_time:166604ms step_avg:156.88ms
step:1073/1480 train_time:166766ms step_avg:156.88ms
step:1074/1480 train_time:166933ms step_avg:156.89ms
step:1075/1480 train_time:167103ms step_avg:156.90ms
step:1076/1480 train_time:167270ms step_avg:156.91ms
step:1077/1480 train_time:167435ms step_avg:156.92ms
step:1078/1480 train_time:167609ms step_avg:156.94ms
step:1079/1480 train_time:167782ms step_avg:156.95ms
step:1080/1480 train_time:167951ms step_avg:156.96ms
step:1081/1480 train_time:168116ms step_avg:156.97ms
step:1082/1480 train_time:168282ms step_avg:156.98ms
step:1083/1480 train_time:168449ms step_avg:156.99ms
step:1084/1480 train_time:168616ms step_avg:157.00ms
step:1085/1480 train_time:168786ms step_avg:157.01ms
step:1086/1480 train_time:168954ms step_avg:157.02ms
step:1087/1480 train_time:169121ms step_avg:157.03ms
step:1088/1480 train_time:169290ms step_avg:157.04ms
step:1089/1480 train_time:169464ms step_avg:157.06ms
step:1090/1480 train_time:169634ms step_avg:157.07ms
step:1091/1480 train_time:169804ms step_avg:157.08ms
step:1092/1480 train_time:169971ms step_avg:157.09ms
step:1093/1480 train_time:170140ms step_avg:157.10ms
step:1094/1480 train_time:170307ms step_avg:157.11ms
step:1095/1480 train_time:170471ms step_avg:157.12ms
step:1096/1480 train_time:170643ms step_avg:157.13ms
step:1097/1480 train_time:170810ms step_avg:157.14ms
step:1098/1480 train_time:170981ms step_avg:157.15ms
step:1099/1480 train_time:171154ms step_avg:157.17ms
step:1100/1480 train_time:171327ms step_avg:157.18ms
step:1101/1480 train_time:171498ms step_avg:157.19ms
step:1102/1480 train_time:171670ms step_avg:157.21ms
step:1103/1480 train_time:171848ms step_avg:157.23ms
step:1104/1480 train_time:172015ms step_avg:157.24ms
step:1105/1480 train_time:172186ms step_avg:157.25ms
step:1106/1480 train_time:172353ms step_avg:157.26ms
step:1107/1480 train_time:172524ms step_avg:157.27ms
step:1108/1480 train_time:172689ms step_avg:157.28ms
step:1109/1480 train_time:172853ms step_avg:157.28ms
step:1110/1480 train_time:173020ms step_avg:157.29ms
step:1111/1480 train_time:173186ms step_avg:157.30ms
step:1112/1480 train_time:173356ms step_avg:157.31ms
step:1113/1480 train_time:173537ms step_avg:157.33ms
step:1114/1480 train_time:173711ms step_avg:157.35ms
step:1115/1480 train_time:173883ms step_avg:157.36ms
step:1116/1480 train_time:174051ms step_avg:157.37ms
step:1117/1480 train_time:174225ms step_avg:157.38ms
step:1118/1480 train_time:174400ms step_avg:157.40ms
step:1119/1480 train_time:174566ms step_avg:157.41ms
step:1120/1480 train_time:174734ms step_avg:157.42ms
step:1121/1480 train_time:174906ms step_avg:157.43ms
step:1122/1480 train_time:175074ms step_avg:157.44ms
step:1123/1480 train_time:175240ms step_avg:157.45ms
step:1124/1480 train_time:175407ms step_avg:157.46ms
step:1125/1480 train_time:175574ms step_avg:157.47ms
step:1125/1480 val_loss:3.3862 train_time:175642ms step_avg:157.53ms
step:1126/1480 train_time:175745ms step_avg:157.48ms
step:1127/1480 train_time:175912ms step_avg:157.49ms
step:1128/1480 train_time:176083ms step_avg:157.50ms
step:1129/1480 train_time:176256ms step_avg:157.51ms
step:1130/1480 train_time:176426ms step_avg:157.52ms
step:1131/1480 train_time:176603ms step_avg:157.54ms
step:1132/1480 train_time:176769ms step_avg:157.55ms
step:1133/1480 train_time:176942ms step_avg:157.56ms
step:1134/1480 train_time:177113ms step_avg:157.57ms
step:1135/1480 train_time:177282ms step_avg:157.58ms
step:1136/1480 train_time:177452ms step_avg:157.60ms
step:1137/1480 train_time:177623ms step_avg:157.61ms
step:1138/1480 train_time:177796ms step_avg:157.62ms
step:1139/1480 train_time:177966ms step_avg:157.63ms
step:1140/1480 train_time:178132ms step_avg:157.64ms
step:1141/1480 train_time:178306ms step_avg:157.65ms
step:1142/1480 train_time:178473ms step_avg:157.66ms
step:1143/1480 train_time:178645ms step_avg:157.67ms
step:1144/1480 train_time:178812ms step_avg:157.68ms
step:1145/1480 train_time:178977ms step_avg:157.69ms
step:1146/1480 train_time:179148ms step_avg:157.70ms
step:1147/1480 train_time:179315ms step_avg:157.71ms
step:1148/1480 train_time:179485ms step_avg:157.72ms
step:1149/1480 train_time:179655ms step_avg:157.73ms
step:1150/1480 train_time:179823ms step_avg:157.74ms
step:1151/1480 train_time:179995ms step_avg:157.75ms
step:1152/1480 train_time:180169ms step_avg:157.77ms
step:1153/1480 train_time:180342ms step_avg:157.78ms
step:1154/1480 train_time:180508ms step_avg:157.79ms
step:1155/1480 train_time:180681ms step_avg:157.80ms
step:1156/1480 train_time:180864ms step_avg:157.82ms
step:1157/1480 train_time:181032ms step_avg:157.83ms
step:1158/1480 train_time:181199ms step_avg:157.84ms
step:1159/1480 train_time:181367ms step_avg:157.85ms
step:1160/1480 train_time:181532ms step_avg:157.85ms
step:1161/1480 train_time:181702ms step_avg:157.86ms
step:1162/1480 train_time:181871ms step_avg:157.87ms
step:1163/1480 train_time:182041ms step_avg:157.88ms
step:1164/1480 train_time:182209ms step_avg:157.89ms
step:1165/1480 train_time:182374ms step_avg:157.90ms
step:1166/1480 train_time:182546ms step_avg:157.91ms
step:1167/1480 train_time:182713ms step_avg:157.92ms
step:1168/1480 train_time:182883ms step_avg:157.93ms
step:1169/1480 train_time:183052ms step_avg:157.94ms
step:1170/1480 train_time:183219ms step_avg:157.95ms
step:1171/1480 train_time:183386ms step_avg:157.96ms
step:1172/1480 train_time:183553ms step_avg:157.96ms
step:1173/1480 train_time:183725ms step_avg:157.97ms
step:1174/1480 train_time:183907ms step_avg:158.00ms
step:1175/1480 train_time:184077ms step_avg:158.01ms
step:1176/1480 train_time:184249ms step_avg:158.02ms
step:1177/1480 train_time:184426ms step_avg:158.03ms
step:1178/1480 train_time:184593ms step_avg:158.04ms
step:1179/1480 train_time:184761ms step_avg:158.05ms
step:1180/1480 train_time:184941ms step_avg:158.07ms
step:1181/1480 train_time:185111ms step_avg:158.08ms
step:1182/1480 train_time:185279ms step_avg:158.09ms
step:1183/1480 train_time:185451ms step_avg:158.10ms
step:1184/1480 train_time:185619ms step_avg:158.11ms
step:1185/1480 train_time:185791ms step_avg:158.12ms
step:1186/1480 train_time:185961ms step_avg:158.13ms
step:1187/1480 train_time:186145ms step_avg:158.15ms
step:1188/1480 train_time:186312ms step_avg:158.16ms
step:1189/1480 train_time:186484ms step_avg:158.17ms
step:1190/1480 train_time:186651ms step_avg:158.18ms
step:1191/1480 train_time:186823ms step_avg:158.19ms
step:1192/1480 train_time:186989ms step_avg:158.20ms
step:1193/1480 train_time:187157ms step_avg:158.21ms
step:1194/1480 train_time:187326ms step_avg:158.21ms
step:1195/1480 train_time:187499ms step_avg:158.23ms
step:1196/1480 train_time:187683ms step_avg:158.25ms
step:1197/1480 train_time:187853ms step_avg:158.26ms
step:1198/1480 train_time:188034ms step_avg:158.28ms
step:1199/1480 train_time:188205ms step_avg:158.29ms
step:1200/1480 train_time:188373ms step_avg:158.30ms
step:1201/1480 train_time:188542ms step_avg:158.31ms
step:1202/1480 train_time:188723ms step_avg:158.32ms
step:1203/1480 train_time:188899ms step_avg:158.34ms
step:1204/1480 train_time:189074ms step_avg:158.35ms
step:1205/1480 train_time:189241ms step_avg:158.36ms
step:1206/1480 train_time:189407ms step_avg:158.37ms
step:1207/1480 train_time:189577ms step_avg:158.38ms
step:1208/1480 train_time:189746ms step_avg:158.39ms
step:1209/1480 train_time:189919ms step_avg:158.40ms
step:1210/1480 train_time:190092ms step_avg:158.41ms
step:1211/1480 train_time:190267ms step_avg:158.42ms
step:1212/1480 train_time:190436ms step_avg:158.43ms
step:1213/1480 train_time:190609ms step_avg:158.44ms
step:1214/1480 train_time:190786ms step_avg:158.46ms
step:1215/1480 train_time:190960ms step_avg:158.47ms
step:1216/1480 train_time:191129ms step_avg:158.48ms
step:1217/1480 train_time:191304ms step_avg:158.50ms
step:1218/1480 train_time:191474ms step_avg:158.50ms
step:1219/1480 train_time:191653ms step_avg:158.52ms
step:1220/1480 train_time:191825ms step_avg:158.53ms
step:1221/1480 train_time:191994ms step_avg:158.54ms
step:1222/1480 train_time:192163ms step_avg:158.55ms
step:1223/1480 train_time:192332ms step_avg:158.56ms
step:1224/1480 train_time:192509ms step_avg:158.57ms
step:1225/1480 train_time:192681ms step_avg:158.59ms
step:1226/1480 train_time:192854ms step_avg:158.60ms
step:1227/1480 train_time:193028ms step_avg:158.61ms
step:1228/1480 train_time:193198ms step_avg:158.62ms
step:1229/1480 train_time:193371ms step_avg:158.63ms
step:1230/1480 train_time:193550ms step_avg:158.65ms
step:1231/1480 train_time:193726ms step_avg:158.66ms
step:1232/1480 train_time:193899ms step_avg:158.67ms
step:1233/1480 train_time:194069ms step_avg:158.68ms
step:1234/1480 train_time:194237ms step_avg:158.69ms
step:1235/1480 train_time:194412ms step_avg:158.70ms
step:1236/1480 train_time:194580ms step_avg:158.71ms
step:1237/1480 train_time:194751ms step_avg:158.72ms
step:1238/1480 train_time:194935ms step_avg:158.74ms
step:1239/1480 train_time:195106ms step_avg:158.75ms
step:1240/1480 train_time:195275ms step_avg:158.76ms
step:1241/1480 train_time:195449ms step_avg:158.77ms
step:1242/1480 train_time:195617ms step_avg:158.78ms
step:1243/1480 train_time:195791ms step_avg:158.79ms
step:1244/1480 train_time:195958ms step_avg:158.80ms
step:1245/1480 train_time:196126ms step_avg:158.81ms
step:1246/1480 train_time:196295ms step_avg:158.81ms
step:1247/1480 train_time:196466ms step_avg:158.82ms
step:1248/1480 train_time:196635ms step_avg:158.83ms
step:1249/1480 train_time:196804ms step_avg:158.84ms
step:1250/1480 train_time:196973ms step_avg:158.85ms
step:1250/1480 val_loss:3.3364 train_time:197045ms step_avg:158.91ms
step:1251/1480 train_time:197156ms step_avg:158.87ms
step:1252/1480 train_time:197324ms step_avg:158.88ms
step:1253/1480 train_time:197493ms step_avg:158.88ms
step:1254/1480 train_time:197665ms step_avg:158.89ms
step:1255/1480 train_time:197853ms step_avg:158.92ms
step:1256/1480 train_time:198025ms step_avg:158.93ms
step:1257/1480 train_time:198196ms step_avg:158.94ms
step:1258/1480 train_time:198369ms step_avg:158.95ms
step:1259/1480 train_time:198540ms step_avg:158.96ms
step:1260/1480 train_time:198709ms step_avg:158.97ms
step:1261/1480 train_time:198881ms step_avg:158.98ms
step:1262/1480 train_time:199058ms step_avg:158.99ms
step:1263/1480 train_time:199231ms step_avg:159.00ms
step:1264/1480 train_time:199397ms step_avg:159.01ms
step:1265/1480 train_time:199564ms step_avg:159.01ms
step:1266/1480 train_time:199735ms step_avg:159.02ms
step:1267/1480 train_time:199905ms step_avg:159.03ms
step:1268/1480 train_time:200077ms step_avg:159.04ms
step:1269/1480 train_time:200252ms step_avg:159.06ms
step:1270/1480 train_time:200421ms step_avg:159.06ms
step:1271/1480 train_time:200592ms step_avg:159.07ms
step:1272/1480 train_time:200758ms step_avg:159.08ms
step:1273/1480 train_time:200929ms step_avg:159.09ms
step:1274/1480 train_time:201101ms step_avg:159.10ms
step:1275/1480 train_time:201267ms step_avg:159.10ms
step:1276/1480 train_time:201435ms step_avg:159.11ms
step:1277/1480 train_time:201606ms step_avg:159.12ms
step:1278/1480 train_time:201775ms step_avg:159.13ms
step:1279/1480 train_time:201945ms step_avg:159.14ms
step:1280/1480 train_time:202126ms step_avg:159.15ms
step:1281/1480 train_time:202297ms step_avg:159.16ms
step:1282/1480 train_time:202462ms step_avg:159.17ms
step:1283/1480 train_time:202634ms step_avg:159.18ms
step:1284/1480 train_time:202802ms step_avg:159.19ms
step:1285/1480 train_time:202971ms step_avg:159.19ms
step:1286/1480 train_time:203140ms step_avg:159.20ms
step:1287/1480 train_time:203313ms step_avg:159.21ms
step:1288/1480 train_time:203484ms step_avg:159.22ms
step:1289/1480 train_time:203665ms step_avg:159.24ms
step:1290/1480 train_time:203845ms step_avg:159.25ms
step:1291/1480 train_time:204019ms step_avg:159.27ms
step:1292/1480 train_time:204193ms step_avg:159.28ms
step:1293/1480 train_time:204368ms step_avg:159.29ms
step:1294/1480 train_time:204539ms step_avg:159.30ms
step:1295/1480 train_time:204709ms step_avg:159.31ms
step:1296/1480 train_time:204883ms step_avg:159.32ms
step:1297/1480 train_time:205056ms step_avg:159.33ms
step:1298/1480 train_time:205225ms step_avg:159.34ms
step:1299/1480 train_time:205396ms step_avg:159.35ms
step:1300/1480 train_time:205564ms step_avg:159.35ms
step:1301/1480 train_time:205732ms step_avg:159.36ms
step:1302/1480 train_time:205905ms step_avg:159.37ms
step:1303/1480 train_time:206083ms step_avg:159.38ms
step:1304/1480 train_time:206257ms step_avg:159.40ms
step:1305/1480 train_time:206427ms step_avg:159.40ms
step:1306/1480 train_time:206601ms step_avg:159.41ms
step:1307/1480 train_time:206768ms step_avg:159.42ms
step:1308/1480 train_time:206937ms step_avg:159.43ms
step:1309/1480 train_time:207110ms step_avg:159.44ms
step:1310/1480 train_time:207279ms step_avg:159.45ms
step:1311/1480 train_time:207448ms step_avg:159.45ms
step:1312/1480 train_time:207622ms step_avg:159.46ms
step:1313/1480 train_time:207792ms step_avg:159.47ms
step:1314/1480 train_time:207965ms step_avg:159.48ms
step:1315/1480 train_time:208135ms step_avg:159.49ms
step:1316/1480 train_time:208300ms step_avg:159.50ms
step:1317/1480 train_time:208472ms step_avg:159.50ms
step:1318/1480 train_time:208655ms step_avg:159.52ms
step:1319/1480 train_time:208832ms step_avg:159.54ms
step:1320/1480 train_time:209008ms step_avg:159.55ms
step:1321/1480 train_time:209180ms step_avg:159.56ms
step:1322/1480 train_time:209358ms step_avg:159.57ms
step:1323/1480 train_time:209532ms step_avg:159.58ms
step:1324/1480 train_time:209707ms step_avg:159.59ms
step:1325/1480 train_time:209889ms step_avg:159.61ms
step:1326/1480 train_time:210063ms step_avg:159.62ms
step:1327/1480 train_time:210234ms step_avg:159.63ms
step:1328/1480 train_time:210404ms step_avg:159.64ms
step:1329/1480 train_time:210601ms step_avg:159.67ms
step:1330/1480 train_time:210780ms step_avg:159.68ms
step:1331/1480 train_time:210951ms step_avg:159.69ms
step:1332/1480 train_time:211125ms step_avg:159.70ms
step:1333/1480 train_time:211301ms step_avg:159.71ms
step:1334/1480 train_time:211471ms step_avg:159.72ms
step:1335/1480 train_time:211640ms step_avg:159.73ms
step:1336/1480 train_time:211824ms step_avg:159.75ms
step:1337/1480 train_time:211999ms step_avg:159.76ms
step:1338/1480 train_time:212169ms step_avg:159.77ms
step:1339/1480 train_time:212342ms step_avg:159.78ms
step:1340/1480 train_time:212515ms step_avg:159.79ms
step:1341/1480 train_time:212682ms step_avg:159.79ms
step:1342/1480 train_time:212857ms step_avg:159.80ms
step:1343/1480 train_time:213027ms step_avg:159.81ms
step:1344/1480 train_time:213200ms step_avg:159.82ms
step:1345/1480 train_time:213377ms step_avg:159.83ms
step:1346/1480 train_time:213545ms step_avg:159.84ms
step:1347/1480 train_time:213715ms step_avg:159.85ms
step:1348/1480 train_time:213883ms step_avg:159.85ms
step:1349/1480 train_time:214054ms step_avg:159.86ms
step:1350/1480 train_time:214228ms step_avg:159.87ms
step:1351/1480 train_time:214404ms step_avg:159.88ms
step:1352/1480 train_time:214574ms step_avg:159.89ms
step:1353/1480 train_time:214750ms step_avg:159.90ms
step:1354/1480 train_time:214923ms step_avg:159.91ms
step:1355/1480 train_time:215092ms step_avg:159.92ms
step:1356/1480 train_time:215264ms step_avg:159.93ms
step:1357/1480 train_time:215437ms step_avg:159.94ms
step:1358/1480 train_time:215610ms step_avg:159.95ms
step:1359/1480 train_time:215782ms step_avg:159.96ms
step:1360/1480 train_time:215958ms step_avg:159.97ms
step:1361/1480 train_time:216136ms step_avg:159.98ms
step:1362/1480 train_time:216312ms step_avg:159.99ms
step:1363/1480 train_time:216493ms step_avg:160.01ms
step:1364/1480 train_time:216661ms step_avg:160.02ms
step:1365/1480 train_time:216827ms step_avg:160.02ms
step:1366/1480 train_time:217001ms step_avg:160.03ms
step:1367/1480 train_time:217173ms step_avg:160.04ms
step:1368/1480 train_time:217346ms step_avg:160.05ms
step:1369/1480 train_time:217528ms step_avg:160.06ms
step:1370/1480 train_time:217705ms step_avg:160.08ms
step:1371/1480 train_time:217879ms step_avg:160.09ms
step:1372/1480 train_time:218055ms step_avg:160.10ms
step:1373/1480 train_time:218223ms step_avg:160.11ms
step:1374/1480 train_time:218399ms step_avg:160.12ms
step:1375/1480 train_time:218568ms step_avg:160.12ms
step:1375/1480 val_loss:3.2979 train_time:218636ms step_avg:160.17ms
step:1376/1480 train_time:218743ms step_avg:160.13ms
step:1377/1480 train_time:218916ms step_avg:160.14ms
step:1378/1480 train_time:219084ms step_avg:160.15ms
step:1379/1480 train_time:219258ms step_avg:160.16ms
step:1380/1480 train_time:219432ms step_avg:160.17ms
step:1381/1480 train_time:219614ms step_avg:160.18ms
step:1382/1480 train_time:219785ms step_avg:160.19ms
step:1383/1480 train_time:219956ms step_avg:160.20ms
step:1384/1480 train_time:220133ms step_avg:160.21ms
step:1385/1480 train_time:220297ms step_avg:160.22ms
step:1386/1480 train_time:220469ms step_avg:160.22ms
step:1387/1480 train_time:220637ms step_avg:160.23ms
step:1388/1480 train_time:220808ms step_avg:160.24ms
step:1389/1480 train_time:220980ms step_avg:160.25ms
step:1390/1480 train_time:221149ms step_avg:160.25ms
step:1391/1480 train_time:221319ms step_avg:160.26ms
step:1392/1480 train_time:221491ms step_avg:160.27ms
step:1393/1480 train_time:221661ms step_avg:160.28ms
step:1394/1480 train_time:221833ms step_avg:160.28ms
step:1395/1480 train_time:222002ms step_avg:160.29ms
step:1396/1480 train_time:222171ms step_avg:160.30ms
step:1397/1480 train_time:222338ms step_avg:160.30ms
step:1398/1480 train_time:222506ms step_avg:160.31ms
step:1399/1480 train_time:222674ms step_avg:160.31ms
step:1400/1480 train_time:222851ms step_avg:160.32ms
step:1401/1480 train_time:223016ms step_avg:160.33ms
step:1402/1480 train_time:223189ms step_avg:160.34ms
step:1403/1480 train_time:223364ms step_avg:160.35ms
step:1404/1480 train_time:223535ms step_avg:160.35ms
step:1405/1480 train_time:223710ms step_avg:160.37ms
step:1406/1480 train_time:223884ms step_avg:160.38ms
step:1407/1480 train_time:224051ms step_avg:160.38ms
step:1408/1480 train_time:224219ms step_avg:160.39ms
step:1409/1480 train_time:224403ms step_avg:160.40ms
step:1410/1480 train_time:224572ms step_avg:160.41ms
step:1411/1480 train_time:224739ms step_avg:160.41ms
step:1412/1480 train_time:224910ms step_avg:160.42ms
step:1413/1480 train_time:225079ms step_avg:160.43ms
step:1414/1480 train_time:225251ms step_avg:160.44ms
step:1415/1480 train_time:225423ms step_avg:160.44ms
step:1416/1480 train_time:225610ms step_avg:160.46ms
step:1417/1480 train_time:225782ms step_avg:160.47ms
step:1418/1480 train_time:225952ms step_avg:160.48ms
step:1419/1480 train_time:226127ms step_avg:160.49ms
step:1420/1480 train_time:226301ms step_avg:160.50ms
step:1421/1480 train_time:226475ms step_avg:160.51ms
step:1422/1480 train_time:226647ms step_avg:160.51ms
step:1423/1480 train_time:226816ms step_avg:160.52ms
step:1424/1480 train_time:226995ms step_avg:160.53ms
step:1425/1480 train_time:227176ms step_avg:160.55ms
step:1426/1480 train_time:227347ms step_avg:160.56ms
step:1427/1480 train_time:227520ms step_avg:160.56ms
step:1428/1480 train_time:227692ms step_avg:160.57ms
step:1429/1480 train_time:227858ms step_avg:160.58ms
step:1430/1480 train_time:228033ms step_avg:160.59ms
step:1431/1480 train_time:228209ms step_avg:160.60ms
step:1432/1480 train_time:228386ms step_avg:160.61ms
step:1433/1480 train_time:228564ms step_avg:160.62ms
step:1434/1480 train_time:228747ms step_avg:160.64ms
step:1435/1480 train_time:228923ms step_avg:160.65ms
step:1436/1480 train_time:229098ms step_avg:160.66ms
step:1437/1480 train_time:229270ms step_avg:160.67ms
step:1438/1480 train_time:229438ms step_avg:160.67ms
step:1439/1480 train_time:229614ms step_avg:160.68ms
step:1440/1480 train_time:229783ms step_avg:160.69ms
step:1441/1480 train_time:229952ms step_avg:160.69ms
step:1442/1480 train_time:230131ms step_avg:160.71ms
step:1443/1480 train_time:230319ms step_avg:160.73ms
step:1444/1480 train_time:230490ms step_avg:160.73ms
step:1445/1480 train_time:230660ms step_avg:160.74ms
step:1446/1480 train_time:230836ms step_avg:160.75ms
step:1447/1480 train_time:231016ms step_avg:160.76ms
step:1448/1480 train_time:231188ms step_avg:160.77ms
step:1449/1480 train_time:231362ms step_avg:160.78ms
step:1450/1480 train_time:231535ms step_avg:160.79ms
step:1451/1480 train_time:231708ms step_avg:160.80ms
step:1452/1480 train_time:231879ms step_avg:160.80ms
step:1453/1480 train_time:232049ms step_avg:160.81ms
step:1454/1480 train_time:232219ms step_avg:160.82ms
step:1455/1480 train_time:232398ms step_avg:160.83ms
step:1456/1480 train_time:232572ms step_avg:160.84ms
step:1457/1480 train_time:232742ms step_avg:160.84ms
step:1458/1480 train_time:232913ms step_avg:160.85ms
step:1459/1480 train_time:233089ms step_avg:160.86ms
step:1460/1480 train_time:233259ms step_avg:160.87ms
step:1461/1480 train_time:233434ms step_avg:160.88ms
step:1462/1480 train_time:233605ms step_avg:160.89ms
step:1463/1480 train_time:233782ms step_avg:160.90ms
step:1464/1480 train_time:233957ms step_avg:160.91ms
step:1465/1480 train_time:234132ms step_avg:160.92ms
step:1466/1480 train_time:234301ms step_avg:160.92ms
step:1467/1480 train_time:234475ms step_avg:160.93ms
step:1468/1480 train_time:234645ms step_avg:160.94ms
step:1469/1480 train_time:234820ms step_avg:160.95ms
step:1470/1480 train_time:235001ms step_avg:160.96ms
step:1471/1480 train_time:235187ms step_avg:160.98ms
step:1472/1480 train_time:235367ms step_avg:160.99ms
step:1473/1480 train_time:235539ms step_avg:161.00ms
step:1474/1480 train_time:235716ms step_avg:161.01ms
step:1475/1480 train_time:235897ms step_avg:161.02ms
step:1476/1480 train_time:236070ms step_avg:161.03ms
step:1477/1480 train_time:236252ms step_avg:161.04ms
step:1478/1480 train_time:236435ms step_avg:161.06ms
step:1479/1480 train_time:236611ms step_avg:161.07ms
step:1480/1480 train_time:236784ms step_avg:161.08ms
step:1480/1480 val_loss:3.2789 train_time:236856ms step_avg:161.13ms