forked from KellerJordan/modded-nanogpt
-
Notifications
You must be signed in to change notification settings - Fork 0
/
dd7304a6-cc43-4d5e-adb8-c070111464a1.txt
3845 lines (3777 loc) · 252 KB
/
dd7304a6-cc43-4d5e-adb8-c070111464a1.txt
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
====================================================================================================
import os
import sys
with open(sys.argv[0]) as f:
code = f.read() # read the code of this file ASAP, for logging
import uuid
import glob
import time
from dataclasses import dataclass
import numpy as np
import torch
from torch import nn
import torch.nn.functional as F
import torch.distributed as dist
import torch._inductor.config as config
from torch.nn.parallel import DistributedDataParallel as DDP
# -----------------------------------------------------------------------------
# Muon optimizer
def zeropower_via_svd(G, steps=None):
U, S, V = G.svd()
return U @ V.T
@torch.compile
def zeropower_via_newtonschulz5(G, steps=10, eps=1e-7):
"""
Newton-Schulz iteration to compute the zeroth power / orthogonalization of G. We opt to use a
quintic iteration whose coefficients are selected to maximize the slope at zero. For the purpose
of minimizing steps, it turns out to be empirically effective to keep increasing the slope at
zero even beyond the point where the iteration no longer converges all the way to one everywhere
on the interval. This iteration therefore does not produce UV^T but rather something like US'V^T
where S' is diagonal with S_{ii}' \sim Uniform(0.5, 1.5), which turns out not to hurt model
performance at all relative to UV^T, where USV^T = G is the SVD.
"""
assert len(G.shape) == 2
a, b, c = (3.4445, -4.7750, 2.0315)
X = G.bfloat16()
X /= (X.norm() + eps) # ensure top singular value <= 1
if G.size(0) > G.size(1):
X = X.T
for _ in range(steps):
A = X @ X.T
B = A @ X
X = a * X + b * B + c * A @ B
if G.size(0) > G.size(1):
X = X.T
return X
zeropower_backends = dict(svd=zeropower_via_svd, newtonschulz5=zeropower_via_newtonschulz5)
class Muon(torch.optim.Optimizer):
"""
Muon - MomentUm Orthogonalized by Newton-schulz
Muon internally runs standard SGD-momentum, and then performs an orthogonalization post-
processing step, in which each 2D parameter's update is replaced with the nearest orthogonal
matrix. To efficiently orthogonalize each update, we use a Newton-Schulz iteration, which has
the advantage that it can be stably run in bfloat16 on the GPU.
Some warnings:
- This optimizer assumes that all parameters passed in are 2D.
- It should not be used for the embedding layer, the final fully connected layer, or any {0,1}-D
parameters; those should all be optimized by a standard method (e.g., AdamW).
- To use it with 4D convolutional filters, it works well to just flatten their last 3 dimensions.
- We believe it is unlikely to work well for training with small batch size.
- We believe it may not work well for finetuning pretrained models, but we haven't tested this.
- We have not yet tried this optimizer for training scenarios larger than NanoGPT (124M).
Arguments:
lr: The learning rate used by the internal SGD.
momentum: The momentum used by the internal SGD.
nesterov: Whether to use Nesterov-style momentum in the internal SGD. (recommended)
backend: The chosen backend for the orthogonalization step. (recommended: 'newtonschulz5')
backend_steps: The number of iteration steps to use in the backend, if it is iterative.
"""
def __init__(self, params, lr=0.02, momentum=0.95, nesterov=True,
backend='newtonschulz5', backend_steps=5):
defaults = dict(lr=lr, momentum=momentum, nesterov=nesterov, backend=backend, backend_steps=backend_steps)
super().__init__(params, defaults)
def step(self):
for group in self.param_groups:
lr = group['lr']
momentum = group['momentum']
zeropower_backend = zeropower_backends[group['backend']]
# generate weight updates in distributed fashion
total_params = sum(p.numel() for p in group['params'])
updates_flat = torch.zeros(total_params, device='cuda', dtype=torch.bfloat16)
curr_idx = 0
for i, p in enumerate(group['params']):
# luckily this will perfectly distribute a transformer with multiple of 4 layers to 8 GPUs
if i % int(os.environ['WORLD_SIZE']) == int(os.environ['RANK']):
g = p.grad
assert g is not None
state = self.state[p]
if 'momentum_buffer' not in state:
state['momentum_buffer'] = torch.zeros_like(g)
buf = state['momentum_buffer']
buf.mul_(momentum).add_(g)
if group['nesterov']:
g = g.add(buf, alpha=momentum)
g = zeropower_backend(g, steps=group['backend_steps'])
g *= max(1, g.size(0)/g.size(1))**0.5
updates_flat[curr_idx:curr_idx+p.numel()] = g.flatten()
curr_idx += p.numel()
# sync updates across devices. we are not memory-constrained so can do this simple deserialization
dist.all_reduce(updates_flat, op=dist.ReduceOp.SUM)
# deserialize and apply updates
curr_idx = 0
for p in group['params']:
g = updates_flat[curr_idx:curr_idx+p.numel()].view_as(p.data).type_as(p.data)
p.data.add_(g, alpha=-lr)
curr_idx += p.numel()
# -----------------------------------------------------------------------------
# PyTorch nn.Module definitions for the GPT-2 model
class Rotary(torch.nn.Module):
def __init__(self, dim, base=10000):
super().__init__()
self.inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2).float() / dim))
self.seq_len_cached = None
self.cos_cached = None
self.sin_cached = None
def forward(self, x):
seq_len = x.shape[1]
if seq_len != self.seq_len_cached:
self.seq_len_cached = seq_len
t = torch.arange(seq_len, device=x.device).type_as(self.inv_freq)
freqs = torch.outer(t, self.inv_freq).to(x.device)
self.cos_cached = freqs.cos().bfloat16()
self.sin_cached = freqs.sin().bfloat16()
return self.cos_cached[None, :, None, :], self.sin_cached[None, :, None, :]
def apply_rotary_emb(x, cos, sin):
assert x.ndim == 4 # multihead attention
d = x.shape[3]//2
x1 = x[..., :d]
x2 = x[..., d:]
y1 = x1 * cos + x2 * sin
y2 = x1 * (-sin) + x2 * cos
return torch.cat([y1, y2], 3).type_as(x)
class CausalSelfAttention(nn.Module):
def __init__(self, config):
super().__init__()
self.n_head = config.n_head
self.n_embd = config.n_embd
self.head_dim = self.n_embd // self.n_head
assert self.n_embd % self.n_head == 0
self.c_q = nn.Linear(self.n_embd, self.n_embd, bias=False)
self.c_k = nn.Linear(self.n_embd, self.n_embd, bias=False)
self.c_v = nn.Linear(self.n_embd, self.n_embd, bias=False)
# output projection
self.c_proj = nn.Linear(self.n_embd, self.n_embd, bias=False)
self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977
self.rotary = Rotary(self.head_dim)
self.lamb = nn.Parameter(torch.tensor(0.5)) # @Grad62304977
def forward(self, x, v1=None):
B, T, C = x.size() # batch size, sequence length, embedding dimensionality (n_embd)
q = self.c_q(x).view(B, T, self.n_head, self.head_dim)
k = self.c_k(x).view(B, T, self.n_head, self.head_dim)
v = self.c_v(x).view(B, T, self.n_head, self.head_dim)
if v1 is None:
v1 = v # This happens if we are in the first block. v needs to be accessed by subsequent blocks
v = (1 - self.lamb) * v + self.lamb * v1.view_as(v) # @Grad62304977
cos, sin = self.rotary(q)
q, k = F.rms_norm(q, (q.size(-1),)), F.rms_norm(k, (k.size(-1),)) # QK norm suggested by @Grad62304977
q, k = apply_rotary_emb(q, cos, sin), apply_rotary_emb(k, cos, sin)
y = F.scaled_dot_product_attention(q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2), is_causal=True)
y = y.transpose(1, 2).contiguous().view_as(x) # re-assemble all head outputs side by side
y = self.c_proj(y)
return y, v1
class MLP(nn.Module):
def __init__(self, config):
super().__init__()
self.c_fc = nn.Linear(config.n_embd, 4 * config.n_embd, bias=False)
self.c_proj = nn.Linear(4 * config.n_embd, config.n_embd, bias=False)
self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977
def forward(self, x):
x = self.c_fc(x)
x = F.relu(x).square() # https://arxiv.org/abs/2109.08668v2; ~1-2% better than GELU; suggested by @SKYLINEZ007 and @Grad62304977
x = self.c_proj(x)
return x
class Block(nn.Module):
def __init__(self, config):
super().__init__()
self.attn = CausalSelfAttention(config)
self.mlp = MLP(config)
self.lambdas = nn.Parameter(torch.tensor([1., 0.]))
def forward(self, x, v1, x0):
x = self.lambdas[0] * x + self.lambdas[1] * x0
x1, v1 = self.attn(F.rms_norm(x, (x.size(-1),)), v1)
x = x + x1
x = x + self.mlp(F.rms_norm(x, (x.size(-1),)))
return x, v1
# -----------------------------------------------------------------------------
# The main GPT-2 model
@dataclass
class GPTConfig:
vocab_size : int = 50304
n_layer : int = 12
n_head : int = 6 # head dim 128 suggested by @Grad62304977
n_embd : int = 768
class GPT(nn.Module):
def __init__(self, config):
super().__init__()
self.config = config
self.transformer = nn.ModuleDict(dict(
wte = nn.Embedding(config.vocab_size, config.n_embd),
h = nn.ModuleList([Block(config) for _ in range(config.n_layer)]),
))
self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False)
self.lm_head.weight.data.zero_() # @Grad62304977
def forward(self, idx, targets=None, return_logits=True):
# forward the GPT model itself
x = self.transformer.wte(idx) # token embeddings of shape (b, t, n_embd)
x = F.rms_norm(x, (x.size(-1),)) # @Grad62304977
x0 = x
v1 = None
for block in self.transformer.h:
x, v1 = block(x, v1, x0)
x = F.rms_norm(x, (x.size(-1),))
if targets is not None:
# if we are given some desired targets also calculate the loss
logits = self.lm_head(x)
logits = 30 * torch.tanh(logits / 30)
logits = logits.float() # use tf32/fp32 for logits
loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1), ignore_index=-1)
else:
# inference-time mini-optimization: only forward the lm_head on the very last position
logits = self.lm_head(x[:, [-1], :]) # note: using list [-1] to preserve the time dim
logits = 30 * torch.tanh(logits / 30)
logits = logits.float() # use tf32/fp32 for logits
loss = None
# there are performance reasons why not returning logits is prudent, if not needed
if not return_logits:
logits = None
return logits, loss
# -----------------------------------------------------------------------------
# Our own simple Distributed Data Loader
def _peek_data_shard(filename):
# only reads the header, returns header data
with open(filename, "rb") as f:
# first read the header, which is 256 int32 integers (4 bytes each)
header = np.frombuffer(f.read(256*4), dtype=np.int32)
if header[0] != 20240520:
print("ERROR: magic number mismatch in the data .bin file!")
print("---> HINT: Are you passing in a correct file with --input_bin?")
print("---> HINT: Dataset encoding changed recently, re-run data prepro or refer again to README")
print("---> HINT: For example re-run: `python dev/data/tinyshakespeare.py`, then re-try")
exit(1)
assert header[1] == 1, "unsupported version"
ntok = header[2] # number of tokens (claimed)
return ntok # for now just return the number of tokens
def _load_data_shard(filename):
with open(filename, "rb") as f:
# first read the header, which is 256 int32 integers (4 bytes each)
header = np.frombuffer(f.read(256*4), dtype=np.int32)
assert header[0] == 20240520, "magic number mismatch in the data .bin file"
assert header[1] == 1, "unsupported version"
ntok = header[2] # number of tokens (claimed)
# the rest of it are tokens, stored as uint16
tokens = np.frombuffer(f.read(), dtype=np.uint16)
assert len(tokens) == ntok, "number of tokens read does not match header?"
return tokens
class DistributedDataLoader:
def __init__(self, filename_pattern, B, T, process_rank, num_processes):
self.process_rank = process_rank
self.num_processes = num_processes
self.B = B
self.T = T
# glob files that match the pattern
self.files = sorted(glob.glob(filename_pattern))
assert len(self.files) > 0, f"did not find any files that match the pattern {filename_pattern}"
# load and validate all data shards, count number of tokens in total
ntok_total = 0
for fname in self.files:
shard_ntok = _peek_data_shard(fname)
assert shard_ntok >= num_processes * B * T + 1
ntok_total += int(shard_ntok)
self.ntok_total = ntok_total
# kick things off
self.reset()
def reset(self):
self.current_shard = 0
self.current_position = self.process_rank * self.B * self.T
self.tokens = _load_data_shard(self.files[self.current_shard])
def advance(self): # advance to next data shard
self.current_shard = (self.current_shard + 1) % len(self.files)
self.current_position = self.process_rank * self.B * self.T
self.tokens = _load_data_shard(self.files[self.current_shard])
def next_batch(self):
B = self.B
T = self.T
buf = self.tokens[self.current_position : self.current_position+B*T+1]
buf = torch.tensor(buf.astype(np.int32), dtype=torch.long)
x = (buf[:-1]).view(B, T) # inputs
y = (buf[1:]).view(B, T) # targets
# advance current position and load next shard if necessary
self.current_position += B * T * self.num_processes
if self.current_position + (B * T * self.num_processes + 1) > len(self.tokens):
self.advance()
return x.cuda(), y.cuda()
# -----------------------------------------------------------------------------
# int main
@dataclass
class Hyperparameters:
# data hyperparams
input_bin : str = 'data/fineweb10B/fineweb_train_*.bin' # input .bin to train on
input_val_bin : str = 'data/fineweb10B/fineweb_val_*.bin' # input .bin to eval validation loss on
# optimization hyperparams
batch_size : int = 8*64 # batch size, in sequences, across all devices
device_batch_size : int = 64 # batch size, in sequences, per device
sequence_length : int = 1024 # sequence length, in tokens
num_iterations : int = 3200 # number of iterations to run
warmup_iters : int = 0
warmdown_iters : int = 914 # number of iterations of linear warmup/warmdown for triangular or trapezoidal schedule
weight_decay : float = 0
# evaluation and logging hyperparams
val_loss_every : int = 125 # every how many steps to evaluate val loss? 0 for only at the end
val_tokens : int = 10485760 # how many tokens of validation data? it's important to keep this fixed for consistent comparisons
save_every : int = 0 # every how many steps to save the checkpoint? 0 for only at the end
args = Hyperparameters()
# set up DDP (distributed data parallel). torchrun sets this env variable
assert torch.cuda.is_available()
dist.init_process_group(backend='nccl')
ddp_rank = int(os.environ['RANK'])
ddp_local_rank = int(os.environ['LOCAL_RANK'])
ddp_world_size = int(os.environ['WORLD_SIZE'])
device = f'cuda:{ddp_local_rank}'
torch.cuda.set_device(device)
print(f"using device: {device}")
master_process = (ddp_rank == 0) # this process will do logging, checkpointing etc.
# convenience variables
B, T = args.device_batch_size, args.sequence_length
# calculate the number of steps to take in the val loop.
assert args.val_tokens % (B * T * ddp_world_size) == 0
val_steps = args.val_tokens // (B * T * ddp_world_size)
# calculate the steps of gradient accumulation required to attain the desired global batch size.
assert args.batch_size % (B * ddp_world_size) == 0
train_accumulation_steps = args.batch_size // (B * ddp_world_size)
# load tokens
train_loader = DistributedDataLoader(args.input_bin, B, T, ddp_rank, ddp_world_size)
val_loader = DistributedDataLoader(args.input_val_bin, B, T, ddp_rank, ddp_world_size)
if master_process:
print(f"Training DataLoader: total number of tokens: {train_loader.ntok_total} across {len(train_loader.files)} files")
print(f"Validation DataLoader: total number of tokens: {val_loader.ntok_total} across {len(val_loader.files)} files")
x, y = train_loader.next_batch()
# there are only 50257 unique GPT-2 tokens; we extend to nearest multiple of 128 for efficiency. suggested to me by @Grad62304977.
# this originates from Karpathy's experiments.
num_vocab = 50304
model = GPT(GPTConfig(vocab_size=num_vocab, n_layer=12, n_head=6, n_embd=768))
model = model.cuda()
if hasattr(config, "coordinate_descent_tuning"):
config.coordinate_descent_tuning = True # suggested by @Chillee
model = torch.compile(model)
# here we wrap model into DDP container
model = DDP(model, device_ids=[ddp_local_rank])
raw_model = model.module # always contains the "raw" unwrapped model
ctx = torch.amp.autocast(device_type='cuda', dtype=torch.bfloat16)
# CUDNN attention is ~4ms faster than Flash, but doesn't get selected by default in PyTorch 2.5.1
from torch.backends.cuda import enable_cudnn_sdp, enable_flash_sdp, enable_math_sdp, enable_mem_efficient_sdp
enable_cudnn_sdp(True)
enable_flash_sdp(False)
enable_mem_efficient_sdp(False)
enable_math_sdp(False)
# init the optimizer(s)
optimizer1 = torch.optim.Adam([raw_model.transformer.wte.weight], lr=0.3, betas=(0.9, 0.95), fused=True)
optimizer2 = torch.optim.Adam([raw_model.lm_head.weight], lr=0.002, betas=(0.9, 0.95), fused=True)
params = list(raw_model.transformer.h.parameters())
matrix_params = [p for p in params if p.ndim == 2]
scalar_params = [p for p in params if p.ndim < 2]
optimizer3 = Muon(matrix_params, lr=0.02, momentum=0.95)
optimizer4 = torch.optim.Adam(scalar_params, lr=0.02, betas=(0.9, 0.95), fused=True) # note that this learning rate is neither sensitive nor tuned
optimizers = [optimizer1, optimizer2, optimizer3, optimizer4]
# learning rate decay scheduler (linear warmup and warmdown)
def get_lr(it):
assert it <= args.num_iterations
# 1) linear warmup for warmup_iters steps
if it < args.warmup_iters:
return (it+1) / args.warmup_iters
# 2) constant lr for a while
elif it < args.num_iterations - args.warmdown_iters:
return 1.0
# 3) linear warmdown
else:
decay_ratio = (args.num_iterations - it) / args.warmdown_iters
return decay_ratio
schedulers = [torch.optim.lr_scheduler.LambdaLR(opt, get_lr) for opt in optimizers]
# begin logging
if master_process:
run_id = str(uuid.uuid4())
logdir = 'logs/%s/' % run_id
os.makedirs(logdir, exist_ok=True)
logfile = 'logs/%s.txt' % run_id
# create the log file
with open(logfile, "w") as f:
# begin the log by printing this file (the Python code)
f.write('='*100 + '\n')
f.write(code)
f.write('='*100 + '\n')
# log information about the hardware/software environment this is running on
# and print the full `nvidia-smi` to file
f.write(f"Running pytorch {torch.version.__version__} compiled for CUDA {torch.version.cuda}\nnvidia-smi:\n")
import subprocess
result = subprocess.run(['nvidia-smi'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
f.write(f'{result.stdout}\n')
f.write('='*100 + '\n')
training_time_ms = 0
# start the clock
torch.cuda.synchronize()
t0 = time.time()
# begin training
train_loader.reset()
for step in range(args.num_iterations + 1):
last_step = (step == args.num_iterations)
# This effectively ignores timing first 10 steps, which are slower for weird reasons.
# Alternately, and slightly more correctly in terms of benchmarking, we could do 10
# steps with dummy data first, and then re-initialize the model and reset the loader.
if step == 10:
training_time_ms = 0
t0 = time.time()
timed_steps = float('nan') if step <= 11 else (step - 10) + 1 # <= 11 to avoid bug in val
# once in a while evaluate the validation dataset
if (last_step or (args.val_loss_every > 0 and step % args.val_loss_every == 0)):
# stop the clock
torch.cuda.synchronize()
training_time_ms += 1000 * (time.time() - t0)
# run validation batches
model.eval()
val_loader.reset()
val_loss = 0.0
for _ in range(val_steps):
x_val, y_val = val_loader.next_batch()
with ctx: # of course, we'd like to use no_grad() here too, but that creates a torch.compile error for some reason
_, loss = model(x_val, y_val, return_logits=False)
val_loss += loss.detach()
del loss
dist.all_reduce(val_loss, op=dist.ReduceOp.AVG)
val_loss /= val_steps
# log val loss to console and to logfile
if master_process:
print(f'step:{step}/{args.num_iterations} val_loss:{val_loss:.4f} train_time:{training_time_ms:.0f}ms step_avg:{training_time_ms/(timed_steps-1):.2f}ms')
with open(logfile, "a") as f:
f.write(f'step:{step}/{args.num_iterations} val_loss:{val_loss:.4f} train_time:{training_time_ms:.0f}ms step_avg:{training_time_ms/(timed_steps-1):.2f}ms\n')
# start the clock again
torch.cuda.synchronize()
t0 = time.time()
if master_process and (last_step or (args.save_every > 0 and step % args.save_every == 0)):
# stop the clock
torch.cuda.synchronize()
training_time_ms += 1000 * (time.time() - t0)
# save the state of the training process
log = dict(step=step, code=code, model=raw_model.state_dict(), optimizers=[opt.state_dict() for opt in optimizers])
torch.save(log, 'logs/%s/state_step%06d.pt' % (run_id, step))
# start the clock again
torch.cuda.synchronize()
t0 = time.time()
# bit confusing: we want to make sure to eval on 0th iteration
# but also after the very last iteration. so we loop for step <= num_iterations
# instead of just < num_iterations (one extra due to <=), only to do
# the validation/sampling one last time, and then we break right here as we're done.
if last_step:
break
# --------------- TRAINING SECTION BEGIN -----------------
model.train()
for i in range(1, train_accumulation_steps+1):
# forward pass
with ctx:
_, loss = model(x, y, return_logits=False)
train_loss = loss.detach()
# advance the dataset for the next batch
x, y = train_loader.next_batch()
# backward pass
if i < train_accumulation_steps:
with model.no_sync(): # there's no need to sync gradients every accumulation step
loss.backward()
else:
loss.backward() # just sync on the last step
for p in model.parameters():
p.grad /= train_accumulation_steps
# momentum warmup for Muon
frac = min(step/500, 1)
optimizer3.param_groups[0]['momentum'] = (1 - frac) * 0.85 + frac * 0.95
# step the optimizers and schedulers
for opt, sched in zip(optimizers, schedulers):
opt.step()
sched.step()
# null the gradients
model.zero_grad(set_to_none=True)
# --------------- TRAINING SECTION END -------------------
# everything that follows now is just diagnostics, prints, logging, etc.
#dist.all_reduce(train_loss, op=dist.ReduceOp.AVG) # all-reducing the training loss would be more correct in terms of logging, but slower
if master_process:
approx_time = training_time_ms + 1000 * (time.time() - t0)
print(f"step:{step+1}/{args.num_iterations} train_loss:{train_loss.item():.4f} train_time:{approx_time:.0f}ms step_avg:{approx_time/timed_steps:.2f}ms")
with open(logfile, "a") as f:
f.write(f"step:{step+1}/{args.num_iterations} train_loss:{train_loss.item():.4f} train_time:{approx_time:.0f}ms step_avg:{approx_time/timed_steps:.2f}ms\n")
if master_process:
print(f"peak memory consumption: {torch.cuda.max_memory_allocated() // 1024 // 1024} MiB")
# -------------------------------------------------------------------------
# clean up nice
dist.destroy_process_group()
====================================================================================================
Running pytorch 2.5.1+cu124 compiled for CUDA 12.4
nvidia-smi:
Wed Nov 6 20:35:58 2024
+-----------------------------------------------------------------------------------------+
| NVIDIA-SMI 555.42.06 Driver Version: 555.42.06 CUDA Version: 12.5 |
|-----------------------------------------+------------------------+----------------------+
| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |
| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |
| | | MIG M. |
|=========================================+========================+======================|
| 0 NVIDIA H100 80GB HBM3 Off | 00000000:18:00.0 Off | 0 |
| N/A 34C P0 142W / 700W | 5304MiB / 81559MiB | 4% Default |
| | | Disabled |
+-----------------------------------------+------------------------+----------------------+
| 1 NVIDIA H100 80GB HBM3 Off | 00000000:2A:00.0 Off | 0 |
| N/A 36C P0 131W / 700W | 5352MiB / 81559MiB | 1% Default |
| | | Disabled |
+-----------------------------------------+------------------------+----------------------+
| 2 NVIDIA H100 80GB HBM3 Off | 00000000:3A:00.0 Off | 0 |
| N/A 36C P0 126W / 700W | 5352MiB / 81559MiB | 3% Default |
| | | Disabled |
+-----------------------------------------+------------------------+----------------------+
| 3 NVIDIA H100 80GB HBM3 Off | 00000000:5D:00.0 Off | 0 |
| N/A 33C P0 138W / 700W | 5352MiB / 81559MiB | 1% Default |
| | | Disabled |
+-----------------------------------------+------------------------+----------------------+
| 4 NVIDIA H100 80GB HBM3 Off | 00000000:9A:00.0 Off | 0 |
| N/A 34C P0 143W / 700W | 5352MiB / 81559MiB | 6% Default |
| | | Disabled |
+-----------------------------------------+------------------------+----------------------+
| 5 NVIDIA H100 80GB HBM3 Off | 00000000:AB:00.0 Off | 0 |
| N/A 38C P0 143W / 700W | 5352MiB / 81559MiB | 0% Default |
| | | Disabled |
+-----------------------------------------+------------------------+----------------------+
| 6 NVIDIA H100 80GB HBM3 Off | 00000000:BA:00.0 Off | 0 |
| N/A 36C P0 143W / 700W | 5352MiB / 81559MiB | 0% Default |
| | | Disabled |
+-----------------------------------------+------------------------+----------------------+
| 7 NVIDIA H100 80GB HBM3 Off | 00000000:DB:00.0 Off | 0 |
| N/A 35C P0 148W / 700W | 5112MiB / 81559MiB | 0% Default |
| | | Disabled |
+-----------------------------------------+------------------------+----------------------+
+-----------------------------------------------------------------------------------------+
| Processes: |
| GPU GI CI PID Type Process name GPU Memory |
| ID ID Usage |
|=========================================================================================|
| 0 N/A N/A 37067 C /usr/bin/python3 0MiB |
| 1 N/A N/A 37068 C /usr/bin/python3 0MiB |
| 2 N/A N/A 37069 C /usr/bin/python3 0MiB |
| 3 N/A N/A 37070 C /usr/bin/python3 0MiB |
| 4 N/A N/A 37071 C /usr/bin/python3 0MiB |
| 5 N/A N/A 37072 C /usr/bin/python3 0MiB |
| 6 N/A N/A 37073 C /usr/bin/python3 0MiB |
| 7 N/A N/A 37074 C /usr/bin/python3 0MiB |
+-----------------------------------------------------------------------------------------+
====================================================================================================
step:0/3200 val_loss:10.8258 train_time:500ms step_avg:nanms
step:1/3200 train_loss:10.8258 train_time:4005ms step_avg:nanms
step:2/3200 train_loss:10.4255 train_time:4113ms step_avg:nanms
step:3/3200 train_loss:9.9515 train_time:4260ms step_avg:nanms
step:4/3200 train_loss:9.0438 train_time:4409ms step_avg:nanms
step:5/3200 train_loss:8.0548 train_time:4559ms step_avg:nanms
step:6/3200 train_loss:7.5162 train_time:4710ms step_avg:nanms
step:7/3200 train_loss:7.0043 train_time:4859ms step_avg:nanms
step:8/3200 train_loss:7.2697 train_time:5015ms step_avg:nanms
step:9/3200 train_loss:6.9117 train_time:5174ms step_avg:nanms
step:10/3200 train_loss:6.7929 train_time:5329ms step_avg:nanms
step:11/3200 train_loss:6.7074 train_time:105ms step_avg:nanms
step:12/3200 train_loss:6.6538 train_time:258ms step_avg:nanms
step:13/3200 train_loss:6.5110 train_time:409ms step_avg:136.37ms
step:14/3200 train_loss:6.4721 train_time:561ms step_avg:140.17ms
step:15/3200 train_loss:6.4485 train_time:714ms step_avg:142.89ms
step:16/3200 train_loss:6.4023 train_time:869ms step_avg:144.84ms
step:17/3200 train_loss:6.4160 train_time:1024ms step_avg:146.23ms
step:18/3200 train_loss:6.4444 train_time:1175ms step_avg:146.87ms
step:19/3200 train_loss:6.2863 train_time:1327ms step_avg:147.43ms
step:20/3200 train_loss:6.3073 train_time:1480ms step_avg:148.00ms
step:21/3200 train_loss:6.0091 train_time:1632ms step_avg:148.40ms
step:22/3200 train_loss:6.3298 train_time:1785ms step_avg:148.79ms
step:23/3200 train_loss:6.5706 train_time:1939ms step_avg:149.16ms
step:24/3200 train_loss:6.2304 train_time:2091ms step_avg:149.37ms
step:25/3200 train_loss:6.3913 train_time:2243ms step_avg:149.53ms
step:26/3200 train_loss:6.0954 train_time:2395ms step_avg:149.69ms
step:27/3200 train_loss:6.0114 train_time:2548ms step_avg:149.88ms
step:28/3200 train_loss:6.1928 train_time:2701ms step_avg:150.05ms
step:29/3200 train_loss:5.8496 train_time:2854ms step_avg:150.19ms
step:30/3200 train_loss:6.1124 train_time:3007ms step_avg:150.33ms
step:31/3200 train_loss:5.9479 train_time:3161ms step_avg:150.55ms
step:32/3200 train_loss:5.9178 train_time:3313ms step_avg:150.61ms
step:33/3200 train_loss:5.7507 train_time:3466ms step_avg:150.68ms
step:34/3200 train_loss:6.0573 train_time:3620ms step_avg:150.83ms
step:35/3200 train_loss:5.9758 train_time:3773ms step_avg:150.90ms
step:36/3200 train_loss:6.1187 train_time:3925ms step_avg:150.98ms
step:37/3200 train_loss:6.0368 train_time:4081ms step_avg:151.13ms
step:38/3200 train_loss:5.9287 train_time:4234ms step_avg:151.22ms
step:39/3200 train_loss:5.8182 train_time:4386ms step_avg:151.24ms
step:40/3200 train_loss:5.8364 train_time:4540ms step_avg:151.35ms
step:41/3200 train_loss:5.7583 train_time:4693ms step_avg:151.40ms
step:42/3200 train_loss:5.7574 train_time:4846ms step_avg:151.45ms
step:43/3200 train_loss:5.6558 train_time:5001ms step_avg:151.56ms
step:44/3200 train_loss:5.7347 train_time:5154ms step_avg:151.58ms
step:45/3200 train_loss:5.7190 train_time:5307ms step_avg:151.63ms
step:46/3200 train_loss:5.8601 train_time:5461ms step_avg:151.68ms
step:47/3200 train_loss:5.6619 train_time:5613ms step_avg:151.72ms
step:48/3200 train_loss:5.5251 train_time:5766ms step_avg:151.73ms
step:49/3200 train_loss:5.7180 train_time:5920ms step_avg:151.79ms
step:50/3200 train_loss:5.6031 train_time:6074ms step_avg:151.84ms
step:51/3200 train_loss:5.7500 train_time:6226ms step_avg:151.85ms
step:52/3200 train_loss:5.6044 train_time:6381ms step_avg:151.93ms
step:53/3200 train_loss:5.4535 train_time:6535ms step_avg:151.98ms
step:54/3200 train_loss:5.5791 train_time:6687ms step_avg:151.97ms
step:55/3200 train_loss:5.4591 train_time:6841ms step_avg:152.02ms
step:56/3200 train_loss:5.7994 train_time:6994ms step_avg:152.04ms
step:57/3200 train_loss:5.4515 train_time:7148ms step_avg:152.08ms
step:58/3200 train_loss:5.3336 train_time:7302ms step_avg:152.13ms
step:59/3200 train_loss:5.4599 train_time:7457ms step_avg:152.17ms
step:60/3200 train_loss:5.4339 train_time:7609ms step_avg:152.18ms
step:61/3200 train_loss:5.5291 train_time:7763ms step_avg:152.21ms
step:62/3200 train_loss:5.2841 train_time:7916ms step_avg:152.23ms
step:63/3200 train_loss:5.3944 train_time:8069ms step_avg:152.24ms
step:64/3200 train_loss:5.3683 train_time:8223ms step_avg:152.27ms
step:65/3200 train_loss:5.1839 train_time:8375ms step_avg:152.28ms
step:66/3200 train_loss:5.1878 train_time:8529ms step_avg:152.30ms
step:67/3200 train_loss:5.3237 train_time:8684ms step_avg:152.35ms
step:68/3200 train_loss:5.2004 train_time:8838ms step_avg:152.38ms
step:69/3200 train_loss:5.4509 train_time:8991ms step_avg:152.39ms
step:70/3200 train_loss:5.1061 train_time:9144ms step_avg:152.40ms
step:71/3200 train_loss:5.1653 train_time:9298ms step_avg:152.43ms
step:72/3200 train_loss:5.3359 train_time:9450ms step_avg:152.42ms
step:73/3200 train_loss:5.2674 train_time:9604ms step_avg:152.44ms
step:74/3200 train_loss:5.1529 train_time:9759ms step_avg:152.48ms
step:75/3200 train_loss:5.2657 train_time:9911ms step_avg:152.48ms
step:76/3200 train_loss:5.2576 train_time:10065ms step_avg:152.49ms
step:77/3200 train_loss:5.2006 train_time:10219ms step_avg:152.53ms
step:78/3200 train_loss:5.2847 train_time:10371ms step_avg:152.51ms
step:79/3200 train_loss:5.3913 train_time:10525ms step_avg:152.53ms
step:80/3200 train_loss:5.1372 train_time:10680ms step_avg:152.57ms
step:81/3200 train_loss:5.2198 train_time:10831ms step_avg:152.55ms
step:82/3200 train_loss:4.9867 train_time:10986ms step_avg:152.58ms
step:83/3200 train_loss:5.1641 train_time:11140ms step_avg:152.61ms
step:84/3200 train_loss:5.1085 train_time:11293ms step_avg:152.60ms
step:85/3200 train_loss:5.1013 train_time:11446ms step_avg:152.61ms
step:86/3200 train_loss:4.9684 train_time:11600ms step_avg:152.63ms
step:87/3200 train_loss:5.1597 train_time:11753ms step_avg:152.64ms
step:88/3200 train_loss:5.0705 train_time:11906ms step_avg:152.64ms
step:89/3200 train_loss:5.1280 train_time:12061ms step_avg:152.68ms
step:90/3200 train_loss:5.0896 train_time:12214ms step_avg:152.68ms
step:91/3200 train_loss:5.0028 train_time:12367ms step_avg:152.68ms
step:92/3200 train_loss:5.0068 train_time:12521ms step_avg:152.69ms
step:93/3200 train_loss:5.1249 train_time:12675ms step_avg:152.71ms
step:94/3200 train_loss:4.9582 train_time:12828ms step_avg:152.71ms
step:95/3200 train_loss:4.9620 train_time:12983ms step_avg:152.74ms
step:96/3200 train_loss:5.0129 train_time:13136ms step_avg:152.74ms
step:97/3200 train_loss:4.9138 train_time:13287ms step_avg:152.73ms
step:98/3200 train_loss:4.9899 train_time:13441ms step_avg:152.74ms
step:99/3200 train_loss:4.9126 train_time:13593ms step_avg:152.73ms
step:100/3200 train_loss:5.0270 train_time:13746ms step_avg:152.74ms
step:101/3200 train_loss:4.9917 train_time:13901ms step_avg:152.75ms
step:102/3200 train_loss:4.8763 train_time:14055ms step_avg:152.77ms
step:103/3200 train_loss:5.0072 train_time:14207ms step_avg:152.77ms
step:104/3200 train_loss:4.9412 train_time:14361ms step_avg:152.77ms
step:105/3200 train_loss:4.8177 train_time:14513ms step_avg:152.77ms
step:106/3200 train_loss:4.8776 train_time:14666ms step_avg:152.77ms
step:107/3200 train_loss:5.0620 train_time:14820ms step_avg:152.79ms
step:108/3200 train_loss:4.8546 train_time:14973ms step_avg:152.79ms
step:109/3200 train_loss:4.6639 train_time:15126ms step_avg:152.79ms
step:110/3200 train_loss:4.8249 train_time:15281ms step_avg:152.81ms
step:111/3200 train_loss:4.8100 train_time:15434ms step_avg:152.81ms
step:112/3200 train_loss:4.7652 train_time:15587ms step_avg:152.81ms
step:113/3200 train_loss:4.9038 train_time:15741ms step_avg:152.82ms
step:114/3200 train_loss:4.8001 train_time:15893ms step_avg:152.82ms
step:115/3200 train_loss:4.6745 train_time:16045ms step_avg:152.81ms
step:116/3200 train_loss:4.8122 train_time:16200ms step_avg:152.83ms
step:117/3200 train_loss:4.7418 train_time:16353ms step_avg:152.83ms
step:118/3200 train_loss:4.6785 train_time:16506ms step_avg:152.83ms
step:119/3200 train_loss:4.8485 train_time:16660ms step_avg:152.84ms
step:120/3200 train_loss:4.7672 train_time:16812ms step_avg:152.84ms
step:121/3200 train_loss:4.6687 train_time:16964ms step_avg:152.83ms
step:122/3200 train_loss:4.6049 train_time:17119ms step_avg:152.85ms
step:123/3200 train_loss:4.7315 train_time:17271ms step_avg:152.84ms
step:124/3200 train_loss:4.5815 train_time:17424ms step_avg:152.84ms
step:125/3200 train_loss:4.8791 train_time:17579ms step_avg:152.86ms
step:125/3200 val_loss:4.6960 train_time:17627ms step_avg:153.27ms
step:126/3200 train_loss:4.7335 train_time:17736ms step_avg:152.90ms
step:127/3200 train_loss:4.6899 train_time:17890ms step_avg:152.91ms
step:128/3200 train_loss:4.7280 train_time:18043ms step_avg:152.91ms
step:129/3200 train_loss:4.6442 train_time:18194ms step_avg:152.89ms
step:130/3200 train_loss:4.9419 train_time:18346ms step_avg:152.88ms
step:131/3200 train_loss:4.6436 train_time:18498ms step_avg:152.87ms
step:132/3200 train_loss:4.6640 train_time:18653ms step_avg:152.89ms
step:133/3200 train_loss:4.6142 train_time:18809ms step_avg:152.92ms
step:134/3200 train_loss:4.6928 train_time:18963ms step_avg:152.93ms
step:135/3200 train_loss:4.5406 train_time:19115ms step_avg:152.92ms
step:136/3200 train_loss:4.6679 train_time:19267ms step_avg:152.91ms
step:137/3200 train_loss:4.4498 train_time:19418ms step_avg:152.90ms
step:138/3200 train_loss:4.6155 train_time:19572ms step_avg:152.91ms
step:139/3200 train_loss:4.5299 train_time:19727ms step_avg:152.93ms
step:140/3200 train_loss:4.6041 train_time:19881ms step_avg:152.93ms
step:141/3200 train_loss:4.6707 train_time:20033ms step_avg:152.92ms
step:142/3200 train_loss:4.5370 train_time:20187ms step_avg:152.93ms
step:143/3200 train_loss:4.5428 train_time:20338ms step_avg:152.92ms
step:144/3200 train_loss:4.4500 train_time:20489ms step_avg:152.91ms
step:145/3200 train_loss:4.5636 train_time:20645ms step_avg:152.92ms
step:146/3200 train_loss:4.5158 train_time:20797ms step_avg:152.92ms
step:147/3200 train_loss:4.3971 train_time:20951ms step_avg:152.93ms
step:148/3200 train_loss:4.5241 train_time:21105ms step_avg:152.93ms
step:149/3200 train_loss:4.5494 train_time:21257ms step_avg:152.93ms
step:150/3200 train_loss:4.5057 train_time:21410ms step_avg:152.93ms
step:151/3200 train_loss:4.6171 train_time:21563ms step_avg:152.93ms
step:152/3200 train_loss:4.4686 train_time:21716ms step_avg:152.93ms
step:153/3200 train_loss:4.4585 train_time:21870ms step_avg:152.94ms
step:154/3200 train_loss:4.5349 train_time:22023ms step_avg:152.94ms
step:155/3200 train_loss:4.5296 train_time:22176ms step_avg:152.94ms
step:156/3200 train_loss:4.4508 train_time:22329ms step_avg:152.94ms
step:157/3200 train_loss:4.4964 train_time:22482ms step_avg:152.94ms
step:158/3200 train_loss:4.5789 train_time:22635ms step_avg:152.94ms
step:159/3200 train_loss:4.4001 train_time:22789ms step_avg:152.95ms
step:160/3200 train_loss:4.4738 train_time:22944ms step_avg:152.96ms
step:161/3200 train_loss:4.2815 train_time:23095ms step_avg:152.95ms
step:162/3200 train_loss:4.4951 train_time:23249ms step_avg:152.95ms
step:163/3200 train_loss:4.5080 train_time:23402ms step_avg:152.96ms
step:164/3200 train_loss:4.4850 train_time:23554ms step_avg:152.95ms
step:165/3200 train_loss:4.3441 train_time:23708ms step_avg:152.96ms
step:166/3200 train_loss:4.4280 train_time:23862ms step_avg:152.96ms
step:167/3200 train_loss:4.5183 train_time:24014ms step_avg:152.96ms
step:168/3200 train_loss:4.3410 train_time:24168ms step_avg:152.96ms
step:169/3200 train_loss:4.4179 train_time:24321ms step_avg:152.96ms
step:170/3200 train_loss:4.3136 train_time:24472ms step_avg:152.95ms
step:171/3200 train_loss:4.1862 train_time:24627ms step_avg:152.96ms
step:172/3200 train_loss:4.3355 train_time:24779ms step_avg:152.96ms
step:173/3200 train_loss:4.3455 train_time:24932ms step_avg:152.96ms
step:174/3200 train_loss:4.4035 train_time:25087ms step_avg:152.97ms
step:175/3200 train_loss:4.5629 train_time:25240ms step_avg:152.97ms
step:176/3200 train_loss:4.3845 train_time:25392ms step_avg:152.96ms
step:177/3200 train_loss:4.2446 train_time:25547ms step_avg:152.98ms
step:178/3200 train_loss:4.2086 train_time:25699ms step_avg:152.97ms
step:179/3200 train_loss:4.3122 train_time:25851ms step_avg:152.96ms
step:180/3200 train_loss:4.2715 train_time:26005ms step_avg:152.97ms
step:181/3200 train_loss:4.2447 train_time:26158ms step_avg:152.97ms
step:182/3200 train_loss:4.4174 train_time:26311ms step_avg:152.97ms
step:183/3200 train_loss:4.2909 train_time:26467ms step_avg:152.99ms
step:184/3200 train_loss:4.2696 train_time:26619ms step_avg:152.98ms
step:185/3200 train_loss:4.2664 train_time:26771ms step_avg:152.98ms
step:186/3200 train_loss:4.3401 train_time:26924ms step_avg:152.98ms
step:187/3200 train_loss:4.3035 train_time:27076ms step_avg:152.97ms
step:188/3200 train_loss:4.3658 train_time:27229ms step_avg:152.97ms
step:189/3200 train_loss:4.2962 train_time:27530ms step_avg:153.80ms
step:190/3200 train_loss:4.2358 train_time:27862ms step_avg:154.79ms
step:191/3200 train_loss:4.3247 train_time:28010ms step_avg:154.75ms
step:192/3200 train_loss:4.2033 train_time:28160ms step_avg:154.73ms
step:193/3200 train_loss:4.1433 train_time:28311ms step_avg:154.71ms
step:194/3200 train_loss:4.3598 train_time:28463ms step_avg:154.69ms
step:195/3200 train_loss:4.2698 train_time:28614ms step_avg:154.67ms
step:196/3200 train_loss:4.4815 train_time:28772ms step_avg:154.69ms
step:197/3200 train_loss:4.3094 train_time:28927ms step_avg:154.69ms
step:198/3200 train_loss:4.1638 train_time:29078ms step_avg:154.67ms
step:199/3200 train_loss:4.2942 train_time:29231ms step_avg:154.66ms
step:200/3200 train_loss:4.1397 train_time:29383ms step_avg:154.65ms
step:201/3200 train_loss:4.2391 train_time:29535ms step_avg:154.64ms
step:202/3200 train_loss:4.1148 train_time:29688ms step_avg:154.62ms
step:203/3200 train_loss:4.3506 train_time:29843ms step_avg:154.63ms
step:204/3200 train_loss:4.1804 train_time:29996ms step_avg:154.62ms
step:205/3200 train_loss:4.2925 train_time:30149ms step_avg:154.61ms
step:206/3200 train_loss:4.3473 train_time:30302ms step_avg:154.60ms
step:207/3200 train_loss:4.0511 train_time:30453ms step_avg:154.59ms
step:208/3200 train_loss:4.2004 train_time:30607ms step_avg:154.58ms
step:209/3200 train_loss:4.1923 train_time:30759ms step_avg:154.57ms
step:210/3200 train_loss:4.3474 train_time:30913ms step_avg:154.56ms
step:211/3200 train_loss:4.2727 train_time:31067ms step_avg:154.56ms
step:212/3200 train_loss:4.1662 train_time:31220ms step_avg:154.56ms
step:213/3200 train_loss:4.1868 train_time:31372ms step_avg:154.54ms
step:214/3200 train_loss:4.1536 train_time:31525ms step_avg:154.54ms
step:215/3200 train_loss:4.2164 train_time:31679ms step_avg:154.53ms
step:216/3200 train_loss:4.0380 train_time:31831ms step_avg:154.52ms
step:217/3200 train_loss:4.0990 train_time:31986ms step_avg:154.52ms
step:218/3200 train_loss:4.1067 train_time:32141ms step_avg:154.52ms
step:219/3200 train_loss:4.1784 train_time:32293ms step_avg:154.51ms
step:220/3200 train_loss:4.1659 train_time:32448ms step_avg:154.51ms
step:221/3200 train_loss:4.1899 train_time:32600ms step_avg:154.50ms
step:222/3200 train_loss:4.1999 train_time:32752ms step_avg:154.49ms
step:223/3200 train_loss:4.1116 train_time:32906ms step_avg:154.49ms
step:224/3200 train_loss:4.0705 train_time:33060ms step_avg:154.49ms
step:225/3200 train_loss:4.3843 train_time:33214ms step_avg:154.48ms
step:226/3200 train_loss:4.0075 train_time:33366ms step_avg:154.47ms
step:227/3200 train_loss:4.0758 train_time:33520ms step_avg:154.47ms
step:228/3200 train_loss:4.0916 train_time:33671ms step_avg:154.45ms
step:229/3200 train_loss:4.2322 train_time:33824ms step_avg:154.45ms
step:230/3200 train_loss:4.0129 train_time:33975ms step_avg:154.43ms
step:231/3200 train_loss:4.1459 train_time:34129ms step_avg:154.43ms
step:232/3200 train_loss:3.9934 train_time:34284ms step_avg:154.43ms
step:233/3200 train_loss:4.0625 train_time:34436ms step_avg:154.42ms
step:234/3200 train_loss:4.1885 train_time:34588ms step_avg:154.41ms
step:235/3200 train_loss:4.1111 train_time:34741ms step_avg:154.41ms
step:236/3200 train_loss:3.9994 train_time:34893ms step_avg:154.39ms
step:237/3200 train_loss:4.1602 train_time:35047ms step_avg:154.39ms
step:238/3200 train_loss:4.1728 train_time:35199ms step_avg:154.38ms
step:239/3200 train_loss:4.0272 train_time:35353ms step_avg:154.38ms
step:240/3200 train_loss:4.1735 train_time:35507ms step_avg:154.38ms
step:241/3200 train_loss:4.2026 train_time:35659ms step_avg:154.37ms
step:242/3200 train_loss:4.0505 train_time:35813ms step_avg:154.36ms
step:243/3200 train_loss:4.2295 train_time:35967ms step_avg:154.36ms
step:244/3200 train_loss:4.1052 train_time:36120ms step_avg:154.36ms
step:245/3200 train_loss:4.1607 train_time:36273ms step_avg:154.35ms
step:246/3200 train_loss:4.2335 train_time:36427ms step_avg:154.35ms
step:247/3200 train_loss:4.1491 train_time:36580ms step_avg:154.35ms
step:248/3200 train_loss:4.0982 train_time:36732ms step_avg:154.34ms
step:249/3200 train_loss:4.2052 train_time:36887ms step_avg:154.34ms
step:250/3200 train_loss:4.0108 train_time:37040ms step_avg:154.33ms
step:250/3200 val_loss:4.0963 train_time:37087ms step_avg:154.53ms
step:251/3200 train_loss:4.0571 train_time:37200ms step_avg:154.36ms
step:252/3200 train_loss:4.1607 train_time:37353ms step_avg:154.35ms
step:253/3200 train_loss:4.2259 train_time:37504ms step_avg:154.34ms
step:254/3200 train_loss:4.0209 train_time:37656ms step_avg:154.33ms
step:255/3200 train_loss:3.9667 train_time:37806ms step_avg:154.31ms
step:256/3200 train_loss:4.1491 train_time:37958ms step_avg:154.30ms
step:257/3200 train_loss:4.0633 train_time:38113ms step_avg:154.31ms
step:258/3200 train_loss:4.0757 train_time:38269ms step_avg:154.31ms
step:259/3200 train_loss:4.0587 train_time:38423ms step_avg:154.31ms
step:260/3200 train_loss:4.1098 train_time:38575ms step_avg:154.30ms
step:261/3200 train_loss:4.1413 train_time:38728ms step_avg:154.30ms
step:262/3200 train_loss:4.1132 train_time:38880ms step_avg:154.28ms
step:263/3200 train_loss:4.0755 train_time:39034ms step_avg:154.28ms
step:264/3200 train_loss:3.9819 train_time:39188ms step_avg:154.28ms
step:265/3200 train_loss:4.0741 train_time:39342ms step_avg:154.28ms
step:266/3200 train_loss:3.9516 train_time:39496ms step_avg:154.28ms
step:267/3200 train_loss:4.0036 train_time:39647ms step_avg:154.27ms
step:268/3200 train_loss:4.0031 train_time:39800ms step_avg:154.26ms
step:269/3200 train_loss:4.0357 train_time:39952ms step_avg:154.25ms
step:270/3200 train_loss:3.9484 train_time:40103ms step_avg:154.24ms
step:271/3200 train_loss:4.1834 train_time:40258ms step_avg:154.25ms
step:272/3200 train_loss:4.0605 train_time:40412ms step_avg:154.24ms
step:273/3200 train_loss:3.9931 train_time:40564ms step_avg:154.23ms
step:274/3200 train_loss:4.0408 train_time:40718ms step_avg:154.23ms
step:275/3200 train_loss:4.1118 train_time:40870ms step_avg:154.23ms
step:276/3200 train_loss:4.1442 train_time:41022ms step_avg:154.22ms
step:277/3200 train_loss:4.3047 train_time:41177ms step_avg:154.22ms
step:278/3200 train_loss:4.1117 train_time:41331ms step_avg:154.22ms
step:279/3200 train_loss:4.1635 train_time:41483ms step_avg:154.21ms
step:280/3200 train_loss:4.0754 train_time:41638ms step_avg:154.21ms
step:281/3200 train_loss:4.2107 train_time:41790ms step_avg:154.21ms
step:282/3200 train_loss:4.0291 train_time:41942ms step_avg:154.20ms
step:283/3200 train_loss:4.0311 train_time:42096ms step_avg:154.20ms
step:284/3200 train_loss:3.9835 train_time:42249ms step_avg:154.19ms
step:285/3200 train_loss:4.1276 train_time:42401ms step_avg:154.19ms
step:286/3200 train_loss:4.1418 train_time:42554ms step_avg:154.18ms
step:287/3200 train_loss:4.1713 train_time:42707ms step_avg:154.18ms
step:288/3200 train_loss:3.9934 train_time:42860ms step_avg:154.17ms
step:289/3200 train_loss:4.0973 train_time:43013ms step_avg:154.17ms
step:290/3200 train_loss:3.9494 train_time:43166ms step_avg:154.16ms
step:291/3200 train_loss:3.9409 train_time:43319ms step_avg:154.16ms
step:292/3200 train_loss:4.0092 train_time:43471ms step_avg:154.15ms
step:293/3200 train_loss:3.9405 train_time:43624ms step_avg:154.15ms
step:294/3200 train_loss:3.9920 train_time:43779ms step_avg:154.15ms
step:295/3200 train_loss:4.0367 train_time:43931ms step_avg:154.14ms
step:296/3200 train_loss:3.9191 train_time:44083ms step_avg:154.14ms
step:297/3200 train_loss:3.9338 train_time:44237ms step_avg:154.14ms
step:298/3200 train_loss:3.9357 train_time:44389ms step_avg:154.13ms
step:299/3200 train_loss:4.0480 train_time:44541ms step_avg:154.12ms
step:300/3200 train_loss:3.9004 train_time:44696ms step_avg:154.13ms
step:301/3200 train_loss:4.0408 train_time:44849ms step_avg:154.12ms
step:302/3200 train_loss:4.0590 train_time:45001ms step_avg:154.11ms
step:303/3200 train_loss:4.0086 train_time:45156ms step_avg:154.12ms
step:304/3200 train_loss:4.0555 train_time:45309ms step_avg:154.11ms
step:305/3200 train_loss:4.0359 train_time:45461ms step_avg:154.11ms
step:306/3200 train_loss:4.5263 train_time:45616ms step_avg:154.11ms
step:307/3200 train_loss:4.0041 train_time:45769ms step_avg:154.10ms
step:308/3200 train_loss:3.9119 train_time:45922ms step_avg:154.10ms
step:309/3200 train_loss:4.0508 train_time:46076ms step_avg:154.10ms
step:310/3200 train_loss:3.9331 train_time:46229ms step_avg:154.10ms
step:311/3200 train_loss:4.1628 train_time:46382ms step_avg:154.09ms
step:312/3200 train_loss:4.0094 train_time:46536ms step_avg:154.09ms
step:313/3200 train_loss:3.9491 train_time:46690ms step_avg:154.09ms
step:314/3200 train_loss:4.0251 train_time:46842ms step_avg:154.09ms
step:315/3200 train_loss:4.1542 train_time:46996ms step_avg:154.09ms
step:316/3200 train_loss:4.0274 train_time:47148ms step_avg:154.08ms
step:317/3200 train_loss:3.8736 train_time:47301ms step_avg:154.07ms
step:318/3200 train_loss:3.9507 train_time:47455ms step_avg:154.08ms
step:319/3200 train_loss:3.9865 train_time:47606ms step_avg:154.07ms
step:320/3200 train_loss:3.9617 train_time:47761ms step_avg:154.07ms
step:321/3200 train_loss:4.0800 train_time:47916ms step_avg:154.07ms
step:322/3200 train_loss:4.0195 train_time:48068ms step_avg:154.07ms
step:323/3200 train_loss:4.0032 train_time:48221ms step_avg:154.06ms
step:324/3200 train_loss:4.0876 train_time:48375ms step_avg:154.06ms
step:325/3200 train_loss:4.0232 train_time:48529ms step_avg:154.06ms
step:326/3200 train_loss:4.0890 train_time:48681ms step_avg:154.05ms
step:327/3200 train_loss:3.9599 train_time:48836ms step_avg:154.06ms
step:328/3200 train_loss:4.4711 train_time:48989ms step_avg:154.05ms
step:329/3200 train_loss:4.1494 train_time:49141ms step_avg:154.05ms
step:330/3200 train_loss:3.8837 train_time:49296ms step_avg:154.05ms
step:331/3200 train_loss:3.8275 train_time:49448ms step_avg:154.04ms
step:332/3200 train_loss:4.0502 train_time:49601ms step_avg:154.04ms
step:333/3200 train_loss:3.9804 train_time:49756ms step_avg:154.04ms
step:334/3200 train_loss:3.9492 train_time:49908ms step_avg:154.04ms
step:335/3200 train_loss:3.9147 train_time:50061ms step_avg:154.04ms
step:336/3200 train_loss:4.0870 train_time:50217ms step_avg:154.04ms
step:337/3200 train_loss:4.0315 train_time:50369ms step_avg:154.03ms
step:338/3200 train_loss:4.4921 train_time:50521ms step_avg:154.03ms
step:339/3200 train_loss:4.0162 train_time:50673ms step_avg:154.02ms
step:340/3200 train_loss:3.9641 train_time:50826ms step_avg:154.02ms
step:341/3200 train_loss:4.0010 train_time:50979ms step_avg:154.02ms
step:342/3200 train_loss:3.9249 train_time:51132ms step_avg:154.01ms
step:343/3200 train_loss:3.8853 train_time:51285ms step_avg:154.01ms
step:344/3200 train_loss:3.9139 train_time:51438ms step_avg:154.01ms
step:345/3200 train_loss:4.0679 train_time:51589ms step_avg:154.00ms
step:346/3200 train_loss:3.9108 train_time:51743ms step_avg:154.00ms
step:347/3200 train_loss:3.8481 train_time:51897ms step_avg:154.00ms
step:348/3200 train_loss:3.8768 train_time:52049ms step_avg:153.99ms
step:349/3200 train_loss:3.9401 train_time:52201ms step_avg:153.98ms
step:350/3200 train_loss:3.9048 train_time:52353ms step_avg:153.98ms
step:351/3200 train_loss:3.6402 train_time:52506ms step_avg:153.98ms
step:352/3200 train_loss:3.9005 train_time:52659ms step_avg:153.97ms
step:353/3200 train_loss:4.2372 train_time:52813ms step_avg:153.97ms
step:354/3200 train_loss:3.7339 train_time:52966ms step_avg:153.97ms
step:355/3200 train_loss:3.9983 train_time:53118ms step_avg:153.97ms
step:356/3200 train_loss:3.8624 train_time:53270ms step_avg:153.96ms
step:357/3200 train_loss:3.9698 train_time:53423ms step_avg:153.96ms
step:358/3200 train_loss:3.8836 train_time:53578ms step_avg:153.96ms
step:359/3200 train_loss:3.9262 train_time:53731ms step_avg:153.96ms
step:360/3200 train_loss:3.9245 train_time:53882ms step_avg:153.95ms
step:361/3200 train_loss:3.5100 train_time:54036ms step_avg:153.95ms
step:362/3200 train_loss:4.0946 train_time:54189ms step_avg:153.95ms
step:363/3200 train_loss:3.9981 train_time:54341ms step_avg:153.94ms
step:364/3200 train_loss:3.9233 train_time:54496ms step_avg:153.94ms
step:365/3200 train_loss:3.8265 train_time:54649ms step_avg:153.94ms
step:366/3200 train_loss:3.9944 train_time:54802ms step_avg:153.94ms
step:367/3200 train_loss:3.9419 train_time:54955ms step_avg:153.94ms
step:368/3200 train_loss:3.9387 train_time:55108ms step_avg:153.93ms
step:369/3200 train_loss:3.9244 train_time:55260ms step_avg:153.93ms
step:370/3200 train_loss:3.8214 train_time:55414ms step_avg:153.93ms
step:371/3200 train_loss:3.9652 train_time:55566ms step_avg:153.92ms
step:372/3200 train_loss:3.8318 train_time:55719ms step_avg:153.92ms
step:373/3200 train_loss:3.7738 train_time:55872ms step_avg:153.92ms
step:374/3200 train_loss:3.9972 train_time:56024ms step_avg:153.91ms
step:375/3200 train_loss:3.9147 train_time:56179ms step_avg:153.91ms
step:375/3200 val_loss:3.9117 train_time:56226ms step_avg:154.04ms
step:376/3200 train_loss:3.8952 train_time:56338ms step_avg:153.93ms
step:377/3200 train_loss:3.9507 train_time:56493ms step_avg:153.93ms
step:378/3200 train_loss:3.8697 train_time:56793ms step_avg:154.33ms