-
Notifications
You must be signed in to change notification settings - Fork 82
/
d6b50d71-f419-4d26-bb39-a60d55ae7a04.txt
5216 lines (5148 loc) · 347 KB
/
d6b50d71-f419-4d26-bb39-a60d55ae7a04.txt
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
====================================================================================================
import os
import sys
with open(sys.argv[0]) as f:
code = f.read() # read the code of this file ASAP, for logging
import uuid
import glob
import time
from dataclasses import dataclass
import numpy as np
import torch
from torch import nn
import torch.nn.functional as F
import torch.distributed as dist
import torch._inductor.config as config
from torch.nn.parallel import DistributedDataParallel as DDP
# -----------------------------------------------------------------------------
# Muon optimizer
def zeropower_via_svd(G, steps=None):
U, S, V = G.svd()
return U @ V.T
@torch.compile
def zeropower_via_newtonschulz5(G, steps=10, eps=1e-7):
"""
Newton-Schulz iteration to compute the zeroth power / orthogonalization of G. We opt to use a
quintic iteration whose coefficients are selected to maximize the slope at zero. For the purpose
of minimizing steps, it turns out to be empirically effective to keep increasing the slope at
zero even beyond the point where the iteration no longer converges all the way to one everywhere
on the interval. This iteration therefore does not produce UV^T but rather something like US'V^T
where S' is diagonal with S_{ii}' \sim Uniform(0.5, 1.5), which turns out not to hurt model
performance at all relative to UV^T, where USV^T = G is the SVD.
"""
assert len(G.shape) == 2
a, b, c = (3.4445, -4.7750, 2.0315)
X = G.bfloat16()
X /= (X.norm() + eps) # ensure top singular value <= 1
if G.size(0) > G.size(1):
X = X.T
for _ in range(steps):
A = X @ X.T
B = A @ X
X = a * X + b * B + c * A @ B
if G.size(0) > G.size(1):
X = X.T
return X
zeropower_backends = dict(svd=zeropower_via_svd, newtonschulz5=zeropower_via_newtonschulz5)
class Muon(torch.optim.Optimizer):
"""
Muon - MomentUm Orthogonalized by Newton-schulz
Muon internally runs standard SGD-momentum, and then performs an orthogonalization post-
processing step, in which each 2D parameter's update is replaced with the nearest orthogonal
matrix. To efficiently orthogonalize each update, we use a Newton-Schulz iteration, which has
the advantage that it can be stably run in bfloat16 on the GPU.
Some warnings:
- This optimizer assumes that all parameters passed in are 2D.
- It should not be used for the embedding layer, the final fully connected layer, or any {0,1}-D
parameters; those should all be optimized by a standard method (e.g., AdamW).
- To use it with 4D convolutional filters, it works well to just flatten their last 3 dimensions.
- We believe it is unlikely to work well for training with small batch size.
- We believe it may not work well for finetuning pretrained models, but we haven't tested this.
- We have not yet tried this optimizer for training scenarios larger than NanoGPT (124M).
Arguments:
lr: The learning rate used by the internal SGD.
momentum: The momentum used by the internal SGD.
nesterov: Whether to use Nesterov-style momentum in the internal SGD. (recommended)
backend: The chosen backend for the orthogonalization step. (recommended: 'newtonschulz5')
backend_steps: The number of iteration steps to use in the backend, if it is iterative.
"""
def __init__(self, params, lr=0.02, momentum=0.95, nesterov=True,
backend='newtonschulz5', backend_steps=5):
defaults = dict(lr=lr, momentum=momentum, nesterov=nesterov, backend=backend, backend_steps=backend_steps)
super().__init__(params, defaults)
def step(self):
for group in self.param_groups:
lr = group['lr']
momentum = group['momentum']
zeropower_backend = zeropower_backends[group['backend']]
# generate weight updates in distributed fashion
total_params = sum(p.numel() for p in group['params'])
updates_flat = torch.zeros(total_params, device='cuda', dtype=torch.bfloat16)
curr_idx = 0
for i, p in enumerate(group['params']):
# luckily this will perfectly distribute a transformer with multiple of 4 layers to 8 GPUs
if i % int(os.environ['WORLD_SIZE']) == int(os.environ['RANK']):
g = p.grad
assert g is not None
state = self.state[p]
if 'momentum_buffer' not in state:
state['momentum_buffer'] = torch.zeros_like(g)
buf = state['momentum_buffer']
buf.mul_(momentum).add_(g)
if group['nesterov']:
g = g.add(buf, alpha=momentum)
g = zeropower_backend(g, steps=group['backend_steps'])
g *= max(1, g.size(0)/g.size(1))**0.5
updates_flat[curr_idx:curr_idx+p.numel()] = g.flatten()
curr_idx += p.numel()
# sync updates across devices. we are not memory-constrained so can do this simple deserialization
dist.all_reduce(updates_flat, op=dist.ReduceOp.SUM)
# deserialize and apply updates
curr_idx = 0
for p in group['params']:
g = updates_flat[curr_idx:curr_idx+p.numel()].view_as(p.data).type_as(p.data)
p.data.add_(g, alpha=-lr)
curr_idx += p.numel()
# -----------------------------------------------------------------------------
# PyTorch nn.Module definitions for the GPT-2 model
class Rotary(torch.nn.Module):
def __init__(self, dim, base=10000):
super().__init__()
self.inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2).float() / dim))
self.seq_len_cached = None
self.cos_cached = None
self.sin_cached = None
def forward(self, x):
seq_len = x.shape[1]
if seq_len != self.seq_len_cached:
self.seq_len_cached = seq_len
t = torch.arange(seq_len, device=x.device).type_as(self.inv_freq)
freqs = torch.outer(t, self.inv_freq).to(x.device)
self.cos_cached = freqs.cos().bfloat16()
self.sin_cached = freqs.sin().bfloat16()
return self.cos_cached[None, :, None, :], self.sin_cached[None, :, None, :]
def apply_rotary_emb(x, cos, sin):
assert x.ndim == 4 # multihead attention
d = x.shape[3]//2
x1 = x[..., :d]
x2 = x[..., d:]
y1 = x1 * cos + x2 * sin
y2 = x1 * (-sin) + x2 * cos
return torch.cat([y1, y2], 3).type_as(x)
class CausalSelfAttention(nn.Module):
def __init__(self, config):
super().__init__()
self.n_head = config.n_head
self.n_embd = config.n_embd
self.head_dim = self.n_embd // self.n_head
assert self.n_embd % self.n_head == 0
self.c_q = nn.Linear(self.n_embd, self.n_embd, bias=False)
self.c_k = nn.Linear(self.n_embd, self.n_embd, bias=False)
self.c_v = nn.Linear(self.n_embd, self.n_embd, bias=False)
# output projection
self.c_proj = nn.Linear(self.n_embd, self.n_embd, bias=False)
self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977
self.rotary = Rotary(self.head_dim)
def forward(self, x):
B, T, C = x.size() # batch size, sequence length, embedding dimensionality (n_embd)
q = self.c_q(x).view(B, T, self.n_head, self.head_dim)
k = self.c_k(x).view(B, T, self.n_head, self.head_dim)
v = self.c_v(x).view(B, T, self.n_head, self.head_dim)
cos, sin = self.rotary(q)
q, k = F.rms_norm(q, (q.size(-1),)), F.rms_norm(k, (k.size(-1),)) # QK norm suggested by @Grad62304977
q, k = apply_rotary_emb(q, cos, sin), apply_rotary_emb(k, cos, sin)
y = F.scaled_dot_product_attention(q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2), is_causal=True)
y = y.transpose(1, 2).contiguous().view_as(x) # re-assemble all head outputs side by side
y = self.c_proj(y)
return y
class MLP(nn.Module):
def __init__(self, config):
super().__init__()
self.c_fc = nn.Linear(config.n_embd, 4 * config.n_embd, bias=False)
self.c_proj = nn.Linear(4 * config.n_embd, config.n_embd, bias=False)
self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977
def forward(self, x):
x = self.c_fc(x)
x = F.relu(x).square() # https://arxiv.org/abs/2109.08668v2; ~1-2% better than GELU; suggested by @SKYLINEZ007 and @Grad62304977
x = self.c_proj(x)
return x
class Block(nn.Module):
def __init__(self, config):
super().__init__()
self.attn = CausalSelfAttention(config)
self.mlp = MLP(config)
def forward(self, x):
x = x + self.attn(F.rms_norm(x, (x.size(-1),)))
x = x + self.mlp(F.rms_norm(x, (x.size(-1),)))
return x
# -----------------------------------------------------------------------------
# The main GPT-2 model
@dataclass
class GPTConfig:
vocab_size : int = 50304
n_layer : int = 12
n_head : int = 6 # head dim 128 suggested by @Grad62304977
n_embd : int = 768
class GPT(nn.Module):
def __init__(self, config):
super().__init__()
self.config = config
self.transformer = nn.ModuleDict(dict(
wte = nn.Embedding(config.vocab_size, config.n_embd),
h = nn.ModuleList([Block(config) for _ in range(config.n_layer)]),
))
self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False)
self.lm_head.weight.data.zero_()
def forward(self, idx, targets=None, return_logits=True):
# forward the GPT model itself
x = self.transformer.wte(idx) # token embeddings of shape (b, t, n_embd)
x = F.rms_norm(x, (x.size(-1),))
for block in self.transformer.h:
x = block(x)
x = F.rms_norm(x, (x.size(-1),))
if targets is not None:
# if we are given some desired targets also calculate the loss
logits = self.lm_head(x)
logits = logits.float() # use tf32/fp32 for logits
loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1), ignore_index=-1)
else:
# inference-time mini-optimization: only forward the lm_head on the very last position
logits = self.lm_head(x[:, [-1], :]) # note: using list [-1] to preserve the time dim
logits = logits.float() # use tf32/fp32 for logits
loss = None
# there are performance reasons why not returning logits is prudent, if not needed
if not return_logits:
logits = None
return logits, loss
# -----------------------------------------------------------------------------
# Our own simple Distributed Data Loader
def _peek_data_shard(filename):
# only reads the header, returns header data
with open(filename, "rb") as f:
# first read the header, which is 256 int32 integers (4 bytes each)
header = np.frombuffer(f.read(256*4), dtype=np.int32)
if header[0] != 20240520:
print("ERROR: magic number mismatch in the data .bin file!")
print("---> HINT: Are you passing in a correct file with --input_bin?")
print("---> HINT: Dataset encoding changed recently, re-run data prepro or refer again to README")
print("---> HINT: For example re-run: `python dev/data/tinyshakespeare.py`, then re-try")
exit(1)
assert header[1] == 1, "unsupported version"
ntok = header[2] # number of tokens (claimed)
return ntok # for now just return the number of tokens
def _load_data_shard(filename):
with open(filename, "rb") as f:
# first read the header, which is 256 int32 integers (4 bytes each)
header = np.frombuffer(f.read(256*4), dtype=np.int32)
assert header[0] == 20240520, "magic number mismatch in the data .bin file"
assert header[1] == 1, "unsupported version"
ntok = header[2] # number of tokens (claimed)
# the rest of it are tokens, stored as uint16
tokens = np.frombuffer(f.read(), dtype=np.uint16)
assert len(tokens) == ntok, "number of tokens read does not match header?"
return tokens
class DistributedDataLoader:
def __init__(self, filename_pattern, B, T, process_rank, num_processes):
self.process_rank = process_rank
self.num_processes = num_processes
self.B = B
self.T = T
# glob files that match the pattern
self.files = sorted(glob.glob(filename_pattern))
assert len(self.files) > 0, f"did not find any files that match the pattern {filename_pattern}"
# load and validate all data shards, count number of tokens in total
ntok_total = 0
for fname in self.files:
shard_ntok = _peek_data_shard(fname)
assert shard_ntok >= num_processes * B * T + 1
ntok_total += int(shard_ntok)
self.ntok_total = ntok_total
# kick things off
self.reset()
def reset(self):
self.current_shard = 0
self.current_position = self.process_rank * self.B * self.T
self.tokens = _load_data_shard(self.files[self.current_shard])
def advance(self): # advance to next data shard
self.current_shard = (self.current_shard + 1) % len(self.files)
self.current_position = self.process_rank * self.B * self.T
self.tokens = _load_data_shard(self.files[self.current_shard])
def next_batch(self):
B = self.B
T = self.T
buf = self.tokens[self.current_position : self.current_position+B*T+1]
buf = torch.tensor(buf.astype(np.int32), dtype=torch.long)
x = (buf[:-1]).view(B, T) # inputs
y = (buf[1:]).view(B, T) # targets
# advance current position and load next shard if necessary
self.current_position += B * T * self.num_processes
if self.current_position + (B * T * self.num_processes + 1) > len(self.tokens):
self.advance()
return x.cuda(), y.cuda()
# -----------------------------------------------------------------------------
# int main
@dataclass
class Hyperparameters:
# data hyperparams
input_bin : str = 'data/fineweb10B/fineweb_train_*.bin' # input .bin to train on
input_val_bin : str = 'data/fineweb10B/fineweb_val_*.bin' # input .bin to eval validation loss on
# optimization hyperparams
batch_size : int = 8*64 # batch size, in sequences, across all devices
device_batch_size : int = 64 # batch size, in sequences, per device
sequence_length : int = 1024 # sequence length, in tokens
num_iterations : int = 4578 # number of iterations to run
warmup_iters : int = 0
warmdown_iters : int = 1308 # number of iterations of linear warmup/warmdown for triangular or trapezoidal schedule
weight_decay : float = 0
# evaluation and logging hyperparams
val_loss_every : int = 125 # every how many steps to evaluate val loss? 0 for only at the end
val_tokens : int = 10485760 # how many tokens of validation data? it's important to keep this fixed for consistent comparisons
save_every : int = 0 # every how many steps to save the checkpoint? 0 for only at the end
args = Hyperparameters()
# set up DDP (distributed data parallel). torchrun sets this env variable
assert torch.cuda.is_available()
dist.init_process_group(backend='nccl')
ddp_rank = int(os.environ['RANK'])
ddp_local_rank = int(os.environ['LOCAL_RANK'])
ddp_world_size = int(os.environ['WORLD_SIZE'])
device = f'cuda:{ddp_local_rank}'
torch.cuda.set_device(device)
print(f"using device: {device}")
master_process = (ddp_rank == 0) # this process will do logging, checkpointing etc.
# convenience variables
B, T = args.device_batch_size, args.sequence_length
# calculate the number of steps to take in the val loop.
assert args.val_tokens % (B * T * ddp_world_size) == 0
val_steps = args.val_tokens // (B * T * ddp_world_size)
# calculate the steps of gradient accumulation required to attain the desired global batch size.
assert args.batch_size % (B * ddp_world_size) == 0
train_accumulation_steps = args.batch_size // (B * ddp_world_size)
# load tokens
train_loader = DistributedDataLoader(args.input_bin, B, T, ddp_rank, ddp_world_size)
val_loader = DistributedDataLoader(args.input_val_bin, B, T, ddp_rank, ddp_world_size)
if master_process:
print(f"Training DataLoader: total number of tokens: {train_loader.ntok_total} across {len(train_loader.files)} files")
print(f"Validation DataLoader: total number of tokens: {val_loader.ntok_total} across {len(val_loader.files)} files")
x, y = train_loader.next_batch()
# there are only 50257 unique GPT-2 tokens; we extend to nearest multiple of 128 for efficiency. suggested to me by @Grad62304977.
# this originates from Karpathy's experiments.
num_vocab = 50304
model = GPT(GPTConfig(vocab_size=num_vocab, n_layer=12, n_head=6, n_embd=768))
model = model.cuda()
if hasattr(config, "coordinate_descent_tuning"):
config.coordinate_descent_tuning = True # suggested by @Chillee
model = torch.compile(model)
# here we wrap model into DDP container
model = DDP(model, device_ids=[ddp_local_rank])
raw_model = model.module # always contains the "raw" unwrapped model
ctx = torch.amp.autocast(device_type='cuda', dtype=torch.bfloat16)
# CUDNN attention is ~4ms faster than Flash, but doesn't get selected by default in PyTorch 2.5.1
from torch.backends.cuda import enable_cudnn_sdp, enable_flash_sdp, enable_math_sdp, enable_mem_efficient_sdp
enable_cudnn_sdp(True)
enable_flash_sdp(False)
enable_mem_efficient_sdp(False)
enable_math_sdp(False)
# init the optimizer(s)
optimizer1 = torch.optim.Adam([raw_model.transformer.wte.weight], lr=0.3, betas=(0.9, 0.95), fused=True)
optimizer2 = torch.optim.Adam([raw_model.lm_head.weight], lr=0.002, betas=(0.9, 0.95), fused=True)
optimizer3 = Muon(raw_model.transformer.h.parameters(), lr=0.02, momentum=0.95)
optimizers = [optimizer1, optimizer2, optimizer3]
# learning rate decay scheduler (linear warmup and warmdown)
def get_lr(it):
assert it <= args.num_iterations
# 1) linear warmup for warmup_iters steps
if it < args.warmup_iters:
return (it+1) / args.warmup_iters
# 2) constant lr for a while
elif it < args.num_iterations - args.warmdown_iters:
return 1.0
# 3) linear warmdown
else:
decay_ratio = (args.num_iterations - it) / args.warmdown_iters
return decay_ratio
schedulers = [torch.optim.lr_scheduler.LambdaLR(opt, get_lr) for opt in optimizers]
# begin logging
if master_process:
run_id = str(uuid.uuid4())
logdir = 'logs/%s/' % run_id
os.makedirs(logdir, exist_ok=True)
logfile = 'logs/%s.txt' % run_id
# create the log file
with open(logfile, "w") as f:
# begin the log by printing this file (the Python code)
f.write('='*100 + '\n')
f.write(code)
f.write('='*100 + '\n')
# log information about the hardware/software environment this is running on
# and print the full `nvidia-smi` to file
f.write(f"Running pytorch {torch.version.__version__} compiled for CUDA {torch.version.cuda}\nnvidia-smi:\n")
import subprocess
result = subprocess.run(['nvidia-smi'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
f.write(f'{result.stdout}\n')
f.write('='*100 + '\n')
training_time_ms = 0
# start the clock
torch.cuda.synchronize()
t0 = time.time()
# begin training
train_loader.reset()
for step in range(args.num_iterations + 1):
last_step = (step == args.num_iterations)
# This effectively ignores timing first 10 steps, which are slower for weird reasons.
# Alternately, and slightly more correctly in terms of benchmarking, we could do 10
# steps with dummy data first, and then re-initialize the model and reset the loader.
if step == 10:
training_time_ms = 0
t0 = time.time()
timed_steps = float('nan') if step <= 11 else (step - 10) + 1 # <= 11 to avoid bug in val
# once in a while evaluate the validation dataset
if (last_step or (args.val_loss_every > 0 and step % args.val_loss_every == 0)):
# stop the clock
torch.cuda.synchronize()
training_time_ms += 1000 * (time.time() - t0)
# run validation batches
model.eval()
val_loader.reset()
val_loss = 0.0
for _ in range(val_steps):
x_val, y_val = val_loader.next_batch()
with ctx: # of course, we'd like to use no_grad() here too, but that creates a torch.compile error for some reason
_, loss = model(x_val, y_val, return_logits=False)
val_loss += loss.detach()
del loss
dist.all_reduce(val_loss, op=dist.ReduceOp.AVG)
val_loss /= val_steps
# log val loss to console and to logfile
if master_process:
print(f'step:{step}/{args.num_iterations} val_loss:{val_loss:.4f} train_time:{training_time_ms:.0f}ms step_avg:{training_time_ms/(timed_steps-1):.2f}ms')
with open(logfile, "a") as f:
f.write(f'step:{step}/{args.num_iterations} val_loss:{val_loss:.4f} train_time:{training_time_ms:.0f}ms step_avg:{training_time_ms/(timed_steps-1):.2f}ms\n')
# start the clock again
torch.cuda.synchronize()
t0 = time.time()
if master_process and (last_step or (args.save_every > 0 and step % args.save_every == 0)):
# stop the clock
torch.cuda.synchronize()
training_time_ms += 1000 * (time.time() - t0)
# save the state of the training process
log = dict(step=step, code=code, model=raw_model.state_dict(), optimizers=[opt.state_dict() for opt in optimizers])
torch.save(log, 'logs/%s/state_step%06d.pt' % (run_id, step))
# start the clock again
torch.cuda.synchronize()
t0 = time.time()
# bit confusing: we want to make sure to eval on 0th iteration
# but also after the very last iteration. so we loop for step <= num_iterations
# instead of just < num_iterations (one extra due to <=), only to do
# the validation/sampling one last time, and then we break right here as we're done.
if last_step:
break
# --------------- TRAINING SECTION BEGIN -----------------
model.train()
for i in range(1, train_accumulation_steps+1):
# forward pass
with ctx:
_, loss = model(x, y, return_logits=False)
train_loss = loss.detach()
# advance the dataset for the next batch
x, y = train_loader.next_batch()
# backward pass
if i < train_accumulation_steps:
with model.no_sync(): # there's no need to sync gradients every accumulation step
loss.backward()
else:
loss.backward() # just sync on the last step
for p in model.parameters():
p.grad /= train_accumulation_steps
# step the optimizers and schedulers
for opt, sched in zip(optimizers, schedulers):
opt.step()
sched.step()
# null the gradients
model.zero_grad(set_to_none=True)
# --------------- TRAINING SECTION END -------------------
# everything that follows now is just diagnostics, prints, logging, etc.
#dist.all_reduce(train_loss, op=dist.ReduceOp.AVG) # all-reducing the training loss would be more correct in terms of logging, but slower
if master_process:
approx_time = training_time_ms + 1000 * (time.time() - t0)
print(f"step:{step+1}/{args.num_iterations} train_loss:{train_loss.item():.4f} train_time:{approx_time:.0f}ms step_avg:{approx_time/timed_steps:.2f}ms")
with open(logfile, "a") as f:
f.write(f"step:{step+1}/{args.num_iterations} train_loss:{train_loss.item():.4f} train_time:{approx_time:.0f}ms step_avg:{approx_time/timed_steps:.2f}ms\n")
if master_process:
print(f"peak memory consumption: {torch.cuda.max_memory_allocated() // 1024 // 1024} MiB")
# -------------------------------------------------------------------------
# clean up nice
dist.destroy_process_group()
====================================================================================================
Running pytorch 2.5.1+cu124 compiled for CUDA 12.4
nvidia-smi:
Sun Nov 3 19:58:03 2024
+-----------------------------------------------------------------------------------------+
| NVIDIA-SMI 555.42.06 Driver Version: 555.42.06 CUDA Version: 12.5 |
|-----------------------------------------+------------------------+----------------------+
| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |
| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |
| | | MIG M. |
|=========================================+========================+======================|
| 0 NVIDIA H100 80GB HBM3 Off | 00000000:18:00.0 Off | 0 |
| N/A 34C P0 141W / 700W | 5304MiB / 81559MiB | 4% Default |
| | | Disabled |
+-----------------------------------------+------------------------+----------------------+
| 1 NVIDIA H100 80GB HBM3 Off | 00000000:2A:00.0 Off | 0 |
| N/A 35C P0 129W / 700W | 5352MiB / 81559MiB | 3% Default |
| | | Disabled |
+-----------------------------------------+------------------------+----------------------+
| 2 NVIDIA H100 80GB HBM3 Off | 00000000:3A:00.0 Off | 0 |
| N/A 36C P0 125W / 700W | 5352MiB / 81559MiB | 2% Default |
| | | Disabled |
+-----------------------------------------+------------------------+----------------------+
| 3 NVIDIA H100 80GB HBM3 Off | 00000000:5D:00.0 Off | 0 |
| N/A 32C P0 129W / 700W | 5352MiB / 81559MiB | 3% Default |
| | | Disabled |
+-----------------------------------------+------------------------+----------------------+
| 4 NVIDIA H100 80GB HBM3 Off | 00000000:9A:00.0 Off | 0 |
| N/A 34C P0 141W / 700W | 5352MiB / 81559MiB | 6% Default |
| | | Disabled |
+-----------------------------------------+------------------------+----------------------+
| 5 NVIDIA H100 80GB HBM3 Off | 00000000:AB:00.0 Off | 0 |
| N/A 37C P0 142W / 700W | 5352MiB / 81559MiB | 0% Default |
| | | Disabled |
+-----------------------------------------+------------------------+----------------------+
| 6 NVIDIA H100 80GB HBM3 Off | 00000000:BA:00.0 Off | 0 |
| N/A 36C P0 142W / 700W | 5352MiB / 81559MiB | 2% Default |
| | | Disabled |
+-----------------------------------------+------------------------+----------------------+
| 7 NVIDIA H100 80GB HBM3 Off | 00000000:DB:00.0 Off | 0 |
| N/A 35C P0 147W / 700W | 5112MiB / 81559MiB | 0% Default |
| | | Disabled |
+-----------------------------------------+------------------------+----------------------+
+-----------------------------------------------------------------------------------------+
| Processes: |
| GPU GI CI PID Type Process name GPU Memory |
| ID ID Usage |
|=========================================================================================|
| 0 N/A N/A 67806 C /usr/bin/python3 0MiB |
| 1 N/A N/A 67807 C /usr/bin/python3 0MiB |
| 2 N/A N/A 67808 C /usr/bin/python3 0MiB |
| 3 N/A N/A 67809 C /usr/bin/python3 0MiB |
| 4 N/A N/A 67810 C /usr/bin/python3 0MiB |
| 5 N/A N/A 67811 C /usr/bin/python3 0MiB |
| 6 N/A N/A 67812 C /usr/bin/python3 0MiB |
| 7 N/A N/A 67813 C /usr/bin/python3 0MiB |
+-----------------------------------------------------------------------------------------+
====================================================================================================
step:0/4578 val_loss:10.8258 train_time:471ms step_avg:nanms
step:1/4578 train_loss:10.8258 train_time:3347ms step_avg:nanms
step:2/4578 train_loss:10.4264 train_time:3434ms step_avg:nanms
step:3/4578 train_loss:9.9036 train_time:3568ms step_avg:nanms
step:4/4578 train_loss:8.9782 train_time:3704ms step_avg:nanms
step:5/4578 train_loss:8.0412 train_time:3842ms step_avg:nanms
step:6/4578 train_loss:7.5182 train_time:3988ms step_avg:nanms
step:7/4578 train_loss:7.0184 train_time:4118ms step_avg:nanms
step:8/4578 train_loss:7.1936 train_time:4258ms step_avg:nanms
step:9/4578 train_loss:6.9028 train_time:4403ms step_avg:nanms
step:10/4578 train_loss:6.7643 train_time:4554ms step_avg:nanms
step:11/4578 train_loss:6.7341 train_time:84ms step_avg:nanms
step:12/4578 train_loss:6.6891 train_time:222ms step_avg:nanms
step:13/4578 train_loss:6.5404 train_time:362ms step_avg:120.70ms
step:14/4578 train_loss:6.5144 train_time:499ms step_avg:124.87ms
step:15/4578 train_loss:6.4994 train_time:647ms step_avg:129.35ms
step:16/4578 train_loss:6.4542 train_time:784ms step_avg:130.65ms
step:17/4578 train_loss:6.4720 train_time:929ms step_avg:132.68ms
step:18/4578 train_loss:6.5019 train_time:1077ms step_avg:134.57ms
step:19/4578 train_loss:6.3335 train_time:1205ms step_avg:133.88ms
step:20/4578 train_loss:6.3792 train_time:1346ms step_avg:134.63ms
step:21/4578 train_loss:6.0579 train_time:1484ms step_avg:134.95ms
step:22/4578 train_loss:6.3952 train_time:1626ms step_avg:135.52ms
step:23/4578 train_loss:6.6283 train_time:1769ms step_avg:136.07ms
step:24/4578 train_loss:6.2806 train_time:1908ms step_avg:136.32ms
step:25/4578 train_loss:6.4435 train_time:2048ms step_avg:136.55ms
step:26/4578 train_loss:6.1504 train_time:2189ms step_avg:136.82ms
step:27/4578 train_loss:6.0603 train_time:2331ms step_avg:137.13ms
step:28/4578 train_loss:6.2238 train_time:2467ms step_avg:137.07ms
step:29/4578 train_loss:5.8878 train_time:2609ms step_avg:137.33ms
step:30/4578 train_loss:6.1468 train_time:2749ms step_avg:137.47ms
step:31/4578 train_loss:5.9902 train_time:2891ms step_avg:137.65ms
step:32/4578 train_loss:5.9577 train_time:3032ms step_avg:137.81ms
step:33/4578 train_loss:5.8020 train_time:3174ms step_avg:137.98ms
step:34/4578 train_loss:6.0915 train_time:3310ms step_avg:137.91ms
step:35/4578 train_loss:6.0166 train_time:3450ms step_avg:137.99ms
step:36/4578 train_loss:6.1607 train_time:3591ms step_avg:138.13ms
step:37/4578 train_loss:6.0773 train_time:3736ms step_avg:138.37ms
step:38/4578 train_loss:5.9808 train_time:3874ms step_avg:138.36ms
step:39/4578 train_loss:5.8647 train_time:4012ms step_avg:138.34ms
step:40/4578 train_loss:5.8961 train_time:4151ms step_avg:138.35ms
step:41/4578 train_loss:5.8117 train_time:4291ms step_avg:138.42ms
step:42/4578 train_loss:5.8123 train_time:4431ms step_avg:138.47ms
step:43/4578 train_loss:5.7165 train_time:4571ms step_avg:138.52ms
step:44/4578 train_loss:5.7897 train_time:4710ms step_avg:138.54ms
step:45/4578 train_loss:5.7751 train_time:4851ms step_avg:138.59ms
step:46/4578 train_loss:5.9386 train_time:4992ms step_avg:138.66ms
step:47/4578 train_loss:5.7310 train_time:5130ms step_avg:138.65ms
step:48/4578 train_loss:5.5976 train_time:5273ms step_avg:138.77ms
step:49/4578 train_loss:5.8028 train_time:5410ms step_avg:138.72ms
step:50/4578 train_loss:5.6744 train_time:5550ms step_avg:138.75ms
step:51/4578 train_loss:5.7978 train_time:5693ms step_avg:138.85ms
step:52/4578 train_loss:5.6750 train_time:5830ms step_avg:138.81ms
step:53/4578 train_loss:5.5255 train_time:5971ms step_avg:138.87ms
step:54/4578 train_loss:5.6645 train_time:6111ms step_avg:138.88ms
step:55/4578 train_loss:5.5355 train_time:6252ms step_avg:138.94ms
step:56/4578 train_loss:5.8584 train_time:6395ms step_avg:139.01ms
step:57/4578 train_loss:5.5132 train_time:6530ms step_avg:138.94ms
step:58/4578 train_loss:5.4048 train_time:6671ms step_avg:138.98ms
step:59/4578 train_loss:5.5517 train_time:6811ms step_avg:139.00ms
step:60/4578 train_loss:5.4975 train_time:6951ms step_avg:139.03ms
step:61/4578 train_loss:5.5916 train_time:7091ms step_avg:139.04ms
step:62/4578 train_loss:5.3815 train_time:7233ms step_avg:139.09ms
step:63/4578 train_loss:5.4760 train_time:7376ms step_avg:139.17ms
step:64/4578 train_loss:5.4380 train_time:7514ms step_avg:139.14ms
step:65/4578 train_loss:5.2123 train_time:7654ms step_avg:139.16ms
step:66/4578 train_loss:5.2547 train_time:7792ms step_avg:139.15ms
step:67/4578 train_loss:5.4417 train_time:7935ms step_avg:139.21ms
step:68/4578 train_loss:5.3031 train_time:8072ms step_avg:139.17ms
step:69/4578 train_loss:5.5388 train_time:8211ms step_avg:139.18ms
step:70/4578 train_loss:5.1888 train_time:8352ms step_avg:139.19ms
step:71/4578 train_loss:5.2521 train_time:8492ms step_avg:139.21ms
step:72/4578 train_loss:5.4173 train_time:8632ms step_avg:139.22ms
step:73/4578 train_loss:5.3452 train_time:8772ms step_avg:139.24ms
step:74/4578 train_loss:5.2481 train_time:8912ms step_avg:139.25ms
step:75/4578 train_loss:5.3636 train_time:9051ms step_avg:139.25ms
step:76/4578 train_loss:5.3295 train_time:9191ms step_avg:139.26ms
step:77/4578 train_loss:5.2757 train_time:9331ms step_avg:139.27ms
step:78/4578 train_loss:5.3746 train_time:9472ms step_avg:139.29ms
step:79/4578 train_loss:5.4590 train_time:9612ms step_avg:139.30ms
step:80/4578 train_loss:5.2231 train_time:9752ms step_avg:139.32ms
step:81/4578 train_loss:5.3148 train_time:9892ms step_avg:139.33ms
step:82/4578 train_loss:5.0748 train_time:10032ms step_avg:139.33ms
step:83/4578 train_loss:5.2599 train_time:10172ms step_avg:139.34ms
step:84/4578 train_loss:5.2215 train_time:10312ms step_avg:139.35ms
step:85/4578 train_loss:5.1973 train_time:10455ms step_avg:139.40ms
step:86/4578 train_loss:5.0714 train_time:10600ms step_avg:139.48ms
step:87/4578 train_loss:5.2601 train_time:10732ms step_avg:139.37ms
step:88/4578 train_loss:5.1616 train_time:10872ms step_avg:139.38ms
step:89/4578 train_loss:5.2234 train_time:11013ms step_avg:139.41ms
step:90/4578 train_loss:5.1953 train_time:11152ms step_avg:139.40ms
step:91/4578 train_loss:5.1039 train_time:11295ms step_avg:139.44ms
step:92/4578 train_loss:5.1005 train_time:11432ms step_avg:139.41ms
step:93/4578 train_loss:5.2141 train_time:11572ms step_avg:139.42ms
step:94/4578 train_loss:5.0542 train_time:11712ms step_avg:139.42ms
step:95/4578 train_loss:5.0576 train_time:11851ms step_avg:139.42ms
step:96/4578 train_loss:5.0957 train_time:11991ms step_avg:139.43ms
step:97/4578 train_loss:5.0051 train_time:12132ms step_avg:139.44ms
step:98/4578 train_loss:5.0792 train_time:12270ms step_avg:139.43ms
step:99/4578 train_loss:5.0092 train_time:12412ms step_avg:139.46ms
step:100/4578 train_loss:5.1197 train_time:12551ms step_avg:139.45ms
step:101/4578 train_loss:5.0968 train_time:12691ms step_avg:139.46ms
step:102/4578 train_loss:4.9741 train_time:12831ms step_avg:139.47ms
step:103/4578 train_loss:5.0998 train_time:12971ms step_avg:139.48ms
step:104/4578 train_loss:5.0433 train_time:13111ms step_avg:139.48ms
step:105/4578 train_loss:4.9137 train_time:13250ms step_avg:139.47ms
step:106/4578 train_loss:4.9778 train_time:13389ms step_avg:139.47ms
step:107/4578 train_loss:5.1866 train_time:13534ms step_avg:139.53ms
step:108/4578 train_loss:4.9637 train_time:13672ms step_avg:139.51ms
step:109/4578 train_loss:4.7614 train_time:13808ms step_avg:139.48ms
step:110/4578 train_loss:4.9330 train_time:13951ms step_avg:139.51ms
step:111/4578 train_loss:4.9214 train_time:14090ms step_avg:139.50ms
step:112/4578 train_loss:4.8841 train_time:14229ms step_avg:139.50ms
step:113/4578 train_loss:5.0065 train_time:14369ms step_avg:139.51ms
step:114/4578 train_loss:4.9086 train_time:14509ms step_avg:139.51ms
step:115/4578 train_loss:4.7789 train_time:14650ms step_avg:139.52ms
step:116/4578 train_loss:4.9368 train_time:14790ms step_avg:139.53ms
step:117/4578 train_loss:4.8412 train_time:14930ms step_avg:139.53ms
step:118/4578 train_loss:4.7938 train_time:15071ms step_avg:139.55ms
step:119/4578 train_loss:4.9543 train_time:15211ms step_avg:139.55ms
step:120/4578 train_loss:4.8933 train_time:15351ms step_avg:139.55ms
step:121/4578 train_loss:4.8092 train_time:15492ms step_avg:139.56ms
step:122/4578 train_loss:4.7206 train_time:15638ms step_avg:139.62ms
step:123/4578 train_loss:4.8468 train_time:15771ms step_avg:139.57ms
step:124/4578 train_loss:4.7070 train_time:15910ms step_avg:139.56ms
step:125/4578 train_loss:5.0148 train_time:16050ms step_avg:139.57ms
step:125/4578 val_loss:4.8310 train_time:16105ms step_avg:140.04ms
step:126/4578 train_loss:4.8736 train_time:16199ms step_avg:139.65ms
step:127/4578 train_loss:4.8294 train_time:16348ms step_avg:139.72ms
step:128/4578 train_loss:4.8768 train_time:16489ms step_avg:139.73ms
step:129/4578 train_loss:4.7598 train_time:16639ms step_avg:139.82ms
step:130/4578 train_loss:5.0642 train_time:16765ms step_avg:139.71ms
step:131/4578 train_loss:4.8122 train_time:16909ms step_avg:139.74ms
step:132/4578 train_loss:4.8106 train_time:17051ms step_avg:139.76ms
step:133/4578 train_loss:4.7769 train_time:17185ms step_avg:139.71ms
step:134/4578 train_loss:4.8157 train_time:17330ms step_avg:139.76ms
step:135/4578 train_loss:4.7057 train_time:17470ms step_avg:139.76ms
step:136/4578 train_loss:4.8151 train_time:17612ms step_avg:139.78ms
step:137/4578 train_loss:4.6081 train_time:17750ms step_avg:139.76ms
step:138/4578 train_loss:4.7709 train_time:17890ms step_avg:139.76ms
step:139/4578 train_loss:4.7057 train_time:18029ms step_avg:139.76ms
step:140/4578 train_loss:4.7442 train_time:18178ms step_avg:139.83ms
step:141/4578 train_loss:4.8084 train_time:18320ms step_avg:139.85ms
step:142/4578 train_loss:4.6876 train_time:18453ms step_avg:139.80ms
step:143/4578 train_loss:4.7408 train_time:18597ms step_avg:139.83ms
step:144/4578 train_loss:4.6072 train_time:18735ms step_avg:139.81ms
step:145/4578 train_loss:4.7237 train_time:18876ms step_avg:139.82ms
step:146/4578 train_loss:4.6742 train_time:19014ms step_avg:139.81ms
step:147/4578 train_loss:4.5621 train_time:19157ms step_avg:139.83ms
step:148/4578 train_loss:4.7062 train_time:19297ms step_avg:139.83ms
step:149/4578 train_loss:4.7123 train_time:19443ms step_avg:139.87ms
step:150/4578 train_loss:4.7207 train_time:19583ms step_avg:139.88ms
step:151/4578 train_loss:4.7784 train_time:19722ms step_avg:139.87ms
step:152/4578 train_loss:4.6449 train_time:19862ms step_avg:139.87ms
step:153/4578 train_loss:4.6496 train_time:20001ms step_avg:139.87ms
step:154/4578 train_loss:4.7309 train_time:20141ms step_avg:139.87ms
step:155/4578 train_loss:4.6879 train_time:20286ms step_avg:139.90ms
step:156/4578 train_loss:4.6220 train_time:20427ms step_avg:139.91ms
step:157/4578 train_loss:4.6772 train_time:20573ms step_avg:139.95ms
step:158/4578 train_loss:4.7779 train_time:20710ms step_avg:139.93ms
step:159/4578 train_loss:4.5850 train_time:20849ms step_avg:139.92ms
step:160/4578 train_loss:4.6382 train_time:20988ms step_avg:139.92ms
step:161/4578 train_loss:4.4713 train_time:21128ms step_avg:139.92ms
step:162/4578 train_loss:4.6548 train_time:21269ms step_avg:139.93ms
step:163/4578 train_loss:4.6779 train_time:21413ms step_avg:139.95ms
step:164/4578 train_loss:4.6777 train_time:21550ms step_avg:139.93ms
step:165/4578 train_loss:4.4936 train_time:21690ms step_avg:139.93ms
step:166/4578 train_loss:4.6014 train_time:21830ms step_avg:139.93ms
step:167/4578 train_loss:4.7317 train_time:21969ms step_avg:139.93ms
step:168/4578 train_loss:4.5137 train_time:22109ms step_avg:139.93ms
step:169/4578 train_loss:4.5967 train_time:22250ms step_avg:139.94ms
step:170/4578 train_loss:4.4733 train_time:22393ms step_avg:139.95ms
step:171/4578 train_loss:4.3776 train_time:22531ms step_avg:139.94ms
step:172/4578 train_loss:4.5249 train_time:22671ms step_avg:139.94ms
step:173/4578 train_loss:4.5022 train_time:22810ms step_avg:139.94ms
step:174/4578 train_loss:4.5522 train_time:22950ms step_avg:139.94ms
step:175/4578 train_loss:4.7197 train_time:23091ms step_avg:139.94ms
step:176/4578 train_loss:4.5608 train_time:23233ms step_avg:139.96ms
step:177/4578 train_loss:4.4025 train_time:23371ms step_avg:139.95ms
step:178/4578 train_loss:4.3816 train_time:23514ms step_avg:139.97ms
step:179/4578 train_loss:4.4595 train_time:23650ms step_avg:139.94ms
step:180/4578 train_loss:4.4469 train_time:23791ms step_avg:139.94ms
step:181/4578 train_loss:4.4280 train_time:23932ms step_avg:139.95ms
step:182/4578 train_loss:4.5780 train_time:24079ms step_avg:140.00ms
step:183/4578 train_loss:4.4443 train_time:24210ms step_avg:139.94ms
step:184/4578 train_loss:4.4247 train_time:24350ms step_avg:139.94ms
step:185/4578 train_loss:4.4099 train_time:24493ms step_avg:139.96ms
step:186/4578 train_loss:4.5159 train_time:24633ms step_avg:139.96ms
step:187/4578 train_loss:4.4444 train_time:24770ms step_avg:139.94ms
step:188/4578 train_loss:4.5794 train_time:24913ms step_avg:139.96ms
step:189/4578 train_loss:4.4448 train_time:25252ms step_avg:141.07ms
step:190/4578 train_loss:4.3807 train_time:25551ms step_avg:141.95ms
step:191/4578 train_loss:4.5040 train_time:25685ms step_avg:141.91ms
step:192/4578 train_loss:4.3494 train_time:25823ms step_avg:141.89ms
step:193/4578 train_loss:4.2896 train_time:25962ms step_avg:141.87ms
step:194/4578 train_loss:4.5209 train_time:26101ms step_avg:141.85ms
step:195/4578 train_loss:4.4334 train_time:26239ms step_avg:141.83ms
step:196/4578 train_loss:4.6294 train_time:26379ms step_avg:141.82ms
step:197/4578 train_loss:4.4681 train_time:26524ms step_avg:141.84ms
step:198/4578 train_loss:4.3227 train_time:26668ms step_avg:141.85ms
step:199/4578 train_loss:4.4152 train_time:26808ms step_avg:141.84ms
step:200/4578 train_loss:4.2852 train_time:26946ms step_avg:141.82ms
step:201/4578 train_loss:4.3835 train_time:27086ms step_avg:141.81ms
step:202/4578 train_loss:4.2677 train_time:27226ms step_avg:141.80ms
step:203/4578 train_loss:4.4988 train_time:27369ms step_avg:141.81ms
step:204/4578 train_loss:4.3408 train_time:27511ms step_avg:141.81ms
step:205/4578 train_loss:4.4276 train_time:27649ms step_avg:141.79ms
step:206/4578 train_loss:4.5047 train_time:27790ms step_avg:141.79ms
step:207/4578 train_loss:4.1994 train_time:27930ms step_avg:141.77ms
step:208/4578 train_loss:4.3388 train_time:28072ms step_avg:141.78ms
step:209/4578 train_loss:4.3274 train_time:28215ms step_avg:141.78ms
step:210/4578 train_loss:4.4841 train_time:28349ms step_avg:141.75ms
step:211/4578 train_loss:4.4320 train_time:28490ms step_avg:141.74ms
step:212/4578 train_loss:4.3024 train_time:28632ms step_avg:141.74ms
step:213/4578 train_loss:4.3634 train_time:28769ms step_avg:141.72ms
step:214/4578 train_loss:4.2713 train_time:28910ms step_avg:141.71ms
step:215/4578 train_loss:4.3544 train_time:29055ms step_avg:141.73ms
step:216/4578 train_loss:4.1654 train_time:29190ms step_avg:141.70ms
step:217/4578 train_loss:4.2517 train_time:29332ms step_avg:141.70ms
step:218/4578 train_loss:4.2490 train_time:29470ms step_avg:141.68ms
step:219/4578 train_loss:4.3053 train_time:29611ms step_avg:141.68ms
step:220/4578 train_loss:4.3086 train_time:29750ms step_avg:141.67ms
step:221/4578 train_loss:4.3108 train_time:29896ms step_avg:141.69ms
step:222/4578 train_loss:4.3338 train_time:30029ms step_avg:141.65ms
step:223/4578 train_loss:4.2603 train_time:30169ms step_avg:141.64ms
step:224/4578 train_loss:4.2229 train_time:30311ms step_avg:141.64ms
step:225/4578 train_loss:4.5038 train_time:30449ms step_avg:141.62ms
step:226/4578 train_loss:4.1279 train_time:30590ms step_avg:141.62ms
step:227/4578 train_loss:4.2145 train_time:30729ms step_avg:141.61ms
step:228/4578 train_loss:4.2100 train_time:30869ms step_avg:141.60ms
step:229/4578 train_loss:4.3583 train_time:31009ms step_avg:141.59ms
step:230/4578 train_loss:4.1440 train_time:31147ms step_avg:141.58ms
step:231/4578 train_loss:4.2730 train_time:31288ms step_avg:141.58ms
step:232/4578 train_loss:4.1320 train_time:31428ms step_avg:141.57ms
step:233/4578 train_loss:4.1819 train_time:31569ms step_avg:141.57ms
step:234/4578 train_loss:4.3241 train_time:31715ms step_avg:141.59ms
step:235/4578 train_loss:4.2267 train_time:31849ms step_avg:141.55ms
step:236/4578 train_loss:4.1263 train_time:31989ms step_avg:141.54ms
step:237/4578 train_loss:4.2925 train_time:32128ms step_avg:141.53ms
step:238/4578 train_loss:4.2899 train_time:32269ms step_avg:141.53ms
step:239/4578 train_loss:4.1580 train_time:32409ms step_avg:141.53ms
step:240/4578 train_loss:4.3004 train_time:32548ms step_avg:141.51ms
step:241/4578 train_loss:4.3190 train_time:32690ms step_avg:141.52ms
step:242/4578 train_loss:4.1730 train_time:32828ms step_avg:141.50ms
step:243/4578 train_loss:4.3674 train_time:32969ms step_avg:141.50ms
step:244/4578 train_loss:4.2254 train_time:33108ms step_avg:141.49ms
step:245/4578 train_loss:4.2740 train_time:33248ms step_avg:141.48ms
step:246/4578 train_loss:4.3434 train_time:33390ms step_avg:141.48ms
step:247/4578 train_loss:4.2680 train_time:33529ms step_avg:141.47ms
step:248/4578 train_loss:4.2060 train_time:33669ms step_avg:141.47ms
step:249/4578 train_loss:4.3266 train_time:33809ms step_avg:141.46ms
step:250/4578 train_loss:4.1242 train_time:33949ms step_avg:141.45ms
step:250/4578 val_loss:4.2102 train_time:34005ms step_avg:141.69ms
step:251/4578 train_loss:4.1648 train_time:34100ms step_avg:141.49ms
step:252/4578 train_loss:4.2786 train_time:34246ms step_avg:141.51ms
step:253/4578 train_loss:4.3323 train_time:34385ms step_avg:141.50ms
step:254/4578 train_loss:4.1373 train_time:34524ms step_avg:141.49ms
step:255/4578 train_loss:4.0899 train_time:34662ms step_avg:141.48ms
step:256/4578 train_loss:4.2716 train_time:34804ms step_avg:141.48ms
step:257/4578 train_loss:4.1917 train_time:34938ms step_avg:141.45ms
step:258/4578 train_loss:4.1876 train_time:35079ms step_avg:141.45ms
step:259/4578 train_loss:4.1603 train_time:35226ms step_avg:141.47ms
step:260/4578 train_loss:4.2162 train_time:35364ms step_avg:141.46ms
step:261/4578 train_loss:4.2481 train_time:35503ms step_avg:141.45ms
step:262/4578 train_loss:4.2173 train_time:35642ms step_avg:141.44ms
step:263/4578 train_loss:4.1753 train_time:35781ms step_avg:141.43ms
step:264/4578 train_loss:4.0944 train_time:35921ms step_avg:141.42ms
step:265/4578 train_loss:4.1757 train_time:36060ms step_avg:141.41ms
step:266/4578 train_loss:4.0488 train_time:36204ms step_avg:141.42ms
step:267/4578 train_loss:4.1090 train_time:36344ms step_avg:141.42ms
step:268/4578 train_loss:4.1103 train_time:36485ms step_avg:141.41ms
step:269/4578 train_loss:4.1301 train_time:36623ms step_avg:141.40ms
step:270/4578 train_loss:4.0432 train_time:36765ms step_avg:141.40ms
step:271/4578 train_loss:4.2764 train_time:36903ms step_avg:141.39ms
step:272/4578 train_loss:4.1740 train_time:37042ms step_avg:141.38ms
step:273/4578 train_loss:4.0885 train_time:37185ms step_avg:141.39ms
step:274/4578 train_loss:4.1372 train_time:37325ms step_avg:141.38ms
step:275/4578 train_loss:4.2189 train_time:37465ms step_avg:141.38ms
step:276/4578 train_loss:4.2435 train_time:37605ms step_avg:141.37ms
step:277/4578 train_loss:4.4167 train_time:37749ms step_avg:141.38ms
step:278/4578 train_loss:4.2109 train_time:37885ms step_avg:141.36ms
step:279/4578 train_loss:4.2697 train_time:38025ms step_avg:141.36ms
step:280/4578 train_loss:4.1820 train_time:38164ms step_avg:141.35ms
step:281/4578 train_loss:4.2966 train_time:38305ms step_avg:141.35ms
step:282/4578 train_loss:4.1367 train_time:38449ms step_avg:141.36ms
step:283/4578 train_loss:4.1436 train_time:38589ms step_avg:141.35ms
step:284/4578 train_loss:4.0890 train_time:38725ms step_avg:141.33ms
step:285/4578 train_loss:4.2348 train_time:38864ms step_avg:141.32ms
step:286/4578 train_loss:4.2331 train_time:39004ms step_avg:141.32ms
step:287/4578 train_loss:4.2647 train_time:39143ms step_avg:141.31ms
step:288/4578 train_loss:4.0974 train_time:39286ms step_avg:141.31ms
step:289/4578 train_loss:4.1992 train_time:39425ms step_avg:141.31ms
step:290/4578 train_loss:4.0477 train_time:39565ms step_avg:141.30ms
step:291/4578 train_loss:4.0503 train_time:39705ms step_avg:141.30ms
step:292/4578 train_loss:4.1244 train_time:39844ms step_avg:141.29ms
step:293/4578 train_loss:4.0499 train_time:39984ms step_avg:141.29ms
step:294/4578 train_loss:4.0922 train_time:40123ms step_avg:141.28ms
step:295/4578 train_loss:4.1330 train_time:40262ms step_avg:141.27ms
step:296/4578 train_loss:4.0182 train_time:40402ms step_avg:141.27ms
step:297/4578 train_loss:4.0255 train_time:40543ms step_avg:141.26ms
step:298/4578 train_loss:4.0318 train_time:40682ms step_avg:141.26ms
step:299/4578 train_loss:4.1370 train_time:40825ms step_avg:141.26ms
step:300/4578 train_loss:4.0042 train_time:40961ms step_avg:141.25ms
step:301/4578 train_loss:4.1424 train_time:41102ms step_avg:141.24ms
step:302/4578 train_loss:4.1503 train_time:41243ms step_avg:141.24ms
step:303/4578 train_loss:4.1006 train_time:41382ms step_avg:141.23ms
step:304/4578 train_loss:4.1533 train_time:41523ms step_avg:141.23ms
step:305/4578 train_loss:4.1358 train_time:41668ms step_avg:141.25ms
step:306/4578 train_loss:4.6216 train_time:41802ms step_avg:141.22ms
step:307/4578 train_loss:4.1035 train_time:41941ms step_avg:141.22ms
step:308/4578 train_loss:4.0137 train_time:42083ms step_avg:141.22ms
step:309/4578 train_loss:4.1657 train_time:42223ms step_avg:141.21ms
step:310/4578 train_loss:4.0309 train_time:42362ms step_avg:141.21ms
step:311/4578 train_loss:4.2431 train_time:42503ms step_avg:141.20ms
step:312/4578 train_loss:4.0990 train_time:42642ms step_avg:141.20ms
step:313/4578 train_loss:4.0440 train_time:42787ms step_avg:141.21ms
step:314/4578 train_loss:4.1316 train_time:42923ms step_avg:141.19ms
step:315/4578 train_loss:4.2574 train_time:43063ms step_avg:141.19ms
step:316/4578 train_loss:4.1216 train_time:43203ms step_avg:141.18ms
step:317/4578 train_loss:3.9632 train_time:43343ms step_avg:141.18ms
step:318/4578 train_loss:4.0441 train_time:43484ms step_avg:141.18ms
step:319/4578 train_loss:4.0781 train_time:43623ms step_avg:141.18ms
step:320/4578 train_loss:4.0508 train_time:43764ms step_avg:141.17ms
step:321/4578 train_loss:4.1630 train_time:43904ms step_avg:141.17ms
step:322/4578 train_loss:4.1143 train_time:44043ms step_avg:141.16ms
step:323/4578 train_loss:4.0938 train_time:44184ms step_avg:141.16ms
step:324/4578 train_loss:4.1724 train_time:44323ms step_avg:141.16ms
step:325/4578 train_loss:4.1214 train_time:44467ms step_avg:141.16ms
step:326/4578 train_loss:4.1944 train_time:44603ms step_avg:141.15ms
step:327/4578 train_loss:4.0595 train_time:44742ms step_avg:141.14ms
step:328/4578 train_loss:4.5599 train_time:44889ms step_avg:141.16ms
step:329/4578 train_loss:4.2354 train_time:45023ms step_avg:141.14ms
step:330/4578 train_loss:3.9793 train_time:45162ms step_avg:141.13ms
step:331/4578 train_loss:3.9210 train_time:45311ms step_avg:141.16ms
step:332/4578 train_loss:4.1373 train_time:45441ms step_avg:141.12ms
step:333/4578 train_loss:4.0659 train_time:45582ms step_avg:141.12ms
step:334/4578 train_loss:4.0441 train_time:45730ms step_avg:141.14ms
step:335/4578 train_loss:4.0030 train_time:45861ms step_avg:141.11ms
step:336/4578 train_loss:4.1796 train_time:46002ms step_avg:141.11ms
step:337/4578 train_loss:4.1195 train_time:46142ms step_avg:141.11ms
step:338/4578 train_loss:4.6032 train_time:46282ms step_avg:141.10ms
step:339/4578 train_loss:4.1055 train_time:46424ms step_avg:141.11ms
step:340/4578 train_loss:4.0519 train_time:46562ms step_avg:141.10ms
step:341/4578 train_loss:4.0923 train_time:46704ms step_avg:141.10ms
step:342/4578 train_loss:4.0080 train_time:46843ms step_avg:141.09ms
step:343/4578 train_loss:3.9760 train_time:46983ms step_avg:141.09ms
step:344/4578 train_loss:4.0187 train_time:47124ms step_avg:141.09ms
step:345/4578 train_loss:4.1556 train_time:47263ms step_avg:141.08ms
step:346/4578 train_loss:3.9999 train_time:47404ms step_avg:141.08ms
step:347/4578 train_loss:3.9318 train_time:47542ms step_avg:141.08ms
step:348/4578 train_loss:3.9737 train_time:47682ms step_avg:141.07ms
step:349/4578 train_loss:4.0180 train_time:47823ms step_avg:141.07ms
step:350/4578 train_loss:3.9826 train_time:47961ms step_avg:141.06ms
step:351/4578 train_loss:3.7127 train_time:48101ms step_avg:141.06ms
step:352/4578 train_loss:3.9698 train_time:48241ms step_avg:141.06ms
step:353/4578 train_loss:4.3214 train_time:48381ms step_avg:141.05ms
step:354/4578 train_loss:3.8241 train_time:48522ms step_avg:141.05ms
step:355/4578 train_loss:4.0818 train_time:48671ms step_avg:141.08ms
step:356/4578 train_loss:3.9455 train_time:48802ms step_avg:141.05ms
step:357/4578 train_loss:4.0555 train_time:48943ms step_avg:141.05ms
step:358/4578 train_loss:3.9895 train_time:49082ms step_avg:141.04ms
step:359/4578 train_loss:4.0083 train_time:49222ms step_avg:141.04ms
step:360/4578 train_loss:4.0328 train_time:49365ms step_avg:141.04ms
step:361/4578 train_loss:3.6226 train_time:49503ms step_avg:141.03ms
step:362/4578 train_loss:4.1793 train_time:49642ms step_avg:141.03ms
step:363/4578 train_loss:4.0773 train_time:49784ms step_avg:141.03ms
step:364/4578 train_loss:4.0000 train_time:49922ms step_avg:141.02ms
step:365/4578 train_loss:3.9082 train_time:50061ms step_avg:141.02ms
step:366/4578 train_loss:4.0738 train_time:50202ms step_avg:141.02ms
step:367/4578 train_loss:4.0250 train_time:50341ms step_avg:141.01ms
step:368/4578 train_loss:4.0180 train_time:50485ms step_avg:141.02ms
step:369/4578 train_loss:4.0008 train_time:50625ms step_avg:141.02ms
step:370/4578 train_loss:3.9009 train_time:50768ms step_avg:141.02ms
step:371/4578 train_loss:4.0444 train_time:50904ms step_avg:141.01ms
step:372/4578 train_loss:3.9289 train_time:51042ms step_avg:141.00ms
step:373/4578 train_loss:3.8569 train_time:51188ms step_avg:141.01ms
step:374/4578 train_loss:4.0756 train_time:51324ms step_avg:141.00ms
step:375/4578 train_loss:3.9941 train_time:51463ms step_avg:141.00ms
step:375/4578 val_loss:3.9903 train_time:51518ms step_avg:141.15ms
step:376/4578 train_loss:3.9667 train_time:51609ms step_avg:141.01ms
step:377/4578 train_loss:4.0259 train_time:51756ms step_avg:141.02ms
step:378/4578 train_loss:3.9513 train_time:52045ms step_avg:141.43ms
step:379/4578 train_loss:4.0054 train_time:52183ms step_avg:141.42ms
step:380/4578 train_loss:4.0282 train_time:52480ms step_avg:141.84ms
step:381/4578 train_loss:4.1068 train_time:52616ms step_avg:141.82ms
step:382/4578 train_loss:4.0012 train_time:52754ms step_avg:141.81ms
step:383/4578 train_loss:3.9800 train_time:52893ms step_avg:141.80ms
step:384/4578 train_loss:3.9475 train_time:53031ms step_avg:141.79ms
step:385/4578 train_loss:4.0263 train_time:53169ms step_avg:141.78ms
step:386/4578 train_loss:3.9416 train_time:53308ms step_avg:141.78ms
step:387/4578 train_loss:4.0492 train_time:53454ms step_avg:141.79ms
step:388/4578 train_loss:4.2390 train_time:53594ms step_avg:141.78ms
step:389/4578 train_loss:3.9602 train_time:53733ms step_avg:141.78ms
step:390/4578 train_loss:3.9474 train_time:53872ms step_avg:141.77ms
step:391/4578 train_loss:4.0460 train_time:54010ms step_avg:141.76ms
step:392/4578 train_loss:3.9680 train_time:54149ms step_avg:141.75ms
step:393/4578 train_loss:4.0756 train_time:54289ms step_avg:141.75ms
step:394/4578 train_loss:3.9176 train_time:54432ms step_avg:141.75ms
step:395/4578 train_loss:4.0483 train_time:54570ms step_avg:141.74ms
step:396/4578 train_loss:3.7949 train_time:54710ms step_avg:141.74ms