1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
|
{
"audit_status": "passed",
"complete_grid": true,
"failed_or_incomplete_cells": [
"resnet56::dualprop"
],
"failure_retaining": true,
"hardware_policy": {
"allowed_device_names": [
"NVIDIA RTX A6000"
],
"allowed_physical_gpu_indices": [
0
],
"profile": "a6000",
"single_gpu_per_process": true
},
"launch_lock": {
"path": "results/resnet_crossover/r2_launch.json",
"registry_sha256": "bc205e39b0403bb2e80c289467a1fb3f746ef970bfe0ee8993a20ebab08713ba",
"sha256": "2d30076208dc9eb69f28e3d0b617b09760a9fea61436e7fb83037eb2379c5e5d"
},
"num_audited_cells": 27,
"num_expected_cells": 27,
"num_finite_cells": 26,
"records": [
{
"all_training_losses_finite": true,
"best_validation_accuracy": 0.1,
"cell_id": "resnet56::ep",
"complete_trajectory": true,
"depth": 56,
"driver_wall_seconds": 50669.00653362274,
"epochs_completed": 100,
"final_validation_accuracy": 0.1,
"final_validation_loss": 0.09450665283203125,
"finite": true,
"first_nonfinite_step": null,
"forward_parameter_count": 853018,
"history_training_examples": 4500000,
"logical_task_loss_queries": 0,
"manifest": "results/resnet_crossover/r2/resnet-r2-ep-d56-lr0p0003.json.manifest.json",
"method": "ep",
"ordinary_training_examples": 4500000,
"ordinary_validation_examples": 505000,
"output_sha256": "f582283d07e8d090b09b62270031ef5fe81d0e79c3da4b1ff4e23084265ec957",
"peak_memory_allocated_bytes": 11683346432,
"physical_gpu_index": 0,
"physical_gpu_uuid": "GPU-b520421c-b120-70f0-eae8-1ff79ad97bf5",
"rate": 0.0003,
"status": "completed",
"total_wall_seconds": 50663.67151045799,
"training_example_presentations": 4500000,
"work": {
"candidate_label_evaluation_presentations": 0,
"feedforward_example_passes": 5005000,
"forward_macs_per_example": 125485696,
"local_vjp_example_evaluations": 118100000,
"relaxation_example_passes": 118100000
}
},
{
"best_validation_accuracy": null,
"cell_id": "resnet56::dualprop",
"complete_trajectory": false,
"depth": 56,
"driver_wall_seconds": 172800.46599197388,
"epochs_completed": 0,
"final_validation_accuracy": null,
"final_validation_loss": null,
"finite": false,
"first_nonfinite_step": null,
"forward_parameter_count": null,
"logical_task_loss_queries": null,
"manifest": "results/resnet_crossover/r2/resnet-r2-dualprop-d56-lr0p05.json.manifest.json",
"method": "dualprop",
"ordinary_training_examples": null,
"ordinary_validation_examples": null,
"output_sha256": null,
"peak_memory_allocated_bytes": null,
"physical_gpu_index": 0,
"physical_gpu_uuid": "GPU-b520421c-b120-70f0-eae8-1ff79ad97bf5",
"rate": 0.05,
"status": "timeout",
"total_wall_seconds": null,
"training_example_presentations": null,
"work": null
},
{
"all_training_losses_finite": true,
"best_validation_accuracy": 0.1122,
"cell_id": "resnet56::ff",
"complete_trajectory": true,
"depth": 56,
"driver_wall_seconds": 23686.061631202698,
"epochs_completed": 2240,
"final_validation_accuracy": 0.1122,
"final_validation_loss": null,
"finite": true,
"first_nonfinite_step": null,
"forward_parameter_count": 853018,
"history_training_examples": 100800000,
"logical_task_loss_queries": 0,
"manifest": "results/resnet_crossover/r2/resnet-r2-ff-d56-lr0p03.json.manifest.json",
"method": "ff",
"ordinary_training_examples": 100800000,
"ordinary_validation_examples": 5000,
"output_sha256": "9b7724301a9ea9ca9b938355594c03d1f5d06bd7e77410c5b27ecac4344445b7",
"peak_memory_allocated_bytes": 13807850496,
"physical_gpu_index": 0,
"physical_gpu_uuid": "GPU-b520421c-b120-70f0-eae8-1ff79ad97bf5",
"rate": 0.03,
"status": "completed",
"total_wall_seconds": 23673.216548919678,
"training_example_presentations": 201600000,
"work": {
"candidate_label_evaluation_presentations": 50000,
"feedforward_example_passes": 201600000,
"forward_macs_per_example": 125485696,
"local_vjp_example_evaluations": 0,
"relaxation_example_passes": 0
}
},
{
"all_training_losses_finite": true,
"best_validation_accuracy": 0.1,
"cell_id": "resnet32::ep",
"complete_trajectory": true,
"depth": 32,
"driver_wall_seconds": 24671.18036866188,
"epochs_completed": 100,
"final_validation_accuracy": 0.1,
"final_validation_loss": 0.09370457153320312,
"finite": true,
"first_nonfinite_step": null,
"forward_parameter_count": 464154,
"history_training_examples": 4500000,
"logical_task_loss_queries": 0,
"manifest": "results/resnet_crossover/r2/resnet-r2-ep-d32-lr0p0003.json.manifest.json",
"method": "ep",
"ordinary_training_examples": 4500000,
"ordinary_validation_examples": 505000,
"output_sha256": "421c661bd6c13d7df5840943805ae1f72ccfd629b8d41b994fe9ca5e2ed65771",
"peak_memory_allocated_bytes": 6988224512,
"physical_gpu_index": 0,
"physical_gpu_uuid": "GPU-2ddb8c2b-c2d4-96ac-fdd7-f2ded951f824",
"rate": 0.0003,
"status": "completed",
"total_wall_seconds": 24661.099477529526,
"training_example_presentations": 4500000,
"work": {
"candidate_label_evaluation_presentations": 0,
"feedforward_example_passes": 5005000,
"forward_macs_per_example": 68862592,
"local_vjp_example_evaluations": 118100000,
"relaxation_example_passes": 118100000
}
},
{
"all_training_losses_finite": true,
"best_validation_accuracy": 0.871,
"cell_id": "resnet32::dualprop",
"complete_trajectory": true,
"depth": 32,
"driver_wall_seconds": 111870.60429954529,
"epochs_completed": 200,
"final_validation_accuracy": 0.8654,
"final_validation_loss": 0.6110852416992187,
"finite": true,
"first_nonfinite_step": null,
"forward_parameter_count": 464154,
"history_training_examples": 9000000,
"logical_task_loss_queries": 0,
"manifest": "results/resnet_crossover/r2/resnet-r2-dualprop-d32-lr0p05.json.manifest.json",
"method": "dualprop",
"ordinary_training_examples": 9000000,
"ordinary_validation_examples": 1005000,
"output_sha256": "d45682de145f1650b61be527d44ba9e38495ef5203112166a19b72ec7d8f0917",
"peak_memory_allocated_bytes": 2322300928,
"physical_gpu_index": 0,
"physical_gpu_uuid": "GPU-9cbd0c6c-f3fb-633d-be44-101a79992530",
"rate": 0.05,
"status": "completed",
"total_wall_seconds": 111860.19822311401,
"training_example_presentations": 9000000,
"work": {
"candidate_label_evaluation_presentations": 0,
"feedforward_example_passes": 10005000,
"forward_macs_per_example": 68862592,
"local_vjp_example_evaluations": 144000000,
"relaxation_example_passes": 144000000
}
},
{
"all_training_losses_finite": true,
"best_validation_accuracy": 0.1228,
"cell_id": "resnet32::ff",
"complete_trajectory": true,
"depth": 32,
"driver_wall_seconds": 7484.067112207413,
"epochs_completed": 1280,
"final_validation_accuracy": 0.1228,
"final_validation_loss": null,
"finite": true,
"first_nonfinite_step": null,
"forward_parameter_count": 464154,
"history_training_examples": 57600000,
"logical_task_loss_queries": 0,
"manifest": "results/resnet_crossover/r2/resnet-r2-ff-d32-lr0p03.json.manifest.json",
"method": "ff",
"ordinary_training_examples": 57600000,
"ordinary_validation_examples": 5000,
"output_sha256": "d48f2e4651502170feb4afc4665bebb9625f077335db9f59c129748c4fdbf731",
"peak_memory_allocated_bytes": 8251766272,
"physical_gpu_index": 0,
"physical_gpu_uuid": "GPU-71e7b0a5-bc0f-e6c0-6965-8a3e2ab76b0b",
"rate": 0.03,
"status": "completed",
"total_wall_seconds": 7469.690595626831,
"training_example_presentations": 115200000,
"work": {
"candidate_label_evaluation_presentations": 50000,
"feedforward_example_passes": 115200000,
"forward_macs_per_example": 68862592,
"local_vjp_example_evaluations": 0,
"relaxation_example_passes": 0
}
},
{
"all_training_losses_finite": true,
"best_validation_accuracy": 0.1,
"cell_id": "resnet20::ep",
"complete_trajectory": true,
"depth": 20,
"driver_wall_seconds": 15028.455446720123,
"epochs_completed": 100,
"final_validation_accuracy": 0.1,
"final_validation_loss": 0.09437472686767578,
"finite": true,
"first_nonfinite_step": null,
"forward_parameter_count": 269722,
"history_training_examples": 4500000,
"logical_task_loss_queries": 0,
"manifest": "results/resnet_crossover/r2/resnet-r2-ep-d20-lr0p0003.json.manifest.json",
"method": "ep",
"ordinary_training_examples": 4500000,
"ordinary_validation_examples": 505000,
"output_sha256": "2be303594e67846889489474ed9bde767bf0abcdea660716ddf421bfc455060f",
"peak_memory_allocated_bytes": 4845611008,
"physical_gpu_index": 0,
"physical_gpu_uuid": "GPU-71e7b0a5-bc0f-e6c0-6965-8a3e2ab76b0b",
"rate": 0.0003,
"status": "completed",
"total_wall_seconds": 15025.534816026688,
"training_example_presentations": 4500000,
"work": {
"candidate_label_evaluation_presentations": 0,
"feedforward_example_passes": 5005000,
"forward_macs_per_example": 40551040,
"local_vjp_example_evaluations": 118100000,
"relaxation_example_passes": 118100000
}
},
{
"all_training_losses_finite": true,
"best_validation_accuracy": 0.9112,
"cell_id": "resnet20::dualprop",
"complete_trajectory": true,
"depth": 20,
"driver_wall_seconds": 49289.51700615883,
"epochs_completed": 200,
"final_validation_accuracy": 0.9104,
"final_validation_loss": 0.43468799438476563,
"finite": true,
"first_nonfinite_step": null,
"forward_parameter_count": 269722,
"history_training_examples": 9000000,
"logical_task_loss_queries": 0,
"manifest": "results/resnet_crossover/r2/resnet-r2-dualprop-d20-lr0p05.json.manifest.json",
"method": "dualprop",
"ordinary_training_examples": 9000000,
"ordinary_validation_examples": 1005000,
"output_sha256": "62576fdd2cce8c465f80416e8eba0f449a025fb0c04e3d13b6aacf6ef5699cd3",
"peak_memory_allocated_bytes": 2008608768,
"physical_gpu_index": 0,
"physical_gpu_uuid": "GPU-71e7b0a5-bc0f-e6c0-6965-8a3e2ab76b0b",
"rate": 0.05,
"status": "completed",
"total_wall_seconds": 49286.44217157364,
"training_example_presentations": 9000000,
"work": {
"candidate_label_evaluation_presentations": 0,
"feedforward_example_passes": 10005000,
"forward_macs_per_example": 40551040,
"local_vjp_example_evaluations": 144000000,
"relaxation_example_passes": 144000000
}
},
{
"all_training_losses_finite": true,
"best_validation_accuracy": 0.1174,
"cell_id": "resnet20::ff",
"complete_trajectory": true,
"depth": 20,
"driver_wall_seconds": 3187.272476911545,
"epochs_completed": 800,
"final_validation_accuracy": 0.1174,
"final_validation_loss": null,
"finite": true,
"first_nonfinite_step": null,
"forward_parameter_count": 269722,
"history_training_examples": 36000000,
"logical_task_loss_queries": 0,
"manifest": "results/resnet_crossover/r2/resnet-r2-ff-d20-lr0p03.json.manifest.json",
"method": "ff",
"ordinary_training_examples": 36000000,
"ordinary_validation_examples": 5000,
"output_sha256": "e6cde27ffd50512b43afb18b6ddccdf1a49b2185001bf6740c96ec84ee7a89ec",
"peak_memory_allocated_bytes": 5474375168,
"physical_gpu_index": 0,
"physical_gpu_uuid": "GPU-71e7b0a5-bc0f-e6c0-6965-8a3e2ab76b0b",
"rate": 0.03,
"status": "completed",
"total_wall_seconds": 3180.144710302353,
"training_example_presentations": 72000000,
"work": {
"candidate_label_evaluation_presentations": 50000,
"feedforward_example_passes": 72000000,
"forward_macs_per_example": 40551040,
"local_vjp_example_evaluations": 0,
"relaxation_example_passes": 0
}
},
{
"all_training_losses_finite": true,
"best_validation_accuracy": 0.2354,
"cell_id": "resnet56::pepita",
"complete_trajectory": true,
"depth": 56,
"driver_wall_seconds": 1602.2039358615875,
"epochs_completed": 100,
"final_validation_accuracy": 0.228,
"final_validation_loss": 2.124036181640625,
"finite": true,
"first_nonfinite_step": null,
"forward_parameter_count": 853018,
"history_training_examples": 4500000,
"logical_task_loss_queries": 0,
"manifest": "results/resnet_crossover/r2/resnet-r2-pepita-d56-lr0p003.json.manifest.json",
"method": "pepita",
"ordinary_training_examples": 4500000,
"ordinary_validation_examples": 505000,
"output_sha256": "dcebb1518d54fbefbe9bd0cb2a3b63a18591187b4dcad3cebeb3e93b1748c6e6",
"peak_memory_allocated_bytes": 3249364480,
"physical_gpu_index": 0,
"physical_gpu_uuid": "GPU-71e7b0a5-bc0f-e6c0-6965-8a3e2ab76b0b",
"rate": 0.003,
"status": "completed",
"total_wall_seconds": 1598.730625152588,
"training_example_presentations": 9000000,
"work": {
"candidate_label_evaluation_presentations": 0,
"feedforward_example_passes": 14005000,
"forward_macs_per_example": 125485696,
"local_vjp_example_evaluations": 0,
"relaxation_example_passes": 0
}
},
{
"all_training_losses_finite": true,
"best_validation_accuracy": 0.935,
"cell_id": "resnet56::sdil",
"complete_trajectory": true,
"depth": 56,
"driver_wall_seconds": 8350.019015789032,
"epochs_completed": 200,
"final_validation_accuracy": 0.9322,
"final_validation_loss": 0.3675199523925781,
"finite": true,
"first_nonfinite_step": null,
"forward_parameter_count": 853018,
"history_training_examples": 9000000,
"logical_task_loss_queries": 0,
"manifest": "results/resnet_crossover/r2/resnet-r2-sdil-d56-lr0p1.json.manifest.json",
"method": "sdil",
"ordinary_training_examples": 9000000,
"ordinary_validation_examples": 1005000,
"output_sha256": "84f1139c05218524c1be2807d0e2bacb54bc0a0bc0d0526a0dcf7d8a2de8f5f6",
"peak_memory_allocated_bytes": 3262279168,
"physical_gpu_index": 0,
"physical_gpu_uuid": "GPU-71e7b0a5-bc0f-e6c0-6965-8a3e2ab76b0b",
"rate": 0.1,
"status": "completed",
"total_wall_seconds": 8344.301742315292,
"training_example_presentations": 9000000,
"work": {
"elementwise_operations_estimate": 115016770519040,
"forward_macs_per_example": 125485696,
"total_forward_equivalent_examples": 9000064,
"total_macs_estimate": 4509538465857536
}
},
{
"all_training_losses_finite": true,
"best_validation_accuracy": 0.9308,
"cell_id": "resnet56::clean_kp",
"complete_trajectory": true,
"depth": 56,
"driver_wall_seconds": 5064.757298946381,
"epochs_completed": 200,
"final_validation_accuracy": 0.9274,
"final_validation_loss": 0.3740764099121094,
"finite": true,
"first_nonfinite_step": null,
"forward_parameter_count": 853018,
"history_training_examples": 9000000,
"logical_task_loss_queries": 0,
"manifest": "results/resnet_crossover/r2/resnet-r2-clean_kp-d56-lr0p1.json.manifest.json",
"method": "clean_kp",
"ordinary_training_examples": 9000000,
"ordinary_validation_examples": 1005000,
"output_sha256": "69d1c59511cfe98a72af9c1be079d4fa581096630b2e93f4ad028a4a935bc9fa",
"peak_memory_allocated_bytes": 3256025088,
"physical_gpu_index": 0,
"physical_gpu_uuid": "GPU-71e7b0a5-bc0f-e6c0-6965-8a3e2ab76b0b",
"rate": 0.1,
"status": "completed",
"total_wall_seconds": 5054.333840370178,
"training_example_presentations": 9000000,
"work": {
"elementwise_operations_estimate": 0,
"forward_macs_per_example": 125485696,
"total_forward_equivalent_examples": 9000000,
"total_macs_estimate": 4509522432000000
}
},
{
"all_training_losses_finite": true,
"best_validation_accuracy": 0.6674,
"cell_id": "resnet56::fa",
"complete_trajectory": true,
"depth": 56,
"driver_wall_seconds": 3838.2244839668274,
"epochs_completed": 200,
"final_validation_accuracy": 0.6436,
"final_validation_loss": 1.0704212158203126,
"finite": true,
"first_nonfinite_step": null,
"forward_parameter_count": 853018,
"history_training_examples": 9000000,
"logical_task_loss_queries": 0,
"manifest": "results/resnet_crossover/r2/resnet-r2-fa-d56-lr0p1.json.manifest.json",
"method": "fa",
"ordinary_training_examples": 9000000,
"ordinary_validation_examples": 1005000,
"output_sha256": "db985851db69598966506f4067c2febdd411b2fa61b02d330263f6a9ed90a776",
"peak_memory_allocated_bytes": 3252628992,
"physical_gpu_index": 0,
"physical_gpu_uuid": "GPU-8c47bc34-8284-ba08-eb44-8f381122ed6d",
"rate": 0.1,
"status": "completed",
"total_wall_seconds": 3827.3347294330597,
"training_example_presentations": 9000000,
"work": {
"elementwise_operations_estimate": 0,
"forward_macs_per_example": 125485696,
"total_forward_equivalent_examples": 9000000,
"total_macs_estimate": 3384132480000000
}
},
{
"all_training_losses_finite": true,
"best_validation_accuracy": 0.4056,
"cell_id": "resnet56::dfa",
"complete_trajectory": true,
"depth": 56,
"driver_wall_seconds": 3246.3810822963715,
"epochs_completed": 200,
"final_validation_accuracy": 0.284,
"final_validation_loss": 56.89802421875,
"finite": true,
"first_nonfinite_step": null,
"forward_parameter_count": 853018,
"history_training_examples": 9000000,
"logical_task_loss_queries": 0,
"manifest": "results/resnet_crossover/r2/resnet-r2-dfa-d56-lr0p01.json.manifest.json",
"method": "dfa",
"ordinary_training_examples": 9000000,
"ordinary_validation_examples": 1005000,
"output_sha256": "f646a557aca62c9dbfb2672fedd765f0cd51d11256c50995933d43d90a18c8f9",
"peak_memory_allocated_bytes": 3276921856,
"physical_gpu_index": 0,
"physical_gpu_uuid": "GPU-8c47bc34-8284-ba08-eb44-8f381122ed6d",
"rate": 0.01,
"status": "completed",
"total_wall_seconds": 3241.7450346946716,
"training_example_presentations": 9000000,
"work": {
"elementwise_operations_estimate": 0,
"forward_macs_per_example": 125485696,
"total_forward_equivalent_examples": 9000000,
"total_macs_estimate": 2306665728000000
}
},
{
"all_training_losses_finite": true,
"best_validation_accuracy": 0.9308,
"cell_id": "resnet56::bp",
"complete_trajectory": true,
"depth": 56,
"driver_wall_seconds": 3326.933441400528,
"epochs_completed": 200,
"final_validation_accuracy": 0.93,
"final_validation_loss": 0.3817935302734375,
"finite": true,
"first_nonfinite_step": null,
"forward_parameter_count": 853018,
"history_training_examples": 9000000,
"logical_task_loss_queries": 0,
"manifest": "results/resnet_crossover/r2/resnet-r2-bp-d56-lr0p1.json.manifest.json",
"method": "bp",
"ordinary_training_examples": 9000000,
"ordinary_validation_examples": 1005000,
"output_sha256": "e4876181a6d1d8ddbda3b439319a758b9c9e42170add0a78f72df5378abd4088",
"peak_memory_allocated_bytes": 3257588736,
"physical_gpu_index": 0,
"physical_gpu_uuid": "GPU-71e7b0a5-bc0f-e6c0-6965-8a3e2ab76b0b",
"rate": 0.1,
"status": "completed",
"total_wall_seconds": 3323.5966560840607,
"training_example_presentations": 9000000,
"work": {
"elementwise_operations_estimate": 0,
"forward_macs_per_example": 125485696,
"total_forward_equivalent_examples": 9000000,
"total_macs_estimate": 3388113792000000
}
},
{
"all_training_losses_finite": true,
"best_validation_accuracy": 0.2682,
"cell_id": "resnet32::pepita",
"complete_trajectory": true,
"depth": 32,
"driver_wall_seconds": 909.2571918964386,
"epochs_completed": 100,
"final_validation_accuracy": 0.2618,
"final_validation_loss": 2.029934326171875,
"finite": true,
"first_nonfinite_step": null,
"forward_parameter_count": 464154,
"history_training_examples": 4500000,
"logical_task_loss_queries": 0,
"manifest": "results/resnet_crossover/r2/resnet-r2-pepita-d32-lr0p003.json.manifest.json",
"method": "pepita",
"ordinary_training_examples": 4500000,
"ordinary_validation_examples": 505000,
"output_sha256": "461ff5f189160d80f86da8972f3742e260ebfb4acc720aab961f403f047ba4f1",
"peak_memory_allocated_bytes": 2322423808,
"physical_gpu_index": 0,
"physical_gpu_uuid": "GPU-8c47bc34-8284-ba08-eb44-8f381122ed6d",
"rate": 0.003,
"status": "completed",
"total_wall_seconds": 896.4814369678497,
"training_example_presentations": 9000000,
"work": {
"candidate_label_evaluation_presentations": 0,
"feedforward_example_passes": 14005000,
"forward_macs_per_example": 68862592,
"local_vjp_example_evaluations": 0,
"relaxation_example_passes": 0
}
},
{
"all_training_losses_finite": true,
"best_validation_accuracy": 0.9254,
"cell_id": "resnet32::sdil",
"complete_trajectory": true,
"depth": 32,
"driver_wall_seconds": 4740.1479415893555,
"epochs_completed": 200,
"final_validation_accuracy": 0.9236,
"final_validation_loss": 0.3822314697265625,
"finite": true,
"first_nonfinite_step": null,
"forward_parameter_count": 464154,
"history_training_examples": 9000000,
"logical_task_loss_queries": 0,
"manifest": "results/resnet_crossover/r2/resnet-r2-sdil-d32-lr0p1.json.manifest.json",
"method": "sdil",
"ordinary_training_examples": 9000000,
"ordinary_validation_examples": 1005000,
"output_sha256": "516a62577e7b024b7686e816ebb55bb95f9de07204117af1905ade3b67f14f09",
"peak_memory_allocated_bytes": 2329526272,
"physical_gpu_index": 0,
"physical_gpu_uuid": "GPU-8c47bc34-8284-ba08-eb44-8f381122ed6d",
"rate": 0.1,
"status": "completed",
"total_wall_seconds": 4736.380233287811,
"training_example_presentations": 9000000,
"work": {
"elementwise_operations_estimate": 65471084756992,
"forward_macs_per_example": 68862592,
"total_forward_equivalent_examples": 9000064,
"total_macs_estimate": 2471099474100224
}
},
{
"all_training_losses_finite": true,
"best_validation_accuracy": 0.9262,
"cell_id": "resnet32::clean_kp",
"complete_trajectory": true,
"depth": 32,
"driver_wall_seconds": 2880.570287704468,
"epochs_completed": 200,
"final_validation_accuracy": 0.9238,
"final_validation_loss": 0.39594262084960935,
"finite": true,
"first_nonfinite_step": null,
"forward_parameter_count": 464154,
"history_training_examples": 9000000,
"logical_task_loss_queries": 0,
"manifest": "results/resnet_crossover/r2/resnet-r2-clean_kp-d32-lr0p1.json.manifest.json",
"method": "clean_kp",
"ordinary_training_examples": 9000000,
"ordinary_validation_examples": 1005000,
"output_sha256": "c8170145b2374f71f7bfdc161acfba10610b8ceb3c561b00b20a6182c369349c",
"peak_memory_allocated_bytes": 2325996544,
"physical_gpu_index": 0,
"physical_gpu_uuid": "GPU-71e7b0a5-bc0f-e6c0-6965-8a3e2ab76b0b",
"rate": 0.1,
"status": "completed",
"total_wall_seconds": 2877.0025794506073,
"training_example_presentations": 9000000,
"work": {
"elementwise_operations_estimate": 0,
"forward_macs_per_example": 68862592,
"total_forward_equivalent_examples": 9000000,
"total_macs_estimate": 2471090688000000
}
},
{
"all_training_losses_finite": true,
"best_validation_accuracy": 0.6762,
"cell_id": "resnet32::fa",
"complete_trajectory": true,
"depth": 32,
"driver_wall_seconds": 2180.355216741562,
"epochs_completed": 200,
"final_validation_accuracy": 0.6548,
"final_validation_loss": 0.9980604614257812,
"finite": true,
"first_nonfinite_step": null,
"forward_parameter_count": 464154,
"history_training_examples": 9000000,
"logical_task_loss_queries": 0,
"manifest": "results/resnet_crossover/r2/resnet-r2-fa-d32-lr0p1.json.manifest.json",
"method": "fa",
"ordinary_training_examples": 9000000,
"ordinary_validation_examples": 1005000,
"output_sha256": "0a5f7017c2ab6fb991e98cf2b9c8e1438685f2575a2eaa41e80bdfdaa4162cf6",
"peak_memory_allocated_bytes": 2324148736,
"physical_gpu_index": 0,
"physical_gpu_uuid": "GPU-71e7b0a5-bc0f-e6c0-6965-8a3e2ab76b0b",
"rate": 0.1,
"status": "completed",
"total_wall_seconds": 2176.771654367447,
"training_example_presentations": 9000000,
"work": {
"elementwise_operations_estimate": 0,
"forward_macs_per_example": 68862592,
"total_forward_equivalent_examples": 9000000,
"total_macs_estimate": 1855308672000000
}
},
{
"all_training_losses_finite": true,
"best_validation_accuracy": 0.4546,
"cell_id": "resnet32::dfa",
"complete_trajectory": true,
"depth": 32,
"driver_wall_seconds": 1879.2923183441162,
"epochs_completed": 200,
"final_validation_accuracy": 0.3874,
"final_validation_loss": 6.95157197265625,
"finite": true,
"first_nonfinite_step": null,
"forward_parameter_count": 464154,
"history_training_examples": 9000000,
"logical_task_loss_queries": 0,
"manifest": "results/resnet_crossover/r2/resnet-r2-dfa-d32-lr0p01.json.manifest.json",
"method": "dfa",
"ordinary_training_examples": 9000000,
"ordinary_validation_examples": 1005000,
"output_sha256": "fdc45fa5f2e27326288de0b06db43af4ff8b0d46164473daf9f1b9a114dcb474",
"peak_memory_allocated_bytes": 2338062336,
"physical_gpu_index": 0,
"physical_gpu_uuid": "GPU-8c47bc34-8284-ba08-eb44-8f381122ed6d",
"rate": 0.01,
"status": "completed",
"total_wall_seconds": 1875.3221385478973,
"training_example_presentations": 9000000,
"work": {
"elementwise_operations_estimate": 0,
"forward_macs_per_example": 68862592,
"total_forward_equivalent_examples": 9000000,
"total_macs_estimate": 1266806016000000
}
},
{
"all_training_losses_finite": true,
"best_validation_accuracy": 0.928,
"cell_id": "resnet32::bp",
"complete_trajectory": true,
"depth": 32,
"driver_wall_seconds": 1930.4826476573944,
"epochs_completed": 200,
"final_validation_accuracy": 0.928,
"final_validation_loss": 0.38430252685546873,
"finite": true,
"first_nonfinite_step": null,
"forward_parameter_count": 464154,
"history_training_examples": 9000000,
"logical_task_loss_queries": 0,
"manifest": "results/resnet_crossover/r2/resnet-r2-bp-d32-lr0p1.json.manifest.json",
"method": "bp",
"ordinary_training_examples": 9000000,
"ordinary_validation_examples": 1005000,
"output_sha256": "2450755a60aeebffe5376d58ac95a5a17b1b4af03202b8a09e71ca6672555cef",
"peak_memory_allocated_bytes": 2330820608,
"physical_gpu_index": 0,
"physical_gpu_uuid": "GPU-71e7b0a5-bc0f-e6c0-6965-8a3e2ab76b0b",
"rate": 0.1,
"status": "completed",
"total_wall_seconds": 1926.7095592021942,
"training_example_presentations": 9000000,
"work": {
"elementwise_operations_estimate": 0,
"forward_macs_per_example": 68862592,
"total_forward_equivalent_examples": 9000000,
"total_macs_estimate": 1859289984000000
}
},
{
"all_training_losses_finite": true,
"best_validation_accuracy": 0.2782,
"cell_id": "resnet20::pepita",
"complete_trajectory": true,
"depth": 20,
"driver_wall_seconds": 565.1650466918945,
"epochs_completed": 100,
"final_validation_accuracy": 0.2726,
"final_validation_loss": 1.9752171875,
"finite": true,
"first_nonfinite_step": null,
"forward_parameter_count": 269722,
"history_training_examples": 4500000,
"logical_task_loss_queries": 0,
"manifest": "results/resnet_crossover/r2/resnet-r2-pepita-d20-lr0p003.json.manifest.json",
"method": "pepita",
"ordinary_training_examples": 4500000,
"ordinary_validation_examples": 505000,
"output_sha256": "b9120fa19a170ae650e432067cbf3df388be3f1606f18489180e646b65ac70e2",
"peak_memory_allocated_bytes": 2008731648,
"physical_gpu_index": 0,
"physical_gpu_uuid": "GPU-8c47bc34-8284-ba08-eb44-8f381122ed6d",
"rate": 0.003,
"status": "completed",
"total_wall_seconds": 561.5238916873932,
"training_example_presentations": 9000000,
"work": {
"candidate_label_evaluation_presentations": 0,
"feedforward_example_passes": 14005000,
"forward_macs_per_example": 40551040,
"local_vjp_example_evaluations": 0,
"relaxation_example_passes": 0
}
},
{
"all_training_losses_finite": true,
"best_validation_accuracy": 0.9158,
"cell_id": "resnet20::sdil",
"complete_trajectory": true,
"depth": 20,
"driver_wall_seconds": 3006.6630001068115,
"epochs_completed": 200,
"final_validation_accuracy": 0.9132,
"final_validation_loss": 0.4233872314453125,
"finite": true,
"first_nonfinite_step": null,
"forward_parameter_count": 269722,
"history_training_examples": 9000000,
"logical_task_loss_queries": 0,
"manifest": "results/resnet_crossover/r2/resnet-r2-sdil-d20-lr0p1.json.manifest.json",
"method": "sdil",
"ordinary_training_examples": 9000000,
"ordinary_validation_examples": 1005000,
"output_sha256": "b0be7a2e395429c4bb29209bdce5d250dccd484048551fde04102553ed2ad272",
"peak_memory_allocated_bytes": 2013018112,
"physical_gpu_index": 0,
"physical_gpu_uuid": "GPU-8c47bc34-8284-ba08-eb44-8f381122ed6d",
"rate": 0.1,
"status": "completed",
"total_wall_seconds": 3003.0224483013153,
"training_example_presentations": 9000000,
"work": {
"elementwise_operations_estimate": 40698241875968,
"forward_macs_per_example": 40551040,
"total_forward_equivalent_examples": 9000064,
"total_macs_estimate": 1451879978221568
}
},
{
"all_training_losses_finite": true,
"best_validation_accuracy": 0.9174,
"cell_id": "resnet20::clean_kp",
"complete_trajectory": true,
"depth": 20,
"driver_wall_seconds": 1784.3792207241058,
"epochs_completed": 200,
"final_validation_accuracy": 0.9144,
"final_validation_loss": 0.4333634704589844,
"finite": true,
"first_nonfinite_step": null,
"forward_parameter_count": 269722,
"history_training_examples": 9000000,
"logical_task_loss_queries": 0,
"manifest": "results/resnet_crossover/r2/resnet-r2-clean_kp-d20-lr0p1.json.manifest.json",
"method": "clean_kp",
"ordinary_training_examples": 9000000,
"ordinary_validation_examples": 1005000,
"output_sha256": "87914e8c8674f9fc2d136f6dbf11ee9512c7e797d2d757b675ea7610ba6db795",
"peak_memory_allocated_bytes": 2010747392,
"physical_gpu_index": 0,
"physical_gpu_uuid": "GPU-71e7b0a5-bc0f-e6c0-6965-8a3e2ab76b0b",
"rate": 0.1,
"status": "completed",
"total_wall_seconds": 1780.5374817848206,
"training_example_presentations": 9000000,
"work": {
"elementwise_operations_estimate": 0,
"forward_macs_per_example": 40551040,
"total_forward_equivalent_examples": 9000000,
"total_macs_estimate": 1451874816000000
}
},
{
"all_training_losses_finite": true,
"best_validation_accuracy": 0.7066,
"cell_id": "resnet20::fa",
"complete_trajectory": true,
"depth": 20,
"driver_wall_seconds": 1365.4268357753754,
"epochs_completed": 200,
"final_validation_accuracy": 0.6854,
"final_validation_loss": 0.9467566040039063,
"finite": true,
"first_nonfinite_step": null,
"forward_parameter_count": 269722,
"history_training_examples": 9000000,
"logical_task_loss_queries": 0,
"manifest": "results/resnet_crossover/r2/resnet-r2-fa-d20-lr0p1.json.manifest.json",
"method": "fa",
"ordinary_training_examples": 9000000,
"ordinary_validation_examples": 1005000,
"output_sha256": "4214a5ec034977d0fe8f927f9de040e282833008270204936eb81b419f16c399",
"peak_memory_allocated_bytes": 2009673728,
"physical_gpu_index": 0,
"physical_gpu_uuid": "GPU-71e7b0a5-bc0f-e6c0-6965-8a3e2ab76b0b",
"rate": 0.1,
"status": "completed",
"total_wall_seconds": 1361.8625671863556,
"training_example_presentations": 9000000,
"work": {
"elementwise_operations_estimate": 0,
"forward_macs_per_example": 40551040,
"total_forward_equivalent_examples": 9000000,
"total_macs_estimate": 1090896768000000
}
},
{
"all_training_losses_finite": true,
"best_validation_accuracy": 0.4502,
"cell_id": "resnet20::dfa",
"complete_trajectory": true,
"depth": 20,
"driver_wall_seconds": 1187.1560876369476,
"epochs_completed": 200,
"final_validation_accuracy": 0.3492,
"final_validation_loss": 3.6838392578125,
"finite": true,
"first_nonfinite_step": null,
"forward_parameter_count": 269722,
"history_training_examples": 9000000,
"logical_task_loss_queries": 0,
"manifest": "results/resnet_crossover/r2/resnet-r2-dfa-d20-lr0p01.json.manifest.json",
"method": "dfa",
"ordinary_training_examples": 9000000,
"ordinary_validation_examples": 1005000,
"output_sha256": "638ace440ba8224c18e4e0f09f2dbac291ec32e4f54bc9efadbbff58ddf2ac45",
"peak_memory_allocated_bytes": 2018397696,
"physical_gpu_index": 0,
"physical_gpu_uuid": "GPU-8c47bc34-8284-ba08-eb44-8f381122ed6d",
"rate": 0.01,
"status": "completed",
"total_wall_seconds": 1183.4196124076843,
"training_example_presentations": 9000000,
"work": {
"elementwise_operations_estimate": 0,
"forward_macs_per_example": 40551040,
"total_forward_equivalent_examples": 9000000,
"total_macs_estimate": 746876160000000
}
},
{
"all_training_losses_finite": true,
"best_validation_accuracy": 0.9196,
"cell_id": "resnet20::bp",
"complete_trajectory": true,
"depth": 20,
"driver_wall_seconds": 1228.3254659175873,
"epochs_completed": 200,
"final_validation_accuracy": 0.9176,
"final_validation_loss": 0.39629163818359375,
"finite": true,
"first_nonfinite_step": null,
"forward_parameter_count": 269722,
"history_training_examples": 9000000,
"logical_task_loss_queries": 0,
"manifest": "results/resnet_crossover/r2/resnet-r2-bp-d20-lr0p1.json.manifest.json",
"method": "bp",
"ordinary_training_examples": 9000000,
"ordinary_validation_examples": 1005000,
"output_sha256": "5cf3b2928b4d27a0968fb42d633d5c23d91dad233b24a25daffb0fa9cd2fe7ee",
"peak_memory_allocated_bytes": 2017119744,
"physical_gpu_index": 0,
"physical_gpu_uuid": "GPU-71e7b0a5-bc0f-e6c0-6965-8a3e2ab76b0b",
"rate": 0.1,
"status": "completed",
"total_wall_seconds": 1224.873178243637,
"training_example_presentations": 9000000,
"work": {
"elementwise_operations_estimate": 0,
"forward_macs_per_example": 40551040,
"total_forward_equivalent_examples": 9000000,
"total_macs_estimate": 1094878080000000
}
}
],
"selector": {
"p1_source": {
"git_commit": "19f3966fc2d15b1ed18862a779d91ee16ed56bfc",
"protocol_path": "/scratch/yurenh2/sdil-runs/resnet-p1/RESNET_CROSSOVER.md",
"protocol_sha256": "048dbaf06866bdd3959a637ba827c06063205f6d1cc693d89aa8fc9c1d24975d",
"python": "/home/yurenh2/miniconda3/envs/ep_pascal/bin/python3",
"tracked_files": {
"RESNET_CROSSOVER.md": "048dbaf06866bdd3959a637ba827c06063205f6d1cc693d89aa8fc9c1d24975d",
"experiments/analyze_resnet_crossover_p1.py": "6ff485a3fecc7b814dedf55d04abdb770a6ce904cdeb9e06a7dddcf26f996d5b",
"experiments/conv_run.py": "9fa2daea597d63981a78248fafb154da39775ccfae57aac9889e78d83dc4eb44",
"experiments/resnet_crossover_grid.py": "669c84846bc3115816ae3c9036be8722f21577612dc9d56a689d3321e837ca95",
"experiments/resnet_crossover_native.py": "c5e54324b000c37c53d495f7d3eb0d4a85dcf15e83ea3aab125a097ce61eff87",
"sdil/conv.py": "09f87eef636aa5397b39a2a7a66909b8861640624507eabbdfaffe98577a5d2c",
"sdil/conv_crossover.py": "3c1373aa3555e77cce37f093dec7b7aaedca475f608b092d13706bfff1dbe1a8",
"sdil/data.py": "262031353b5952fb4d1b0e3c3d1e9b3186ae42b88601ccb89e1748ffd36ed36d"
}
},
"path": "results/resnet_crossover/p1_selector.json",
"selected_rates": {
"bp": 0.1,
"clean_kp": 0.1,
"dfa": 0.01,
"dualprop": 0.05,
"ep": 0.0003,
"fa": 0.1,
"ff": 0.03,
"pepita": 0.003,
"sdil": 0.1
},
"sha256": "0f0fcdfcd0056b034a9cd95e57970782bb2c33785832ab1cbe27e5dbe212111b"
},
"source": {
"git_commit": "f55872beaee658ff76ac15fd421b73017820c320",
"protocol_path": "/orion/u/oscarwan/scratch/sdil-work/sdil-a6000-full-rerun/RESNET_CROSSOVER.md",
"protocol_sha256": "16d110298f88d383f528b9c9b81f20ba11920c89596534ebb4a5c1e057f97223",
"python": "/orion/u/oscarwan/scratch/miniconda3/envs/sdil-main/bin/python",
"tracked_files": {
"RESNET_CROSSOVER.md": "16d110298f88d383f528b9c9b81f20ba11920c89596534ebb4a5c1e057f97223",
"experiments/analyze_resnet_crossover_r2.py": "ddad2a41b45ef41b7f26ddde0d6fdb9255592307714351d65b98e109b73ca6d1",
"experiments/conv_run.py": "9fa2daea597d63981a78248fafb154da39775ccfae57aac9889e78d83dc4eb44",
"experiments/crossover_hardware.py": "f7fd5ab940d6790d1f60c1bf0ec2115db8db7163a4ee034b536908ef72ac025f",
"experiments/resnet_crossover_native.py": "c04a5311c2aa4f7820f0f6883aca6e678b1c3227085c5d88fef46c83ac65e83d",
"experiments/resnet_crossover_r2.py": "8fe2844211a0220873a11ad801d672db8e2869fcb914bc972396bfd24adb040f",
"results/resnet_crossover/p1_selector.json": "0f0fcdfcd0056b034a9cd95e57970782bb2c33785832ab1cbe27e5dbe212111b",
"sdil/conv.py": "09f87eef636aa5397b39a2a7a66909b8861640624507eabbdfaffe98577a5d2c",
"sdil/conv_crossover.py": "3c1373aa3555e77cce37f093dec7b7aaedca475f608b092d13706bfff1dbe1a8",
"sdil/data.py": "0e8496c3bbcaa090c280afc3dbdd8b4c9490eeaca3306afff8aea081dccde254"
}
},
"stage": "resnet_crossover_r2",
"test_policy": "none"
}
|
