diff options
Diffstat (limited to 'results/kp_innovation_diagnosis/matched.json')
| -rw-r--r-- | results/kp_innovation_diagnosis/matched.json | 7316 |
1 files changed, 7316 insertions, 0 deletions
diff --git a/results/kp_innovation_diagnosis/matched.json b/results/kp_innovation_diagnosis/matched.json new file mode 100644 index 0000000..e930160 --- /dev/null +++ b/results/kp_innovation_diagnosis/matched.json @@ -0,0 +1,7316 @@ +{ + "first_any_nonfinite_groups": { + "bn_running_var": [ + 2 + ] + }, + "first_any_nonfinite_step": 32, + "first_nonfinite_by_group": { + "bn_running_var": { + "indices": [ + 2 + ], + "step": 32 + }, + "forward_momentum": { + "indices": [ + 0 + ], + "step": 69 + }, + "forward_weight": { + "indices": [ + 0 + ], + "step": 69 + } + }, + "first_nonfinite_by_metric": { + "raw_apical_rms": 68, + "traffic_rms": 68 + }, + "first_nonfinite_metrics": [], + "first_training_failure_groups": { + "forward_momentum": [ + 0 + ], + "forward_weight": [ + 0 + ] + }, + "first_training_failure_step": 69, + "initial_parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.0, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.0, + "nonfinite_indices": [] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.0, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.4011031687259674, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.0, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.8331910371780396, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.0, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.0, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.0, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.0, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.0, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.3956821858882904, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.0, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.1162850856781006, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.027761053293943405, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.09883774071931839, + "nonfinite_indices": [] + } + }, + "max_steps": 352, + "predictor_warmup": { + "first_mse": 0.0002822713358909823, + "last_mse": 5.6679267276115794e-06, + "post_warmup_traffic_residual_rms_ratio": 0.13994200326198175, + "steps": 20, + "task_loader_state_restored": true + }, + "protocol": "kp_mixed_traffic_nonfinite_diagnosis_v1", + "provenance": { + "git_commit": "e9962317b3d976fe25e28a5693f2757e3758480a", + "git_tracked_dirty": false + }, + "rule": "matched", + "scope": "training_only_no_validation_or_test_evaluation", + "split": { + "dataset": "cifar10", + "input_layout": "NCHW", + "input_shape": [ + 3, + 32, + 32 + ], + "loader_seed": 0, + "normalization_mean": [ + 0.49140000343322754, + 0.4821999967098236, + 0.4465000033378601 + ], + "normalization_std": [ + 0.24699999392032623, + 0.2434999942779541, + 0.26159998774528503 + ], + "split_from_training_only": true, + "split_seed": 2027, + "test_examples": 10000, + "train_examples": 45000, + "training_augmentation": "random_crop_32_padding_4_zero_then_horizontal_flip_p0.5", + "validation_class_counts": { + "0": 500, + "1": 500, + "2": 500, + "3": 500, + "4": 500, + "5": 500, + "6": 500, + "7": 500, + "8": 500, + "9": 500 + }, + "validation_examples": 5000, + "validation_index_sha256": "8328b206a97c420e49e54e3eca4abe3274c4756b084355784ea3fb8059e4515b" + }, + "test_evaluations": 0, + "traffic_calibration": { + "instruction_rms": [ + 0.008471464738249779, + 0.006617916747927666, + 0.003988116513937712, + 0.0043865954503417015, + 0.002713605063036084, + 0.003158062230795622, + 0.002004207344725728, + 0.003757704282179475, + 0.0027085733599960804, + 0.0030200635083019733, + 0.002056022873148322, + 0.0023899395018815994, + 0.0016842411132529378, + 0.0034010226372629404, + 0.002549725119024515, + 0.0030542907770723104, + 0.0020918569061905146, + 0.002605808200314641, + 0.0018916188273578882 + ], + "realized_traffic_instruction_rms_ratio": [ + 4.0, + 4.0, + 4.0, + 4.0, + 4.0, + 4.0, + 3.999999523162842, + 4.0, + 4.000000476837158, + 3.999999761581421, + 4.0, + 4.0, + 3.999999761581421, + 4.0, + 4.000000476837158, + 4.000000476837158, + 4.0, + 4.0, + 4.0 + ], + "target_ratio": 4.0, + "traffic_gain": [ + 0.0443243607878685, + 0.035817068070173264, + 0.013778294436633587, + 0.022762222215533257, + 0.007214079145342112, + 0.017218582332134247, + 0.004616298712790012, + 0.02042512409389019, + 0.007327491883188486, + 0.01611751690506935, + 0.004703814629465342, + 0.012562948279082775, + 0.003324519144371152, + 0.01805141195654869, + 0.006063441745936871, + 0.016296451911330223, + 0.0044285994954407215, + 0.013874167576432228, + 0.0036007820162922144 + ], + "unscaled_traffic_rms": [ + 0.7644973993301392, + 0.7390796542167664, + 1.157796859741211, + 0.7708553671836853, + 1.504616141319275, + 0.7336404919624329, + 1.736635684967041, + 0.7358984351158142, + 1.478581428527832, + 0.7495108246803284, + 1.7483876943588257, + 0.7609485983848572, + 2.0264477729797363, + 0.7536302804946899, + 1.6820316314697266, + 0.7496824264526367, + 1.8894071578979492, + 0.7512690424919128, + 2.10134220123291 + ] + }, + "trajectory": [ + { + "batch_loss": 2.9134020805358887, + "innovation_rms": 0.0047711752050758295, + "instruction_rms": 0.004215397969971197, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.40285658836364746, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.9875131845474243, + "nonfinite_indices": [] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.8092546463012695, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.4058851897716522, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.6322960257530212, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.8331910371780396, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.5760545134544373, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 5.760544776916504, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.69287109375, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 6.9287109375, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.8092595338821411, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.3794170618057251, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.632293164730072, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.114573359489441, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.0261825043708086, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.1002056896686554, + "nonfinite_indices": [] + } + }, + "predictor_mse": 4.976208305399031e-06, + "predictor_updated": true, + "raw_apical_rms": 0.01731826254949561, + "step": 1, + "teaching_rms": 0.0047711752485875515, + "traffic_rms": 0.016800932589918403 + }, + { + "batch_loss": 2.8719682693481445, + "innovation_rms": 0.0030266363028306265, + "instruction_rms": 0.001145980237518358, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.090774655342102, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": true, + "max_abs_over_finite_tensors": 7.9997076988220215, + "nonfinite_indices": [] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.8277368545532227, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.4217219352722168, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.6493300199508667, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.8331910371780396, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.485217571258545, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 9.091630935668945, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 2.7201173305511475, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 10.289445877075195, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.8278238773345947, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.3897894620895386, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.6493247747421265, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.1112847328186035, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.0261825043708086, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.1002056896686554, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 0.03202896469662201, + "step": 2, + "teaching_rms": 0.0030266363158957077, + "traffic_rms": 0.03201675847811013 + }, + { + "batch_loss": 4.823637008666992, + "innovation_rms": 0.0050067298630821185, + "instruction_rms": 0.000896361821657787, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 3.1607327461242676, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": true, + "max_abs_over_finite_tensors": 42.92159652709961, + "nonfinite_indices": [] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 2.960029125213623, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.44191792607307434, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.919400930404663, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.8331910371780396, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 3.084751605987549, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 15.995339393615723, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 4.417537689208984, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 16.97420310974121, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 2.9600157737731934, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.376192182302475, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.9193487167358398, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.1031781435012817, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.0261825043708086, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.1002056896686554, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 0.05582828645223682, + "step": 3, + "teaching_rms": 0.005006729885788924, + "traffic_rms": 0.055838764401403174 + }, + { + "batch_loss": 12.992831230163574, + "innovation_rms": 0.009176180255596859, + "instruction_rms": 0.000969024823408796, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 9.619744300842285, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": true, + "max_abs_over_finite_tensors": 299.88580322265625, + "nonfinite_indices": [] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 4.021809101104736, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.8468732237815857, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 6.011368751525879, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.8331910371780396, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 6.185171127319336, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 31.004196166992188, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 7.410898208618164, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 30.688411712646484, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 4.021817207336426, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.5456865429878235, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 6.011358737945557, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.137107491493225, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.0261825043708086, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.1002056896686554, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 0.09631751081446588, + "step": 4, + "teaching_rms": 0.009176180124162098, + "traffic_rms": 0.0963438970013651 + }, + { + "batch_loss": 40.78102493286133, + "innovation_rms": 0.0170937088699459, + "instruction_rms": 0.0011944011917212948, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 25.037996292114258, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": true, + "max_abs_over_finite_tensors": 3124.3330078125, + "nonfinite_indices": [] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 3.6131322383880615, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.4538477659225464, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 14.207832336425781, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.1268450021743774, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 12.339202880859375, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 61.540321350097656, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 13.166619300842285, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 57.78557586669922, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 9.865116119384766, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.3744301795959473, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 14.207892417907715, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.409882664680481, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.0261825043708086, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.1002056896686554, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 0.17034175872804092, + "step": 5, + "teaching_rms": 0.017093708838716058, + "traffic_rms": 0.17040170092793797 + }, + { + "batch_loss": 142.98423767089844, + "innovation_rms": 0.03204800427212605, + "instruction_rms": 0.002545680320603145, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 60.35129928588867, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": true, + "max_abs_over_finite_tensors": 23701.150390625, + "nonfinite_indices": [] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 9.707615852355957, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 2.425274610519409, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 22.89688491821289, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.4470754861831665, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 24.551959991455078, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 122.12757873535156, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 24.273162841796875, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 111.06543731689453, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 21.945295333862305, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 2.341252326965332, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 22.89682388305664, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 2.3092474937438965, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.0261825043708086, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.1002056896686554, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 0.30763197201412334, + "step": 6, + "teaching_rms": 0.03204800457411708, + "traffic_rms": 0.30777346909107683 + }, + { + "batch_loss": 957.0724487304688, + "innovation_rms": 0.060249450271203195, + "instruction_rms": 0.006955649734660194, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 141.80287170410156, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": true, + "max_abs_over_finite_tensors": 149655.578125, + "nonfinite_indices": [] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 30.096025466918945, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 3.7502431869506836, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 45.717063903808594, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 3.3656625747680664, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 48.7338981628418, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 241.81936645507812, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 45.16677474975586, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 208.93612670898438, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 67.373291015625, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 3.854081392288208, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 45.717010498046875, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 6.786167144775391, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.0261825043708086, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.1002056896686554, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 0.5666255595996695, + "step": 7, + "teaching_rms": 0.06024944964375605, + "traffic_rms": 0.5672159518177116 + }, + { + "batch_loss": 2516.896728515625, + "innovation_rms": 0.11463434948054016, + "instruction_rms": 0.010416129946149344, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 676.8906860351562, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": true, + "max_abs_over_finite_tensors": 5953663.5, + "nonfinite_indices": [] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 73.1310043334961, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 7.238974571228027, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 90.01850128173828, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 7.138430118560791, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 96.53346252441406, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 477.9956359863281, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 84.43772888183594, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 392.7095031738281, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 123.81836700439453, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 7.175546646118164, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 90.01856994628906, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 13.721455574035645, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.0261825043708086, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.1002056896686554, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 1.065368551233577, + "step": 8, + "teaching_rms": 0.11463434722593957, + "traffic_rms": 1.0661871334601167 + }, + { + "batch_loss": 12562.6298828125, + "innovation_rms": 0.21906842247060332, + "instruction_rms": 0.023890223646902072, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 2941.41455078125, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": true, + "max_abs_over_finite_tensors": 73791240.0, + "nonfinite_indices": [] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 66.87805938720703, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 13.542716979980469, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 166.09530639648438, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 13.8262357711792, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 191.1508026123047, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 946.17333984375, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 159.6721649169922, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 752.3444213867188, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 225.3828582763672, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 13.429659843444824, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 166.0952911376953, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 24.92508888244629, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.0261825043708086, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.1002056896686554, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 2.024478018763723, + "step": 9, + "teaching_rms": 0.2190684279551091, + "traffic_rms": 2.026687289958201 + }, + { + "batch_loss": 41096.91796875, + "innovation_rms": 0.42033320389995077, + "instruction_rms": 0.042051454657006576, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 8817.6533203125, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": true, + "max_abs_over_finite_tensors": 496269408.0, + "nonfinite_indices": [] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 61.62916564941406, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 21.990135192871094, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 239.3929901123047, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 19.989152908325195, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 376.84564208984375, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1856.9482421875, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 302.9001770019531, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1432.2801513671875, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 685.1502685546875, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 22.007396697998047, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 239.39297485351562, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 67.72527313232422, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.0261825043708086, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.1002056896686554, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 3.870581658548543, + "step": 10, + "teaching_rms": 0.4203332023646312, + "traffic_rms": 3.874389597577872 + }, + { + "batch_loss": 124862.0859375, + "innovation_rms": 0.8093676972566564, + "instruction_rms": 0.06362515517174869, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 22300.4375, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": true, + "max_abs_over_finite_tensors": 2603202816.0, + "nonfinite_indices": [] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 133.48045349121094, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 52.99477767944336, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 665.3596801757812, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 25.835433959960938, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 742.773193359375, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 3659.275634765625, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 577.0498046875, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 2741.49609375, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 616.894287109375, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 52.847938537597656, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 665.3597412109375, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 129.41470336914062, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.0261825043708086, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.1002056896686554, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 7.441567999133049, + "step": 11, + "teaching_rms": 0.8093676670012973, + "traffic_rms": 7.447661733428294 + }, + { + "batch_loss": 401691.25, + "innovation_rms": 1.559591518893618, + "instruction_rms": 0.13834452550643744, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 50808.71484375, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": true, + "max_abs_over_finite_tensors": 11259937792.0, + "nonfinite_indices": [] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 774.3778076171875, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 102.32852172851562, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 855.5213012695312, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 79.6632308959961, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 1463.8865966796875, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 7211.1337890625, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 1105.9173583984375, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 5345.939453125, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 2050.651123046875, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 102.18168640136719, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 855.5213623046875, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 208.39300537109375, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.0261825043708086, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.1002056896686554, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 14.300715009784797, + "step": 12, + "teaching_rms": 1.5595915724301312, + "traffic_rms": 14.310771240033576 + }, + { + "batch_loss": 2402618.0, + "innovation_rms": 2.9904745645326396, + "instruction_rms": 0.4212605830856891, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 148625.90625, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": true, + "max_abs_over_finite_tensors": 147667501056.0, + "nonfinite_indices": [] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 3336.0, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 193.5659637451172, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 2443.235595703125, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 351.6991882324219, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 2879.03515625, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 14151.486328125, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 2126.80322265625, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 10208.857421875, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 3336.000244140625, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 193.52435302734375, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 2443.23583984375, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 391.2945556640625, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.0261825043708086, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.1002056896686554, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 27.351043812320068, + "step": 13, + "teaching_rms": 2.9904746722602304, + "traffic_rms": 27.38133589764681 + }, + { + "batch_loss": 9321861.0, + "innovation_rms": 5.855092715569528, + "instruction_rms": 1.1294882783900348, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 1116235.875, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": true, + "max_abs_over_finite_tensors": 9418870620160.0, + "nonfinite_indices": [] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 7494.45263671875, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 360.65252685546875, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 4648.84375, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 774.2950439453125, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 5654.7578125, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 27815.66015625, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 4100.36181640625, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 19735.5859375, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 7494.45263671875, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 360.758056640625, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 4648.84375, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 774.3240356445312, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.0261825043708086, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.1002056896686554, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 53.08121896419405, + "step": 14, + "teaching_rms": 5.855092710590196, + "traffic_rms": 53.136161440073835 + }, + { + "batch_loss": 38468964.0, + "innovation_rms": 11.327182100279796, + "instruction_rms": 2.083964751623628, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 13876809.0, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": true, + "max_abs_over_finite_tensors": 1243749514477568.0, + "nonfinite_indices": [] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 23501.75390625, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 712.19140625, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 9306.037109375, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 2351.82763671875, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 11108.0244140625, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 54967.109375, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 7953.603515625, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 38532.41796875, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 23501.75390625, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 712.138427734375, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 9306.037109375, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 2351.876220703125, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.0261825043708086, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.1002056896686554, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 102.95399884310817, + "step": 15, + "teaching_rms": 11.327181863943343, + "traffic_rms": 103.07497727430156 + }, + { + "batch_loss": 125100336.0, + "innovation_rms": 21.898488531878794, + "instruction_rms": 3.7994835882984095, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 65247588.0, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": true, + "max_abs_over_finite_tensors": 2.014340520817459e+16, + "nonfinite_indices": [] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 21145.576171875, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 1586.7659912109375, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 19559.03515625, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 4466.38525390625, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 21963.986328125, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 108559.609375, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 15210.0947265625, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 72564.90625, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 21145.576171875, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 1586.98095703125, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 19559.03515625, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 4466.43408203125, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.0261825043708086, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.1002056896686554, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 199.1116505559766, + "step": 16, + "teaching_rms": 21.898488039576947, + "traffic_rms": 199.3150802870113 + }, + { + "batch_loss": 406332640.0, + "innovation_rms": 43.4989279872756, + "instruction_rms": 11.814265867935642, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 283797792.0, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": true, + "max_abs_over_finite_tensors": 2.5248195459783066e+17, + "nonfinite_indices": [] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 19044.82421875, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 3288.462890625, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 39172.61328125, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 6370.86767578125, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 43422.44921875, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 214584.625, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 28707.908203125, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 134978.125, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 37808.76953125, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 3288.47998046875, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 39172.61328125, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 6370.91650390625, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 34.87675857543945, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.10143683850765228, + "nonfinite_indices": [] + } + }, + "predictor_mse": 1786.4998506803806, + "predictor_updated": true, + "raw_apical_rms": 386.07989562731865, + "step": 17, + "teaching_rms": 43.49892816457474, + "traffic_rms": 386.30057467395926 + }, + { + "batch_loss": 1876102784.0, + "innovation_rms": 78.36551715918127, + "instruction_rms": 35.43632012032303, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 842817408.0, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.7595716851130696e+18, + "nonfinite_indices": [] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 17153.703125, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 5961.1337890625, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 70390.171875, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 8086.23779296875, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 82438.0, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 390155.5, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 52523.421875, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 238155.125, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 49051.6484375, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 5961.15087890625, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 70390.171875, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 8086.28662109375, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 34.87675857543945, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.10143683850765228, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 751.0638513169803, + "step": 18, + "teaching_rms": 78.36551690097482, + "traffic_rms": 751.1796847962376 + }, + { + "batch_loss": 7756428800.0, + "innovation_rms": 155.55207735763057, + "instruction_rms": 80.24130903932937, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 2082047232.0, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": true, + "max_abs_over_finite_tensors": 7.09434424128281e+18, + "nonfinite_indices": [] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 58303.796875, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 11256.408203125, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 145102.84375, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 9630.576171875, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 155694.796875, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 732568.0, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 95824.796875, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 433013.75, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 58303.796875, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 11256.26171875, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 145102.84375, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 9994.77734375, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 34.87675857543945, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.10143683850765228, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 1406.7281071438276, + "step": 19, + "teaching_rms": 155.5520763596266, + "traffic_rms": 1406.5372413991738 + }, + { + "batch_loss": 25476413440.0, + "innovation_rms": 352.0562300268429, + "instruction_rms": 247.02279571324402, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 4634672640.0, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": true, + "max_abs_over_finite_tensors": 9.528446931144016e+19, + "nonfinite_indices": [] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 452802.875, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 21397.203125, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 260395.046875, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 45520.20703125, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 294313.8125, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1386190.0, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 175403.9375, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 809516.1875, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 452802.875, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 21397.056640625, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 260395.046875, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 45520.37109375, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 34.87675857543945, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.10143683850765228, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 2624.8605174338586, + "step": 20, + "teaching_rms": 352.0562290318634, + "traffic_rms": 2616.9130921938345 + }, + { + "batch_loss": 75591802880.0, + "innovation_rms": 538.3110576557918, + "instruction_rms": 256.499206197742, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 17902606336.0, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": true, + "max_abs_over_finite_tensors": 3.590727301251891e+21, + "nonfinite_indices": [] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 407473.6875, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 38487.71875, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 469803.28125, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 86267.578125, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 556835.0625, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 2625212.75, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 325503.3125, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1525460.75, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 407473.6875, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 38487.609375, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 469803.28125, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 86267.7421875, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 34.87675857543945, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.10143683850765228, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 4898.42575712612, + "step": 21, + "teaching_rms": 538.3110524500474, + "traffic_rms": 4897.572550319883 + }, + { + "batch_loss": 388053598208.0, + "innovation_rms": 918.2520110837629, + "instruction_rms": 264.953018332314, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 76294840320.0, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": true, + "max_abs_over_finite_tensors": 3.54064873706541e+22, + "nonfinite_indices": [] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 366310.5625, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 73924.09375, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 934063.0625, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 122898.6328125, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 1054451.625, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 4976166.0, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 611740.375, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 2862370.5, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 704088.25, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 73924.125, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 934063.0625, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 122898.796875, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 34.87675857543945, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.10143683850765228, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 9112.967106540247, + "step": 22, + "teaching_rms": 918.2520124709389, + "traffic_rms": 9125.282734985898 + }, + { + "batch_loss": 1290553917440.0, + "innovation_rms": 1850.1333710003532, + "instruction_rms": 841.9603339924298, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 221404217344.0, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": true, + "max_abs_over_finite_tensors": 2.3238449777882044e+23, + "nonfinite_indices": [] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 855653.3125, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 146958.328125, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1795860.0, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 155803.484375, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 1997824.625, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 9433730.0, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 1150768.375, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 5390280.0, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1107408.875, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 146958.359375, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1795860.0, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 155803.65625, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 34.87675857543945, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.10143683850765228, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 17049.091355620974, + "step": 23, + "teaching_rms": 1850.1333216593885, + "traffic_rms": 17057.213157419923 + }, + { + "batch_loss": 3708327034880.0, + "innovation_rms": 3592.879400879788, + "instruction_rms": 1803.5179418880082, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 541278404608.0, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.4978581808474584e+24, + "nonfinite_indices": [] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 2240393.75, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 251259.203125, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1943588.25, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 227409.078125, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 3786846.75, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 17890222.0, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 2146363.75, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 9955953.0, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 2401297.25, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 251259.09375, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1943588.25, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 245764.578125, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 34.87675857543945, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.10143683850765228, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 32044.791877423566, + "step": 24, + "teaching_rms": 3592.879415817244, + "traffic_rms": 32038.190557034522 + }, + { + "batch_loss": 9880979111936.0, + "innovation_rms": 6404.828404834937, + "instruction_rms": 2625.7189367129095, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 1198418558976.0, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.7887215126622784e+25, + "nonfinite_indices": [] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 5847975.0, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 543186.5, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 6718434.0, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 600033.25, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 7179247.5, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 33924008.0, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 4039358.5, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 18929946.0, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 5847975.0, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 543186.25, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 6718434.0, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 600033.125, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 34.87675857543945, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.10143683850765228, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 59989.34354345325, + "step": 25, + "teaching_rms": 6404.828375666216, + "traffic_rms": 59994.730643837574 + }, + { + "batch_loss": 51177890775040.0, + "innovation_rms": 11406.657778535764, + "instruction_rms": 3240.587408590824, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 5008838361088.0, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.269158497298987e+26, + "nonfinite_indices": [] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 5262193.5, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 1006085.4375, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 11107101.0, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 1126252.625, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 13615999.0, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 64367512.0, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 7605074.5, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 35657160.0, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 5472081.5, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 1006085.1875, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 11107101.0, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 1126252.5, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 34.87675857543945, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.10143683850765228, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 112724.71282270785, + "step": 26, + "teaching_rms": 11406.65778627819, + "traffic_rms": 112850.36394028923 + }, + { + "batch_loss": 215583400394752.0, + "innovation_rms": 21324.80421108959, + "instruction_rms": 5960.228649046364, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 21327102607360.0, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": true, + "max_abs_over_finite_tensors": 7.980302031829473e+26, + "nonfinite_indices": [] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 4734888.0, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 1642832.375, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 22819428.0, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 1599741.375, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 25789916.0, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 121739160.0, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 14316775.0, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 67117008.0, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 4852504.0, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 1642832.25, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 22819428.0, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 1599741.25, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 34.87675857543945, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.10143683850765228, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 211604.99588003894, + "step": 27, + "teaching_rms": 21324.804192485004, + "traffic_rms": 211908.0223644778 + }, + { + "batch_loss": 621414121996288.0, + "innovation_rms": 41268.64956440379, + "instruction_rms": 14598.393613235552, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 61453298040832.0, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": true, + "max_abs_over_finite_tensors": 6.367699175674086e+27, + "nonfinite_indices": [] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 14747591.0, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 3189767.5, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 42317956.0, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 2024561.75, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 48855004.0, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 230650896.0, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 26960114.0, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 126433384.0, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 18738458.0, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 3189767.5, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 42317956.0, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 2024561.625, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 34.87675857543945, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.10143683850765228, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 397754.5371817846, + "step": 28, + "teaching_rms": 41268.64958126046, + "traffic_rms": 398081.2764087075 + }, + { + "batch_loss": 2447422152245248.0, + "innovation_rms": 77244.16429231007, + "instruction_rms": 26867.638212735314, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 151000413372416.0, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": true, + "max_abs_over_finite_tensors": 5.294007528911071e+28, + "nonfinite_indices": [] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 20010302.0, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 5933548.5, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 82186424.0, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 2807743.5, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 92529736.0, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 436747328.0, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 50822772.0, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 238626576.0, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 24096832.0, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 5933548.5, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 82186424.0, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 3557889.5, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 34.87675857543945, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.10143683850765228, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 747411.110312879, + "step": 29, + "teaching_rms": 77244.16438912335, + "traffic_rms": 748172.8964952126 + }, + { + "batch_loss": 8588165065474048.0, + "innovation_rms": 174703.42735544467, + "instruction_rms": 110193.72896541958, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 338035468140544.0, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.9081184565950123e+29, + "nonfinite_indices": [] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 262652112.0, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 11140782.0, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 152254656.0, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 26274578.0, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 175166640.0, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 826368960.0, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 95748584.0, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 449258144.0, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 262652112.0, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 11140782.0, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 152254656.0, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 26274578.0, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 34.87675857543945, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.10143683850765228, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 1402575.6587049598, + "step": 30, + "teaching_rms": 174703.42967689433, + "traffic_rms": 1400577.66314253 + }, + { + "batch_loss": 2.806604361629696e+16, + "innovation_rms": 264924.50688549323, + "instruction_rms": 72916.09453158562, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 4250259639238656.0, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": true, + "max_abs_over_finite_tensors": 7.163915969585317e+31, + "nonfinite_indices": [] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 623829760.0, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 24135010.0, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 305817600.0, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 62872944.0, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 331736832.0, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1565701888.0, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 180599616.0, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 848510336.0, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 623829760.0, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 24135010.0, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 305817600.0, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 62872944.0, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 34.87675857543945, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.10143683850765228, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 2624685.649440279, + "step": 31, + "teaching_rms": 264924.5180506097, + "traffic_rms": 2627727.361342049 + }, + { + "batch_loss": 1.0868606298069402e+17, + "innovation_rms": 491477.6974776303, + "instruction_rms": 120402.69522911347, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.829003018318643e+16, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": false, + "max_abs_over_finite_tensors": 1.0629836176722922e+32, + "nonfinite_indices": [ + 2 + ] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 586455168.0, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 44699012.0, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 570195584.0, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 119017000.0, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 628130688.0, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 2963938816.0, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 337183072.0, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1565834624.0, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 586455168.0, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 44699012.0, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 570195584.0, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 119017000.0, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 34.87675857543945, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.10143683850765228, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 4917847.6396826105, + "step": 32, + "teaching_rms": 491477.7072157871, + "traffic_rms": 4924786.6371714305 + }, + { + "batch_loss": 4.558429179871232e+17, + "innovation_rms": 932941.5113839154, + "instruction_rms": 275044.67365834606, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 5.683223132635136e+16, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": false, + "max_abs_over_finite_tensors": 9.02863172403141e+32, + "nonfinite_indices": [ + 1, + 2 + ] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 4397747200.0, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 82793136.0, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1101971200.0, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 440058016.0, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 1188713216.0, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 5605825536.0, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 632294784.0, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 2951117312.0, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 4397747200.0, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 82793136.0, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1101971200.0, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 440058016.0, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 734350.5625, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.1025448739528656, + "nonfinite_indices": [] + } + }, + "predictor_mse": 828895664443.6073, + "predictor_updated": true, + "raw_apical_rms": 9230354.221622748, + "step": 33, + "teaching_rms": 932941.516657185, + "traffic_rms": 9245036.648267511 + }, + { + "batch_loss": 1.4377044994886205e+18, + "innovation_rms": 1630905.8140535704, + "instruction_rms": 778996.5591467236, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 2.2015425115691418e+17, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": false, + "max_abs_over_finite_tensors": 3.592052080173146e+32, + "nonfinite_indices": [ + 1, + 2, + 3, + 5, + 7 + ] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 3957928448.0, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 160324352.0, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 2097789568.0, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 835850880.0, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 2168863232.0, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 9801499648.0, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 1149754368.0, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 5174596096.0, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 3957928448.0, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 160324352.0, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 2097789568.0, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 835850880.0, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 734350.5625, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.1025448739528656, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 17328520.098337166, + "step": 34, + "teaching_rms": 1630905.7658985215, + "traffic_rms": 17342804.19631902 + }, + { + "batch_loss": 4.715965350473105e+18, + "innovation_rms": 3071681.6172405006, + "instruction_rms": 1563839.6575119062, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 9.017156019725271e+17, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": false, + "max_abs_over_finite_tensors": 7.353650106989244e+32, + "nonfinite_indices": [ + 1, + 2, + 3, + 5, + 7, + 11 + ] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 3562051840.0, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 286753952.0, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 3678433536.0, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 1192056064.0, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 3935846144.0, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 17669828608.0, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 2080810368.0, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 9310560256.0, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 3562051840.0, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 286753952.0, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 3678433536.0, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 1192056064.0, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 734350.5625, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.1025448739528656, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 31278835.776587892, + "step": 35, + "teaching_rms": 3071681.6784642367, + "traffic_rms": 31296740.980714537 + }, + { + "batch_loss": 1.169217366442115e+19, + "innovation_rms": 6107161.256647392, + "instruction_rms": 3701370.7145119845, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 2.706031706619511e+18, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": false, + "max_abs_over_finite_tensors": 1.0540743759804995e+33, + "nonfinite_indices": [ + 1, + 2, + 3, + 5, + 6, + 7, + 9, + 11, + 13, + 15 + ] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 3205727232.0, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 547618624.0, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 6541569536.0, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 1512628736.0, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 7154016256.0, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 32181702656.0, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 3773047552.0, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 16922372096.0, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 3205727232.0, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 547618624.0, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 6541569536.0, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 1512628736.0, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 734350.5625, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.1025448739528656, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 56555827.00227312, + "step": 36, + "teaching_rms": 6107161.150270437, + "traffic_rms": 56512560.933729336 + }, + { + "batch_loss": 4.465618537505253e+19, + "innovation_rms": 22884459.671562485, + "instruction_rms": 21112871.173231423, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 6.961380300135858e+18, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": false, + "max_abs_over_finite_tensors": 1.8265280908081528e+30, + "nonfinite_indices": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 9, + 11, + 13, + 15, + 17 + ] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 2885003264.0, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 911123648.0, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 12269554688.0, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 1801129088.0, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 13023738880.0, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 58697228288.0, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 6843403264.0, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 30703554560.0, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 7166167040.0, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 911123648.0, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 12269554688.0, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 1801129088.0, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 734350.5625, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.1025448739528656, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 104305906.4211801, + "step": 37, + "teaching_rms": 22884459.649897385, + "traffic_rms": 102312342.55797268 + }, + { + "batch_loss": 1.6445741982710707e+20, + "innovation_rms": 36050992.676571175, + "instruction_rms": 32294724.55443593, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 2.013098917927636e+19, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": false, + "max_abs_over_finite_tensors": 7.53895957589338e+31, + "nonfinite_indices": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 9, + 11, + 13, + 15, + 17 + ] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 2596322560.0, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 1790105344.0, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 20802514944.0, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 2060761344.0, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 23738376192.0, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 107146371072.0, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 12429577216.0, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 55861739520.0, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 32268038144.0, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 1790105344.0, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 20802514944.0, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 3230062336.0, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 734350.5625, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.1025448739528656, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 187856251.11052728, + "step": 38, + "teaching_rms": 36050991.8669079, + "traffic_rms": 185393605.01866084 + }, + { + "batch_loss": 4.624345003661229e+20, + "innovation_rms": 76675481.30660689, + "instruction_rms": 70901461.77702333, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 5.025385624666479e+19, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": false, + "max_abs_over_finite_tensors": 1.0512674243126724e+33, + "nonfinite_indices": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 9, + 11, + 13, + 15, + 17 + ] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 2336484096.0, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 2979149056.0, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 39319547904.0, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 2294409728.0, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 43316940800.0, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 195785654272.0, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 22614042624.0, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 101844647936.0, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 29044692992.0, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 2979149056.0, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 39319547904.0, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 6134531584.0, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 734350.5625, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.1025448739528656, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 342914968.22547644, + "step": 39, + "teaching_rms": 76675481.32503909, + "traffic_rms": 336020683.30498403 + }, + { + "batch_loss": 2.186346748500859e+21, + "innovation_rms": 65680749.726455815, + "instruction_rms": 39341059.124830544, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.1486039423468962e+20, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": false, + "max_abs_over_finite_tensors": 7.046196866257376e+32, + "nonfinite_indices": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 11, + 13, + 15, + 17 + ] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 2102606080.0, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 5449299456.0, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 63949631488.0, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 2504670464.0, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 79169110016.0, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 358521700352.0, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 41168171008.0, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 185541296128.0, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 27971747840.0, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 5449299456.0, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 63949631488.0, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 8746092544.0, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 734350.5625, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.1025448739528656, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 609585854.6884223, + "step": 40, + "teaching_rms": 65680749.61331441, + "traffic_rms": 609668932.6573776 + }, + { + "batch_loss": 6.348771119550322e+21, + "innovation_rms": 105919759.12961195, + "instruction_rms": 45230820.53852924, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 2.4797160836711213e+20, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": false, + "max_abs_over_finite_tensors": 2.7000454871720327e+32, + "nonfinite_indices": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 13, + 15, + 17 + ] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1892094976.0, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 9590532096.0, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 133608169472.0, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 2693880064.0, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 144703111168.0, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 655340011520.0, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 75010285568.0, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 338421121024.0, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 41843515392.0, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 9590532096.0, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 133608169472.0, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 11087418368.0, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 734350.5625, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.1025448739528656, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 1105045016.2017956, + "step": 41, + "teaching_rms": 105919759.13304187, + "traffic_rms": 1106302175.7715905 + }, + { + "batch_loss": 1.9624367822272526e+22, + "innovation_rms": 188750645.33160675, + "instruction_rms": 72718352.50208162, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 5.1583899398804026e+20, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": false, + "max_abs_over_finite_tensors": 8.749641045940287e+32, + "nonfinite_indices": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 15, + 17 + ] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1702615936.0, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 18135588864.0, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 245362851840.0, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 2864141568.0, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 264566013952.0, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1198629060608.0, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 135753531392.0, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 607432409088.0, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 38084112384.0, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 18135588864.0, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 245362851840.0, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 13186196480.0, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 734350.5625, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.1025448739528656, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 2005978197.4074543, + "step": 42, + "teaching_rms": 188750644.1394708, + "traffic_rms": 2008378415.9746413 + }, + { + "batch_loss": 6.90186595849705e+22, + "innovation_rms": 343104194.1755184, + "instruction_rms": 133636035.70609435, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.0460816958530624e+21, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": false, + "max_abs_over_finite_tensors": 2.852642177767109e+33, + "nonfinite_indices": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 15, + 17 + ] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1532067840.0, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 33692041216.0, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 462867038208.0, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 3017348352.0, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 483656925184.0, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 2190909112320.0, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 242685739008.0, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1071206236160.0, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 303714336768.0, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 33692041216.0, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 462867038208.0, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 30562426880.0, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 734350.5625, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.1025448739528656, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 3643555534.23608, + "step": 43, + "teaching_rms": 343104193.5547307, + "traffic_rms": 3648303915.502025 + }, + { + "batch_loss": 1.7492436716989296e+23, + "innovation_rms": 665062902.8597152, + "instruction_rms": 330469266.29840773, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 2.0789295642010966e+21, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": false, + "max_abs_over_finite_tensors": 3.242531509196918e+32, + "nonfinite_indices": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 17 + ] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 5199283712.0, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 68234629120.0, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 751146303488.0, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 3155204352.0, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 884552433664.0, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 4008955084800.0, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 440363778048.0, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1976780455936.0, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 374071689216.0, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 68234629120.0, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 751146303488.0, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 57989234688.0, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 734350.5625, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.1025448739528656, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 6624699095.451093, + "step": 44, + "teaching_rms": 665062913.9133137, + "traffic_rms": 6626466945.365421 + }, + { + "batch_loss": 8.08633490827765e+23, + "innovation_rms": 1182546426.717914, + "instruction_rms": 561724426.2461624, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 4.07595694705381e+21, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": false, + "max_abs_over_finite_tensors": 3.088351222492212e+33, + "nonfinite_indices": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 17 + ] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 8062496256.0, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 102501179392.0, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1443043213312.0, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 3279243264.0, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 1617922359296.0, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 7333699649536.0, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 801693237248.0, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 3613294854144.0, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 437574762496.0, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 102501179392.0, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1443043213312.0, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 82840969216.0, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 734350.5625, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.1025448739528656, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 12019784905.352076, + "step": 45, + "teaching_rms": 1182546476.8745863, + "traffic_rms": 12032101840.046946 + }, + { + "batch_loss": 2.3439986581782814e+24, + "innovation_rms": 2038196397.8221831, + "instruction_rms": 745738997.7654105, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 7.887741747165321e+21, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": false, + "max_abs_over_finite_tensors": 8.739012599090855e+22, + "nonfinite_indices": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 8670181376.0, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 197775720448.0, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 2715475509248.0, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 3390845440.0, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 2957072400384.0, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 13391500410880.0, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 1462042624000.0, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 6603493343232.0, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 863094374400.0, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 197775720448.0, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 2715475509248.0, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 105448300544.0, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 734350.5625, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.1025448739528656, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 21837507161.30762, + "step": 46, + "teaching_rms": 2038196384.971172, + "traffic_rms": 21865384267.01417 + }, + { + "batch_loss": 7.855109582999719e+24, + "innovation_rms": 3553559734.945827, + "instruction_rms": 868244453.4951788, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.5138343492447843e+22, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": false, + "max_abs_over_finite_tensors": 2.0797704143990243e+23, + "nonfinite_indices": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 10974387200.0, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 390880559104.0, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 5104688168960.0, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 3491253504.0, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 5401906511872.0, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 24448342163456.0, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 2667921801216.0, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 12058791772160.0, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 2246027771904.0, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 390880559104.0, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 5104688168960.0, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 213666430976.0, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 734350.5625, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.1025448739528656, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 39664893553.34978, + "step": 47, + "teaching_rms": 3553559734.903124, + "traffic_rms": 39730032199.1302 + }, + { + "batch_loss": 2.2973291450707742e+25, + "innovation_rms": 6402711945.945457, + "instruction_rms": 1272611182.8580687, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 2.8807261505676496e+22, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": false, + "max_abs_over_finite_tensors": 9.839208661420225e+23, + "nonfinite_indices": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 9876857856.0, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 760697126912.0, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 8912675799040.0, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 3581585664.0, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 9888673562624.0, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 44867669458944.0, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 4867270639616.0, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 21993487335424.0, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 2806785507328.0, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 760697126912.0, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 8912675799040.0, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 414114775040.0, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 734350.5625, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.1025448739528656, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 72062335392.84178, + "step": 48, + "teaching_rms": 6402711976.273842, + "traffic_rms": 72171629936.54803 + }, + { + "batch_loss": 7.350969406129424e+25, + "innovation_rms": 11520202117.415573, + "instruction_rms": 1613939528.6153402, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 5.451433059626406e+22, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": false, + "max_abs_over_finite_tensors": 4.284910882303284e+24, + "nonfinite_indices": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 8888983552.0, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 1255557562368.0, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 15863276109824.0, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 3662849024.0, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 18071131521024.0, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 81824587972608.0, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 8877832667136.0, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 40105616080896.0, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 3118330806272.0, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 1255557562368.0, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 15863276109824.0, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 591650750464.0, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 9421697024.0, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.10354210436344147, + "nonfinite_indices": [] + } + }, + "predictor_mse": 1.3507288403250502e+20, + "predictor_updated": true, + "raw_apical_rms": 130926686259.11829, + "step": 49, + "teaching_rms": 11520202119.526861, + "traffic_rms": 131130416276.4666 + }, + { + "batch_loss": 2.9509546108595127e+26, + "innovation_rms": 17882632778.5258, + "instruction_rms": 3296595429.4604616, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.0272283808787427e+23, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": false, + "max_abs_over_finite_tensors": 1.0982356706317332e+25, + "nonfinite_indices": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 7999807488.0, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 2155823497216.0, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 29070572126208.0, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 3820654848.0, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 31901458890752.0, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 138303282085888.0, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 15479623122944.0, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 66149693784064.0, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 28049458331648.0, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 2155823497216.0, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 29070572126208.0, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 2816138018816.0, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 9421697024.0, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.10354210436344147, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 237850947392.67108, + "step": 50, + "teaching_rms": 17882632775.724625, + "traffic_rms": 238300059231.5897 + }, + { + "batch_loss": 8.036544678770688e+26, + "innovation_rms": 32126992994.75963, + "instruction_rms": 5083706377.036581, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.8902985737559947e+23, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": false, + "max_abs_over_finite_tensors": 1.284886744698554e+26, + "nonfinite_indices": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 7199469056.0, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 4121473056768.0, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 54616251170816.0, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 4419167744.0, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 56032395001856.0, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 241309365305344.0, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 26805425668096.0, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 115532799934464.0, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 25233977245696.0, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 4121473056768.0, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 54616251170816.0, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 5339535638528.0, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 9421697024.0, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.10354210436344147, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 418180373247.3266, + "step": 51, + "teaching_rms": 32126992160.08077, + "traffic_rms": 418879771682.8226 + }, + { + "batch_loss": 2.871050362188439e+27, + "innovation_rms": 56834209207.51279, + "instruction_rms": 9963569430.28195, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 3.4316395134088735e+23, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": false, + "max_abs_over_finite_tensors": 5.781939714831011e+26, + "nonfinite_indices": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 6479092224.0, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 7876136402944.0, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 95757629652992.0, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 5039434240.0, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 98496443056128.0, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 424640463765504.0, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 47145629515776.0, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 203402051059712.0, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 23639472734208.0, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 7876136402944.0, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 95757629652992.0, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 7608550817792.0, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 9421697024.0, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.10354210436344147, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 730837205123.7609, + "step": 52, + "teaching_rms": 56834207915.74035, + "traffic_rms": 732257241462.028 + }, + { + "batch_loss": 1.0431297895366657e+28, + "innovation_rms": 100253947710.13574, + "instruction_rms": 20196949753.126984, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 6.177814590285329e+23, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": false, + "max_abs_over_finite_tensors": 1.4391149174909634e+27, + "nonfinite_indices": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 5830688256.0, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 11649742274560.0, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 168526295859200.0, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 5597623808.0, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 173407509413888.0, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 749110613245952.0, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 83069558063104.0, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 359239319027712.0, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 51376499458048.0, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 11649742274560.0, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 168526295859200.0, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 9648919805952.0, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 9421697024.0, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.10354210436344147, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 1279107993806.6724, + "step": 53, + "teaching_rms": 100253947702.4989, + "traffic_rms": 1282025981652.1418 + }, + { + "batch_loss": 2.3914876116286937e+28, + "innovation_rms": 176567217492.92838, + "instruction_rms": 28811875108.219406, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.1082767090111739e+24, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": false, + "max_abs_over_finite_tensors": 2.863492214632606e+27, + "nonfinite_indices": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 5247066624.0, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 22268983902208.0, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 302314929782784.0, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 6099938304.0, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 305807543500800.0, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1324000173096960.0, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 146550021947392.0, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 634804689174528.0, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 47077564350464.0, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 22268983902208.0, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 302314929782784.0, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 11457151369216.0, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 9421697024.0, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.10354210436344147, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 2242629095755.0547, + "step": 54, + "teaching_rms": 176567217328.19217, + "traffic_rms": 2246461639803.4985 + }, + { + "batch_loss": 7.605432139189193e+28, + "innovation_rms": 310829627793.7019, + "instruction_rms": 52319848004.14343, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.9841807917321452e+24, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": false, + "max_abs_over_finite_tensors": 4.867099341724065e+27, + "nonfinite_indices": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 4721754624.0, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 46362926252032.0, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 503153069916160.0, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 6551960576.0, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 539864638423040.0, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 2340570882113536.0, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 258929418305536.0, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1123793896472576.0, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 72574268604416.0, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 46362926252032.0, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 503153069916160.0, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 14483177078784.0, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 9421697024.0, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.10354210436344147, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 3937520992124.9077, + "step": 55, + "teaching_rms": 310829627852.5968, + "traffic_rms": 3944421361720.805 + }, + { + "batch_loss": 2.8427010399125598e+29, + "innovation_rms": 546592211630.80585, + "instruction_rms": 104115912248.1194, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 3.543606854423054e+24, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": false, + "max_abs_over_finite_tensors": 7.010979347674779e+27, + "nonfinite_indices": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 4248926720.0, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 67208306753536.0, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 888640678920192.0, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 6958714880.0, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 953703930527744.0, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 4138393189482496.0, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 457944982355968.0, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1990155774722048.0, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 591060782809088.0, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 67208306753536.0, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 888640678920192.0, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 59054693023744.0, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 9421697024.0, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.10354210436344147, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 6919523846933.549, + "step": 56, + "teaching_rms": 546592211025.75885, + "traffic_rms": 6934193406454.941 + }, + { + "batch_loss": 9.348485847767946e+29, + "innovation_rms": 961513169068.5793, + "instruction_rms": 190078842253.97253, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 6.325263450891755e+24, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": false, + "max_abs_over_finite_tensors": 6.034806103552129e+28, + "nonfinite_indices": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 3823339264.0, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 109699965911040.0, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1542020631887872.0, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 7330688512.0, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 1686112355483648.0, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 7324083712688128.0, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 807062137208832.0, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 3491171414310912.0, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 532012465127424.0, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 109699965911040.0, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1542020631887872.0, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 112255941214208.0, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 9421697024.0, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.10354210436344147, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 12168444805305.852, + "step": 57, + "teaching_rms": 961513169934.4867, + "traffic_rms": 12196017919582.797 + }, + { + "batch_loss": 2.7159174379899785e+30, + "innovation_rms": 1690621009380.8713, + "instruction_rms": 313534251355.0122, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.1265537286278865e+25, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": false, + "max_abs_over_finite_tensors": 2.7079383009069308e+29, + "nonfinite_indices": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 3440272128.0, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 197442817490944.0, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 2843641743671296.0, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 7674715648.0, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 2979045419515904.0, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.2929331177193472e+16, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 1424153575948288.0, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 6170914521612288.0, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 478293866315776.0, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 197442817490944.0, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 2843641743671296.0, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 160085326168064.0, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 9421697024.0, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.10354210436344147, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 21399779349622.55, + "step": 58, + "teaching_rms": 1690621008917.6174, + "traffic_rms": 21445443085127.484 + }, + { + "batch_loss": 8.52247428556369e+30, + "innovation_rms": 2970566259885.1143, + "instruction_rms": 544759606758.4581, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 2.004693225265659e+25, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": false, + "max_abs_over_finite_tensors": 7.222456812135463e+29, + "nonfinite_indices": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 3095477248.0, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 395365396250624.0, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 5328727806312448.0, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 7984263168.0, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 5262898305695744.0, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 2.2838527251185664e+16, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 2513916030615552.0, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.0897624815108096e+16, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 978527566430208.0, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 395365396250624.0, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 5328727806312448.0, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 202967252729856.0, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 9421697024.0, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.10354210436344147, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 37631006264012.195, + "step": 59, + "teaching_rms": 2970566176039.0728, + "traffic_rms": 37713366598632.64 + }, + { + "batch_loss": 2.3413394494613402e+31, + "innovation_rms": 5230416958847.151, + "instruction_rms": 888261408253.7384, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 3.5612892115392094e+25, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": false, + "max_abs_over_finite_tensors": 1.379381035634288e+30, + "nonfinite_indices": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 2785131008.0, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 791768228429824.0, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 8833743242395648.0, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 8262776320.0, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 9299680085147648.0, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 4.036782208948634e+16, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 4440718621802496.0, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.9268024301256704e+16, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1552358215516160.0, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 791768228429824.0, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 8833743242395648.0, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 250502726025216.0, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 9421697024.0, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.10354210436344147, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 66199022123638.266, + "step": 60, + "teaching_rms": 5230416958547.659, + "traffic_rms": 66327352946130.055 + }, + { + "batch_loss": 6.503336173942475e+31, + "innovation_rms": 9219152848496.922, + "instruction_rms": 1528860527138.3665, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 6.32307485999956e+25, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": false, + "max_abs_over_finite_tensors": 2.2167256729345215e+30, + "nonfinite_indices": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 2505791488.0, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 1192044819120128.0, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.543866796736512e+16, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 8513355264.0, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.6478851064397824e+16, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 7.179170549753446e+16, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 7741034842816512.0, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 3.300316006265651e+16, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 2.4094891084611584e+16, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 1192044819120128.0, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.543866796736512e+16, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 2404507912765440.0, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 9421697024.0, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.10354210436344147, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 116444718517472.53, + "step": 61, + "teaching_rms": 9219152870246.0, + "traffic_rms": 116645360476532.25 + }, + { + "batch_loss": 2.2974526858584605e+32, + "innovation_rms": 16220192144376.395, + "instruction_rms": 2923296378750.67, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.1216384092020059e+26, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": false, + "max_abs_over_finite_tensors": 3.7686937443787767e+31, + "nonfinite_indices": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 2254360832.0, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 2209484587925504.0, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 2.916575925501952e+16, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 8738791424.0, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 2.914425005880115e+16, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.2665399853396787e+17, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.3635241264545792e+16, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 5.894206099606733e+16, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 2.1688807455719424e+16, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 2209484587925504.0, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 2.916575925501952e+16, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 4573388550963200.0, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 9421697024.0, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.10354210436344147, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 204975231950662.84, + "step": 62, + "teaching_rms": 16220192181462.807, + "traffic_rms": 205377585030774.4 + }, + { + "batch_loss": 6.967999288825816e+32, + "innovation_rms": 28563121813911.613, + "instruction_rms": 5137832953927.019, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.9879938029074712e+26, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": false, + "max_abs_over_finite_tensors": 1.723388167428323e+32, + "nonfinite_indices": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 2028050816.0, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 4131801219989504.0, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 4.602425536361267e+16, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 8941596672.0, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 5.151658223232614e+16, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 2.2372331744028262e+17, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 2.40951960272896e+16, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.0459953903750349e+17, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.952313891618816e+16, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 4131801219989504.0, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 4.602425536361267e+16, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 6525702442582016.0, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 9421697024.0, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.10354210436344147, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 360760987326826.5, + "step": 63, + "teaching_rms": 28563121728797.934, + "traffic_rms": 361452069837501.06 + }, + { + "batch_loss": 2.392130959033026e+33, + "innovation_rms": 49894981592420.02, + "instruction_rms": 9427128438646.328, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 3.522569219027332e+26, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": false, + "max_abs_over_finite_tensors": 0.0, + "nonfinite_indices": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1824351488.0, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 6364986376978432.0, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 8.823271575257088e+16, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 9124031488.0, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 9.103665079202611e+16, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 3.952006941869343e+17, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 4.261185660230042e+16, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.8516662292997734e+17, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.7570172189540352e+16, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 6364986376978432.0, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 8.823271575257088e+16, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 8282719876284416.0, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 9421697024.0, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.10354210436344147, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 631025438736166.4, + "step": 64, + "teaching_rms": 49894981311117.86, + "traffic_rms": 632386391882606.2 + }, + { + "batch_loss": 8.303959420421813e+33, + "innovation_rms": 87337939462290.6, + "instruction_rms": 17647430197379.742, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 6.237914820347238e+26, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": false, + "max_abs_over_finite_tensors": 0.0, + "nonfinite_indices": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1641003904.0, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.1552852140883968e+16, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.577964627998802e+17, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 9288131584.0, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.607795065655132e+17, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 6.974285405550019e+17, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 7.525347061294694e+16, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 3.264161401064653e+17, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.820519384167219e+16, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.1552852140883968e+16, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.577964627998802e+17, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 9863952888496128.0, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 76020409434112.0, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.1044396162033081, + "nonfinite_indices": [] + } + }, + "predictor_mse": 7.818302689677862e+27, + "predictor_updated": true, + "raw_apical_rms": 1105716951359009.1, + "step": 65, + "teaching_rms": 87337938794220.56, + "traffic_rms": 1108417709186258.9 + }, + { + "batch_loss": 2.042142779418334e+34, + "innovation_rms": 131884035089892.84, + "instruction_rms": 26799978249040.1, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.1042741200705416e+27, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": false, + "max_abs_over_finite_tensors": 0.0, + "nonfinite_indices": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1475974656.0, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 2.1152172766920704e+16, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 2.8107697068127027e+17, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 9435728896.0, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 2.7554357533527245e+17, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.147640756417069e+18, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.2906705718974874e+17, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 5.381359001277563e+17, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 7.51363812145234e+16, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 2.1152172766920704e+16, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 2.8107697068127027e+17, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.1286963600490496e+16, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 76020409434112.0, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.1044396162033081, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 1937949817799875.0, + "step": 66, + "teaching_rms": 131884038246111.05, + "traffic_rms": 1941750730643242.8 + }, + { + "batch_loss": 6.258915899903426e+34, + "innovation_rms": 226466291266054.62, + "instruction_rms": 48493876383630.95, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.9231185224616184e+27, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": false, + "max_abs_over_finite_tensors": 0.0, + "nonfinite_indices": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1327433600.0, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 4.243508433833165e+16, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 4.959200137403433e+17, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 9568472064.0, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 4.6989450479140864e+17, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.94350912276267e+18, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 2.2024320851954893e+17, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 9.117614961181327e+17, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.4580151249534976e+17, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 4.243508433833165e+16, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 4.959200137403433e+17, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.4159937018003456e+16, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 76020409434112.0, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.1044396162033081, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": 3251712622963390.5, + "step": 67, + "teaching_rms": 226466284523035.6, + "traffic_rms": 3258621291669245.5 + }, + { + "batch_loss": 1.913553639506425e+35, + "innovation_rms": 384802294754270.94, + "instruction_rms": 87140158410673.97, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 3.314895552884581e+27, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": false, + "max_abs_over_finite_tensors": 0.0, + "nonfinite_indices": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1193733376.0, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 7.327312991223808e+16, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 7.517415974685901e+17, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 9687845888.0, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 8.020228826820772e+17, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 3.3212837789066854e+18, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 3.761082588861563e+17, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.558650641105027e+18, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.8435391203835904e+17, + "nonfinite_indices": [] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 7.327312991223808e+16, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 7.517415974685901e+17, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 2.728193076441907e+16, + "nonfinite_indices": [] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 76020409434112.0, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.1044396162033081, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": Infinity, + "step": 68, + "teaching_rms": 384802294620885.9, + "traffic_rms": Infinity + }, + { + "batch_loss": 6.060198595670836e+35, + "innovation_rms": 656291305267410.9, + "instruction_rms": 157200467366231.16, + "parameter_state": { + "bn_running_mean": { + "all_finite": true, + "max_abs_over_finite_tensors": 5.685550385744514e+27, + "nonfinite_indices": [] + }, + "bn_running_var": { + "all_finite": false, + "max_abs_over_finite_tensors": 0.0, + "nonfinite_indices": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ] + }, + "feedback_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1073391296.0, + "nonfinite_indices": [] + }, + "feedback_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 9.864699783308902e+16, + "nonfinite_indices": [] + }, + "feedback_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.3529375131062764e+18, + "nonfinite_indices": [] + }, + "feedback_weight": { + "all_finite": true, + "max_abs_over_finite_tensors": 9795184640.0, + "nonfinite_indices": [] + }, + "forward_bn_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.3712299506186322e+18, + "nonfinite_indices": [] + }, + "forward_bn_bias_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 5.692070541926597e+18, + "nonfinite_indices": [] + }, + "forward_bn_scale": { + "all_finite": true, + "max_abs_over_finite_tensors": 6.430752827475231e+17, + "nonfinite_indices": [] + }, + "forward_bn_scale_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 2.6696703073331446e+18, + "nonfinite_indices": [] + }, + "forward_momentum": { + "all_finite": false, + "max_abs_over_finite_tensors": 1073391296.0, + "nonfinite_indices": [ + 0 + ] + }, + "forward_readout": { + "all_finite": true, + "max_abs_over_finite_tensors": 9.864699783308902e+16, + "nonfinite_indices": [] + }, + "forward_readout_momentum": { + "all_finite": true, + "max_abs_over_finite_tensors": 1.3529375131062764e+18, + "nonfinite_indices": [] + }, + "forward_weight": { + "all_finite": false, + "max_abs_over_finite_tensors": 9795184640.0, + "nonfinite_indices": [ + 0 + ] + }, + "predictor_bias": { + "all_finite": true, + "max_abs_over_finite_tensors": 76020409434112.0, + "nonfinite_indices": [] + }, + "predictor_slope": { + "all_finite": true, + "max_abs_over_finite_tensors": 0.1044396162033081, + "nonfinite_indices": [] + } + }, + "predictor_mse": null, + "predictor_updated": false, + "raw_apical_rms": Infinity, + "step": 69, + "teaching_rms": 656291305228304.9, + "traffic_rms": Infinity + } + ], + "trajectory_metrics_finite": false, + "validation_evaluations": 0 +} |