diff options
| -rw-r--r-- | results/oral_a_short/bp_lr0.1.json | 464 | ||||
| -rw-r--r-- | results/oral_a_short/dfa_lr0.01.json | 464 | ||||
| -rw-r--r-- | results/oral_a_short/dfa_lr0.03.json | 464 | ||||
| -rw-r--r-- | results/oral_a_short/dfa_lr0.1.json | 464 | ||||
| -rw-r--r-- | results/oral_a_short/sdil_channel_gated_lr0.01.json | 639 | ||||
| -rw-r--r-- | results/oral_a_short/sdil_channel_gated_lr0.03.json | 639 | ||||
| -rw-r--r-- | results/oral_a_short/sdil_channel_gated_lr0.1.json | 639 | ||||
| -rw-r--r-- | results/oral_a_short/sdil_spatial_template_lr0.01.json | 639 | ||||
| -rw-r--r-- | results/oral_a_short/sdil_spatial_template_lr0.03.json | 639 | ||||
| -rw-r--r-- | results/oral_a_short/sdil_spatial_template_lr0.1.json | 639 | ||||
| -rw-r--r-- | results/oral_a_short_selection.json | 176 |
11 files changed, 5866 insertions, 0 deletions
diff --git a/results/oral_a_short/bp_lr0.1.json b/results/oral_a_short/bp_lr0.1.json new file mode 100644 index 0000000..a733e18 --- /dev/null +++ b/results/oral_a_short/bp_lr0.1.json @@ -0,0 +1,464 @@ +{ + "architecture": { + "adaptive_apical_parameters": 0, + "base_width": 16, + "blocks_per_stage": 3, + "bn_eps": 1e-05, + "bn_momentum": 0.1, + "depth": 20, + "family": "CIFAR 6n+2 ResNet, option-A shortcuts", + "fixed_traffic_coefficients": 0, + "forward_parameters": 269722, + "hidden_shapes": [ + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 32, + 16, + 16 + ], + [ + 32, + 16, + 16 + ], + [ + 32, + 16, + 16 + ], + [ + 32, + 16, + 16 + ], + [ + 32, + 16, + 16 + ], + [ + 32, + 16, + 16 + ], + [ + 64, + 8, + 8 + ], + [ + 64, + 8, + 8 + ], + [ + 64, + 8, + 8 + ], + [ + 64, + 8, + 8 + ], + [ + 64, + 8, + 8 + ], + [ + 64, + 8, + 8 + ] + ], + "normalization": "batchnorm", + "predictor_parameters": 0, + "residual_scale": 1.0, + "vectorizer_mode": null, + "vectorizer_parameters": 0 + }, + "args": { + "a_scale": 1.0, + "a_warmup_steps": 0, + "alignment_probe": 0, + "apical_seed": null, + "augment_train": 1, + "batch_size": 128, + "bn_eps": 1e-05, + "bn_momentum": 0.1, + "data_dir": "/home/yurenh2/sdrn/data", + "depth": 20, + "device": "cuda:0", + "epochs": 20, + "eta_A": 0.01, + "eta_P": 0.01, + "eval_every": 0, + "eval_split": "validation", + "learn_P": 0, + "loader_seed": 0, + "lr": 0.1, + "lr_gamma": 0.1, + "lr_milestones": "100,150", + "lr_schedule": "cosine", + "max_steps": 0, + "mode": "bp", + "momentum": 0.9, + "normalization": "batchnorm", + "nuisance_scale": 0.0, + "out": "results/oral_a_short/bp_lr0.1.json", + "output_lr": null, + "pert_directions": 1, + "pert_every": 4, + "pert_sigma": 0.01, + "perturb_seed": 1000, + "predictor_warmup_steps": 0, + "residual_scale": null, + "seed": 0, + "split_seed": 2027, + "train_limit": 10000, + "use_residual": 1, + "val_examples": 5000, + "vectorizer_mode": "spatial_template", + "warmup_epochs": 0, + "weight_decay": 0.0001, + "weight_scale": 1.0, + "width": 16 + }, + "counters": { + "apical_warmup_examples": 0, + "calibration_event_examples": 0, + "causal_scalar_observations": 0, + "logical_batch_loss_queries": 0, + "ordinary_examples": 200000, + "per_example_loss_terms": 0, + "perturbation_events": 0, + "perturbation_forward_examples": 0, + "predictor_warmup_examples": 0 + }, + "diagnostics": null, + "epochs": [ + { + "epoch": 1, + "lr": 0.1, + "output_lr": 0.1, + "step": 79, + "train_examples": 10000, + "train_loss": 1.9995735134124757 + }, + { + "epoch": 2, + "lr": 0.0993844170297569, + "output_lr": 0.0993844170297569, + "step": 158, + "train_examples": 10000, + "train_loss": 1.6754533306121826 + }, + { + "epoch": 3, + "lr": 0.09755282581475769, + "output_lr": 0.09755282581475769, + "step": 237, + "train_examples": 10000, + "train_loss": 1.5343981451034545 + }, + { + "epoch": 4, + "lr": 0.09455032620941839, + "output_lr": 0.09455032620941839, + "step": 316, + "train_examples": 10000, + "train_loss": 1.412572357749939 + }, + { + "epoch": 5, + "lr": 0.09045084971874738, + "output_lr": 0.09045084971874738, + "step": 395, + "train_examples": 10000, + "train_loss": 1.3134462215423583 + }, + { + "epoch": 6, + "lr": 0.08535533905932738, + "output_lr": 0.08535533905932738, + "step": 474, + "train_examples": 10000, + "train_loss": 1.1909215053558349 + }, + { + "epoch": 7, + "lr": 0.07938926261462366, + "output_lr": 0.07938926261462366, + "step": 553, + "train_examples": 10000, + "train_loss": 1.122572089958191 + }, + { + "epoch": 8, + "lr": 0.07269952498697733, + "output_lr": 0.07269952498697733, + "step": 632, + "train_examples": 10000, + "train_loss": 1.0368709173202515 + }, + { + "epoch": 9, + "lr": 0.06545084971874737, + "output_lr": 0.06545084971874737, + "step": 711, + "train_examples": 10000, + "train_loss": 0.9793957938194275 + }, + { + "epoch": 10, + "lr": 0.05782172325201155, + "output_lr": 0.05782172325201155, + "step": 790, + "train_examples": 10000, + "train_loss": 0.9197323471069336 + }, + { + "epoch": 11, + "lr": 0.05, + "output_lr": 0.05, + "step": 869, + "train_examples": 10000, + "train_loss": 0.8693982077598572 + }, + { + "epoch": 12, + "lr": 0.04217827674798845, + "output_lr": 0.04217827674798845, + "step": 948, + "train_examples": 10000, + "train_loss": 0.8178238990783692 + }, + { + "epoch": 13, + "lr": 0.03454915028125263, + "output_lr": 0.03454915028125263, + "step": 1027, + "train_examples": 10000, + "train_loss": 0.7762831295013428 + }, + { + "epoch": 14, + "lr": 0.027300475013022664, + "output_lr": 0.027300475013022664, + "step": 1106, + "train_examples": 10000, + "train_loss": 0.7175337448120117 + }, + { + "epoch": 15, + "lr": 0.02061073738537635, + "output_lr": 0.02061073738537635, + "step": 1185, + "train_examples": 10000, + "train_loss": 0.6878689844131469 + }, + { + "epoch": 16, + "lr": 0.014644660940672627, + "output_lr": 0.014644660940672627, + "step": 1264, + "train_examples": 10000, + "train_loss": 0.6573675689697266 + }, + { + "epoch": 17, + "lr": 0.009549150281252633, + "output_lr": 0.009549150281252633, + "step": 1343, + "train_examples": 10000, + "train_loss": 0.6273343200683594 + }, + { + "epoch": 18, + "lr": 0.005449673790581611, + "output_lr": 0.005449673790581611, + "step": 1422, + "train_examples": 10000, + "train_loss": 0.593037739944458 + }, + { + "epoch": 19, + "lr": 0.0024471741852423235, + "output_lr": 0.0024471741852423235, + "step": 1501, + "train_examples": 10000, + "train_loss": 0.5827967615127564 + }, + { + "epoch": 20, + "lr": 0.0006155829702431171, + "output_lr": 0.0006155829702431171, + "step": 1580, + "train_examples": 10000, + "train_loss": 0.5674787203788757 + } + ], + "evaluation_protocol": { + "test_evaluations": 0, + "test_used_for_selection": false, + "validation_evaluations": 1 + }, + "final": { + "accuracy": 0.7494, + "epoch": 20, + "evaluation_split": "validation", + "finite": true, + "loss": 0.7565972534179688, + "step": 1580 + }, + "hardware": { + "cuda_device_name": "NVIDIA GeForce GTX 1080", + "cuda_visible_devices": "5", + "device": "cuda:0", + "device_total_memory_bytes": 8507949056, + "peak_memory_allocated_bytes": 1738277888, + "peak_memory_reserved_bytes": 2371878912, + "torch_version": "2.3.1+cu118" + }, + "protocol_family": "oral_a_cifar_local_resnet_development", + "provenance": { + "git_commit": "a1d060e6d96d4ab973d16cd75149dd61899c7c95", + "git_tracked_dirty": false + }, + "schema_version": 1, + "split": { + "cifar_source_files": [ + { + "bytes": 31035704, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/data_batch_1", + "sha256": "54636561a3ce25bd3e19253c6b0d8538147b0ae398331ac4a2d86c6d987368cd" + }, + { + "bytes": 31035320, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/data_batch_2", + "sha256": "766b2cef9fbc745cf056b3152224f7cf77163b330ea9a15f9392beb8b89bc5a8" + }, + { + "bytes": 31035999, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/data_batch_3", + "sha256": "0f00d98ebfb30b3ec0ad19f9756dc2630b89003e10525f5e148445e82aa6a1f9" + }, + { + "bytes": 31035696, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/data_batch_4", + "sha256": "3f7bb240661948b8f4d53e36ec720d8306f5668bd0071dcb4e6c947f78e9682b" + }, + { + "bytes": 31035623, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/data_batch_5", + "sha256": "d91802434d8376bbaeeadf58a737e3a1b12ac839077e931237e0dcd43adcb154" + }, + { + "bytes": 31035526, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/test_batch", + "sha256": "f53d8d457504f7cff4ea9e021afcf0e0ad8e24a91f3fc42091b8adef61157831" + } + ], + "dataset": "cifar10", + "evaluation_split": "validation", + "input_layout": "NCHW", + "input_shape": [ + 3, + 32, + 32 + ], + "loader_seed": 0, + "normalization_mean": [ + 0.49140000343322754, + 0.4821999967098236, + 0.4465000033378601 + ], + "normalization_std": [ + 0.24699999392032623, + 0.2434999942779541, + 0.26159998774528503 + ], + "split_from_training_only": true, + "split_seed": 2027, + "test_examples": 10000, + "train_examples": 10000, + "training_augmentation": "random_crop_32_padding_4_zero_then_horizontal_flip_p0.5", + "validation_class_counts": { + "0": 500, + "1": 500, + "2": 500, + "3": 500, + "4": 500, + "5": 500, + "6": 500, + "7": 500, + "8": 500, + "9": 500 + }, + "validation_examples": 5000, + "validation_index_sha256": "8328b206a97c420e49e54e3eca4abe3274c4756b084355784ea3fb8059e4515b" + }, + "timing": { + "apical_warmup_wall_s": 0.0, + "evaluation_wall_s": 0.3821086883544922, + "predictor_warmup_wall_s": 0.0, + "timing_excludes_data_loading_hashing_and_model_construction": true, + "total_timed_wall_s": 75.83793830871582, + "train_wall_s": 75.45246243476868 + }, + "work": { + "apical_macs_per_example": 0, + "causal_scalar_observations": 0, + "components": { + "apical_projection_macs": 0, + "apical_regression_macs": 0, + "bp_reverse_macs_estimate": 16220416000000, + "local_weight_correlation_macs": 0, + "ordinary_forward_macs": 8110208000000, + "perturbation_forward_macs": 0, + "warmup_clean_forward_macs": 0 + }, + "definition": "multiply-accumulates in conv/linear maps; one local weight correlation equals one forward-weight MAC count; BP reverse is estimated as one weight-gradient plus one activation-gradient convolution per forward convolution; elementwise nonlinearities and optimizer arithmetic excluded", + "forward_macs_per_example": 40551040, + "logical_batch_loss_queries": 0, + "per_example_cross_entropy_terms": 0, + "total_clean_forward_examples": 200000, + "total_forward_equivalent_examples": 200000, + "total_macs_estimate": 24330624000000 + } +} diff --git a/results/oral_a_short/dfa_lr0.01.json b/results/oral_a_short/dfa_lr0.01.json new file mode 100644 index 0000000..29b210e --- /dev/null +++ b/results/oral_a_short/dfa_lr0.01.json @@ -0,0 +1,464 @@ +{ + "architecture": { + "adaptive_apical_parameters": 2260992, + "base_width": 16, + "blocks_per_stage": 3, + "bn_eps": 1e-05, + "bn_momentum": 0.1, + "depth": 20, + "family": "CIFAR 6n+2 ResNet, option-A shortcuts", + "fixed_traffic_coefficients": 188416, + "forward_parameters": 269722, + "hidden_shapes": [ + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 32, + 16, + 16 + ], + [ + 32, + 16, + 16 + ], + [ + 32, + 16, + 16 + ], + [ + 32, + 16, + 16 + ], + [ + 32, + 16, + 16 + ], + [ + 32, + 16, + 16 + ], + [ + 64, + 8, + 8 + ], + [ + 64, + 8, + 8 + ], + [ + 64, + 8, + 8 + ], + [ + 64, + 8, + 8 + ], + [ + 64, + 8, + 8 + ], + [ + 64, + 8, + 8 + ] + ], + "normalization": "batchnorm", + "predictor_parameters": 376832, + "residual_scale": 1.0, + "vectorizer_mode": "spatial_template", + "vectorizer_parameters": 1884160 + }, + "args": { + "a_scale": 1.0, + "a_warmup_steps": 0, + "alignment_probe": 0, + "apical_seed": null, + "augment_train": 1, + "batch_size": 128, + "bn_eps": 1e-05, + "bn_momentum": 0.1, + "data_dir": "/home/yurenh2/sdrn/data", + "depth": 20, + "device": "cuda:0", + "epochs": 20, + "eta_A": 0.01, + "eta_P": 0.01, + "eval_every": 0, + "eval_split": "validation", + "learn_P": 0, + "loader_seed": 0, + "lr": 0.01, + "lr_gamma": 0.1, + "lr_milestones": "100,150", + "lr_schedule": "cosine", + "max_steps": 0, + "mode": "dfa", + "momentum": 0.9, + "normalization": "batchnorm", + "nuisance_scale": 0.0, + "out": "results/oral_a_short/dfa_lr0.01.json", + "output_lr": 0.1, + "pert_directions": 1, + "pert_every": 4, + "pert_sigma": 0.01, + "perturb_seed": 1000, + "predictor_warmup_steps": 0, + "residual_scale": null, + "seed": 0, + "split_seed": 2027, + "train_limit": 10000, + "use_residual": 1, + "val_examples": 5000, + "vectorizer_mode": "spatial_template", + "warmup_epochs": 0, + "weight_decay": 0.0001, + "weight_scale": 1.0, + "width": 16 + }, + "counters": { + "apical_warmup_examples": 0, + "calibration_event_examples": 0, + "causal_scalar_observations": 0, + "logical_batch_loss_queries": 0, + "ordinary_examples": 200000, + "per_example_loss_terms": 0, + "perturbation_events": 0, + "perturbation_forward_examples": 0, + "predictor_warmup_examples": 0 + }, + "diagnostics": null, + "epochs": [ + { + "epoch": 1, + "lr": 0.01, + "output_lr": 0.1, + "step": 79, + "train_examples": 10000, + "train_loss": 2.658185742378235 + }, + { + "epoch": 2, + "lr": 0.009938441702975689, + "output_lr": 0.0993844170297569, + "step": 158, + "train_examples": 10000, + "train_loss": 2.601708058166504 + }, + { + "epoch": 3, + "lr": 0.009755282581475769, + "output_lr": 0.09755282581475769, + "step": 237, + "train_examples": 10000, + "train_loss": 2.4101997394561767 + }, + { + "epoch": 4, + "lr": 0.00945503262094184, + "output_lr": 0.09455032620941839, + "step": 316, + "train_examples": 10000, + "train_loss": 2.6492575775146485 + }, + { + "epoch": 5, + "lr": 0.009045084971874737, + "output_lr": 0.09045084971874738, + "step": 395, + "train_examples": 10000, + "train_loss": 2.3147915424346923 + }, + { + "epoch": 6, + "lr": 0.008535533905932738, + "output_lr": 0.08535533905932738, + "step": 474, + "train_examples": 10000, + "train_loss": 2.3464014919281007 + }, + { + "epoch": 7, + "lr": 0.007938926261462366, + "output_lr": 0.07938926261462366, + "step": 553, + "train_examples": 10000, + "train_loss": 2.3019121459960936 + }, + { + "epoch": 8, + "lr": 0.007269952498697734, + "output_lr": 0.07269952498697733, + "step": 632, + "train_examples": 10000, + "train_loss": 2.2836993824005125 + }, + { + "epoch": 9, + "lr": 0.006545084971874737, + "output_lr": 0.06545084971874737, + "step": 711, + "train_examples": 10000, + "train_loss": 2.20195352973938 + }, + { + "epoch": 10, + "lr": 0.0057821723252011546, + "output_lr": 0.05782172325201155, + "step": 790, + "train_examples": 10000, + "train_loss": 2.1341996364593507 + }, + { + "epoch": 11, + "lr": 0.005, + "output_lr": 0.05, + "step": 869, + "train_examples": 10000, + "train_loss": 2.092274705505371 + }, + { + "epoch": 12, + "lr": 0.004217827674798845, + "output_lr": 0.04217827674798845, + "step": 948, + "train_examples": 10000, + "train_loss": 2.0705755081176758 + }, + { + "epoch": 13, + "lr": 0.003454915028125263, + "output_lr": 0.03454915028125263, + "step": 1027, + "train_examples": 10000, + "train_loss": 2.0116215423583985 + }, + { + "epoch": 14, + "lr": 0.0027300475013022664, + "output_lr": 0.027300475013022664, + "step": 1106, + "train_examples": 10000, + "train_loss": 1.995820753479004 + }, + { + "epoch": 15, + "lr": 0.0020610737385376348, + "output_lr": 0.02061073738537635, + "step": 1185, + "train_examples": 10000, + "train_loss": 1.9530410385131836 + }, + { + "epoch": 16, + "lr": 0.0014644660940672626, + "output_lr": 0.014644660940672627, + "step": 1264, + "train_examples": 10000, + "train_loss": 1.9572180179595948 + }, + { + "epoch": 17, + "lr": 0.0009549150281252633, + "output_lr": 0.009549150281252633, + "step": 1343, + "train_examples": 10000, + "train_loss": 1.935197975730896 + }, + { + "epoch": 18, + "lr": 0.0005449673790581611, + "output_lr": 0.005449673790581611, + "step": 1422, + "train_examples": 10000, + "train_loss": 1.9201665208816527 + }, + { + "epoch": 19, + "lr": 0.00024471741852423234, + "output_lr": 0.0024471741852423235, + "step": 1501, + "train_examples": 10000, + "train_loss": 1.9088203987121581 + }, + { + "epoch": 20, + "lr": 6.15582970243117e-05, + "output_lr": 0.0006155829702431171, + "step": 1580, + "train_examples": 10000, + "train_loss": 1.9052054294586183 + } + ], + "evaluation_protocol": { + "test_evaluations": 0, + "test_used_for_selection": false, + "validation_evaluations": 1 + }, + "final": { + "accuracy": 0.314, + "epoch": 20, + "evaluation_split": "validation", + "finite": true, + "loss": 1.8915036865234376, + "step": 1580 + }, + "hardware": { + "cuda_device_name": "NVIDIA GeForce GTX 1080", + "cuda_visible_devices": "5", + "device": "cuda:0", + "device_total_memory_bytes": 8507949056, + "peak_memory_allocated_bytes": 1739555840, + "peak_memory_reserved_bytes": 2424307712, + "torch_version": "2.3.1+cu118" + }, + "protocol_family": "oral_a_cifar_local_resnet_development", + "provenance": { + "git_commit": "a1d060e6d96d4ab973d16cd75149dd61899c7c95", + "git_tracked_dirty": false + }, + "schema_version": 1, + "split": { + "cifar_source_files": [ + { + "bytes": 31035704, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/data_batch_1", + "sha256": "54636561a3ce25bd3e19253c6b0d8538147b0ae398331ac4a2d86c6d987368cd" + }, + { + "bytes": 31035320, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/data_batch_2", + "sha256": "766b2cef9fbc745cf056b3152224f7cf77163b330ea9a15f9392beb8b89bc5a8" + }, + { + "bytes": 31035999, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/data_batch_3", + "sha256": "0f00d98ebfb30b3ec0ad19f9756dc2630b89003e10525f5e148445e82aa6a1f9" + }, + { + "bytes": 31035696, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/data_batch_4", + "sha256": "3f7bb240661948b8f4d53e36ec720d8306f5668bd0071dcb4e6c947f78e9682b" + }, + { + "bytes": 31035623, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/data_batch_5", + "sha256": "d91802434d8376bbaeeadf58a737e3a1b12ac839077e931237e0dcd43adcb154" + }, + { + "bytes": 31035526, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/test_batch", + "sha256": "f53d8d457504f7cff4ea9e021afcf0e0ad8e24a91f3fc42091b8adef61157831" + } + ], + "dataset": "cifar10", + "evaluation_split": "validation", + "input_layout": "NCHW", + "input_shape": [ + 3, + 32, + 32 + ], + "loader_seed": 0, + "normalization_mean": [ + 0.49140000343322754, + 0.4821999967098236, + 0.4465000033378601 + ], + "normalization_std": [ + 0.24699999392032623, + 0.2434999942779541, + 0.26159998774528503 + ], + "split_from_training_only": true, + "split_seed": 2027, + "test_examples": 10000, + "train_examples": 10000, + "training_augmentation": "random_crop_32_padding_4_zero_then_horizontal_flip_p0.5", + "validation_class_counts": { + "0": 500, + "1": 500, + "2": 500, + "3": 500, + "4": 500, + "5": 500, + "6": 500, + "7": 500, + "8": 500, + "9": 500 + }, + "validation_examples": 5000, + "validation_index_sha256": "8328b206a97c420e49e54e3eca4abe3274c4756b084355784ea3fb8059e4515b" + }, + "timing": { + "apical_warmup_wall_s": 0.0, + "evaluation_wall_s": 0.3657495975494385, + "predictor_warmup_wall_s": 0.0, + "timing_excludes_data_loading_hashing_and_model_construction": true, + "total_timed_wall_s": 74.26688361167908, + "train_wall_s": 73.89868807792664 + }, + "work": { + "apical_macs_per_example": 1884160, + "causal_scalar_observations": 0, + "components": { + "apical_projection_macs": 376832000000, + "apical_regression_macs": 0, + "bp_reverse_macs_estimate": 0, + "local_weight_correlation_macs": 8110208000000, + "ordinary_forward_macs": 8110208000000, + "perturbation_forward_macs": 0, + "warmup_clean_forward_macs": 0 + }, + "definition": "multiply-accumulates in conv/linear maps; one local weight correlation equals one forward-weight MAC count; BP reverse is estimated as one weight-gradient plus one activation-gradient convolution per forward convolution; elementwise nonlinearities and optimizer arithmetic excluded", + "forward_macs_per_example": 40551040, + "logical_batch_loss_queries": 0, + "per_example_cross_entropy_terms": 0, + "total_clean_forward_examples": 200000, + "total_forward_equivalent_examples": 200000, + "total_macs_estimate": 16597248000000 + } +} diff --git a/results/oral_a_short/dfa_lr0.03.json b/results/oral_a_short/dfa_lr0.03.json new file mode 100644 index 0000000..8356bcc --- /dev/null +++ b/results/oral_a_short/dfa_lr0.03.json @@ -0,0 +1,464 @@ +{ + "architecture": { + "adaptive_apical_parameters": 2260992, + "base_width": 16, + "blocks_per_stage": 3, + "bn_eps": 1e-05, + "bn_momentum": 0.1, + "depth": 20, + "family": "CIFAR 6n+2 ResNet, option-A shortcuts", + "fixed_traffic_coefficients": 188416, + "forward_parameters": 269722, + "hidden_shapes": [ + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 32, + 16, + 16 + ], + [ + 32, + 16, + 16 + ], + [ + 32, + 16, + 16 + ], + [ + 32, + 16, + 16 + ], + [ + 32, + 16, + 16 + ], + [ + 32, + 16, + 16 + ], + [ + 64, + 8, + 8 + ], + [ + 64, + 8, + 8 + ], + [ + 64, + 8, + 8 + ], + [ + 64, + 8, + 8 + ], + [ + 64, + 8, + 8 + ], + [ + 64, + 8, + 8 + ] + ], + "normalization": "batchnorm", + "predictor_parameters": 376832, + "residual_scale": 1.0, + "vectorizer_mode": "spatial_template", + "vectorizer_parameters": 1884160 + }, + "args": { + "a_scale": 1.0, + "a_warmup_steps": 0, + "alignment_probe": 0, + "apical_seed": null, + "augment_train": 1, + "batch_size": 128, + "bn_eps": 1e-05, + "bn_momentum": 0.1, + "data_dir": "/home/yurenh2/sdrn/data", + "depth": 20, + "device": "cuda:0", + "epochs": 20, + "eta_A": 0.01, + "eta_P": 0.01, + "eval_every": 0, + "eval_split": "validation", + "learn_P": 0, + "loader_seed": 0, + "lr": 0.03, + "lr_gamma": 0.1, + "lr_milestones": "100,150", + "lr_schedule": "cosine", + "max_steps": 0, + "mode": "dfa", + "momentum": 0.9, + "normalization": "batchnorm", + "nuisance_scale": 0.0, + "out": "results/oral_a_short/dfa_lr0.03.json", + "output_lr": 0.1, + "pert_directions": 1, + "pert_every": 4, + "pert_sigma": 0.01, + "perturb_seed": 1000, + "predictor_warmup_steps": 0, + "residual_scale": null, + "seed": 0, + "split_seed": 2027, + "train_limit": 10000, + "use_residual": 1, + "val_examples": 5000, + "vectorizer_mode": "spatial_template", + "warmup_epochs": 0, + "weight_decay": 0.0001, + "weight_scale": 1.0, + "width": 16 + }, + "counters": { + "apical_warmup_examples": 0, + "calibration_event_examples": 0, + "causal_scalar_observations": 0, + "logical_batch_loss_queries": 0, + "ordinary_examples": 200000, + "per_example_loss_terms": 0, + "perturbation_events": 0, + "perturbation_forward_examples": 0, + "predictor_warmup_examples": 0 + }, + "diagnostics": null, + "epochs": [ + { + "epoch": 1, + "lr": 0.03, + "output_lr": 0.1, + "step": 79, + "train_examples": 10000, + "train_loss": 2.632198575592041 + }, + { + "epoch": 2, + "lr": 0.029815325108927065, + "output_lr": 0.0993844170297569, + "step": 158, + "train_examples": 10000, + "train_loss": 2.4454536556243895 + }, + { + "epoch": 3, + "lr": 0.029265847744427302, + "output_lr": 0.09755282581475769, + "step": 237, + "train_examples": 10000, + "train_loss": 3.108930400085449 + }, + { + "epoch": 4, + "lr": 0.028365097862825515, + "output_lr": 0.09455032620941839, + "step": 316, + "train_examples": 10000, + "train_loss": 2.558008377456665 + }, + { + "epoch": 5, + "lr": 0.02713525491562421, + "output_lr": 0.09045084971874738, + "step": 395, + "train_examples": 10000, + "train_loss": 2.4858694175720215 + }, + { + "epoch": 6, + "lr": 0.02560660171779821, + "output_lr": 0.08535533905932738, + "step": 474, + "train_examples": 10000, + "train_loss": 2.5335412963867188 + }, + { + "epoch": 7, + "lr": 0.023816778784387097, + "output_lr": 0.07938926261462366, + "step": 553, + "train_examples": 10000, + "train_loss": 2.650238174057007 + }, + { + "epoch": 8, + "lr": 0.0218098574960932, + "output_lr": 0.07269952498697733, + "step": 632, + "train_examples": 10000, + "train_loss": 2.3551125690460206 + }, + { + "epoch": 9, + "lr": 0.01963525491562421, + "output_lr": 0.06545084971874737, + "step": 711, + "train_examples": 10000, + "train_loss": 2.258931183242798 + }, + { + "epoch": 10, + "lr": 0.017346516975603462, + "output_lr": 0.05782172325201155, + "step": 790, + "train_examples": 10000, + "train_loss": 2.1323540115356447 + }, + { + "epoch": 11, + "lr": 0.015, + "output_lr": 0.05, + "step": 869, + "train_examples": 10000, + "train_loss": 2.103953749084473 + }, + { + "epoch": 12, + "lr": 0.012653483024396533, + "output_lr": 0.04217827674798845, + "step": 948, + "train_examples": 10000, + "train_loss": 1.9935882570266723 + }, + { + "epoch": 13, + "lr": 0.01036474508437579, + "output_lr": 0.03454915028125263, + "step": 1027, + "train_examples": 10000, + "train_loss": 1.9480989864349365 + }, + { + "epoch": 14, + "lr": 0.0081901425039068, + "output_lr": 0.027300475013022664, + "step": 1106, + "train_examples": 10000, + "train_loss": 1.9207990268707276 + }, + { + "epoch": 15, + "lr": 0.006183221215612904, + "output_lr": 0.02061073738537635, + "step": 1185, + "train_examples": 10000, + "train_loss": 1.8341702373504638 + }, + { + "epoch": 16, + "lr": 0.004393398282201788, + "output_lr": 0.014644660940672627, + "step": 1264, + "train_examples": 10000, + "train_loss": 1.8207256023406981 + }, + { + "epoch": 17, + "lr": 0.00286474508437579, + "output_lr": 0.009549150281252633, + "step": 1343, + "train_examples": 10000, + "train_loss": 1.8012396924972534 + }, + { + "epoch": 18, + "lr": 0.001634902137174483, + "output_lr": 0.005449673790581611, + "step": 1422, + "train_examples": 10000, + "train_loss": 1.7742164363861084 + }, + { + "epoch": 19, + "lr": 0.000734152255572697, + "output_lr": 0.0024471741852423235, + "step": 1501, + "train_examples": 10000, + "train_loss": 1.7600377252578736 + }, + { + "epoch": 20, + "lr": 0.0001846748910729351, + "output_lr": 0.0006155829702431171, + "step": 1580, + "train_examples": 10000, + "train_loss": 1.7556010025024413 + } + ], + "evaluation_protocol": { + "test_evaluations": 0, + "test_used_for_selection": false, + "validation_evaluations": 1 + }, + "final": { + "accuracy": 0.3716, + "epoch": 20, + "evaluation_split": "validation", + "finite": true, + "loss": 1.728732666015625, + "step": 1580 + }, + "hardware": { + "cuda_device_name": "NVIDIA GeForce GTX 1080", + "cuda_visible_devices": "5", + "device": "cuda:0", + "device_total_memory_bytes": 8507949056, + "peak_memory_allocated_bytes": 1739555840, + "peak_memory_reserved_bytes": 2424307712, + "torch_version": "2.3.1+cu118" + }, + "protocol_family": "oral_a_cifar_local_resnet_development", + "provenance": { + "git_commit": "a1d060e6d96d4ab973d16cd75149dd61899c7c95", + "git_tracked_dirty": false + }, + "schema_version": 1, + "split": { + "cifar_source_files": [ + { + "bytes": 31035704, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/data_batch_1", + "sha256": "54636561a3ce25bd3e19253c6b0d8538147b0ae398331ac4a2d86c6d987368cd" + }, + { + "bytes": 31035320, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/data_batch_2", + "sha256": "766b2cef9fbc745cf056b3152224f7cf77163b330ea9a15f9392beb8b89bc5a8" + }, + { + "bytes": 31035999, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/data_batch_3", + "sha256": "0f00d98ebfb30b3ec0ad19f9756dc2630b89003e10525f5e148445e82aa6a1f9" + }, + { + "bytes": 31035696, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/data_batch_4", + "sha256": "3f7bb240661948b8f4d53e36ec720d8306f5668bd0071dcb4e6c947f78e9682b" + }, + { + "bytes": 31035623, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/data_batch_5", + "sha256": "d91802434d8376bbaeeadf58a737e3a1b12ac839077e931237e0dcd43adcb154" + }, + { + "bytes": 31035526, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/test_batch", + "sha256": "f53d8d457504f7cff4ea9e021afcf0e0ad8e24a91f3fc42091b8adef61157831" + } + ], + "dataset": "cifar10", + "evaluation_split": "validation", + "input_layout": "NCHW", + "input_shape": [ + 3, + 32, + 32 + ], + "loader_seed": 0, + "normalization_mean": [ + 0.49140000343322754, + 0.4821999967098236, + 0.4465000033378601 + ], + "normalization_std": [ + 0.24699999392032623, + 0.2434999942779541, + 0.26159998774528503 + ], + "split_from_training_only": true, + "split_seed": 2027, + "test_examples": 10000, + "train_examples": 10000, + "training_augmentation": "random_crop_32_padding_4_zero_then_horizontal_flip_p0.5", + "validation_class_counts": { + "0": 500, + "1": 500, + "2": 500, + "3": 500, + "4": 500, + "5": 500, + "6": 500, + "7": 500, + "8": 500, + "9": 500 + }, + "validation_examples": 5000, + "validation_index_sha256": "8328b206a97c420e49e54e3eca4abe3274c4756b084355784ea3fb8059e4515b" + }, + "timing": { + "apical_warmup_wall_s": 0.0, + "evaluation_wall_s": 0.3621516227722168, + "predictor_warmup_wall_s": 0.0, + "timing_excludes_data_loading_hashing_and_model_construction": true, + "total_timed_wall_s": 74.16393518447876, + "train_wall_s": 73.79931426048279 + }, + "work": { + "apical_macs_per_example": 1884160, + "causal_scalar_observations": 0, + "components": { + "apical_projection_macs": 376832000000, + "apical_regression_macs": 0, + "bp_reverse_macs_estimate": 0, + "local_weight_correlation_macs": 8110208000000, + "ordinary_forward_macs": 8110208000000, + "perturbation_forward_macs": 0, + "warmup_clean_forward_macs": 0 + }, + "definition": "multiply-accumulates in conv/linear maps; one local weight correlation equals one forward-weight MAC count; BP reverse is estimated as one weight-gradient plus one activation-gradient convolution per forward convolution; elementwise nonlinearities and optimizer arithmetic excluded", + "forward_macs_per_example": 40551040, + "logical_batch_loss_queries": 0, + "per_example_cross_entropy_terms": 0, + "total_clean_forward_examples": 200000, + "total_forward_equivalent_examples": 200000, + "total_macs_estimate": 16597248000000 + } +} diff --git a/results/oral_a_short/dfa_lr0.1.json b/results/oral_a_short/dfa_lr0.1.json new file mode 100644 index 0000000..33aad3d --- /dev/null +++ b/results/oral_a_short/dfa_lr0.1.json @@ -0,0 +1,464 @@ +{ + "architecture": { + "adaptive_apical_parameters": 2260992, + "base_width": 16, + "blocks_per_stage": 3, + "bn_eps": 1e-05, + "bn_momentum": 0.1, + "depth": 20, + "family": "CIFAR 6n+2 ResNet, option-A shortcuts", + "fixed_traffic_coefficients": 188416, + "forward_parameters": 269722, + "hidden_shapes": [ + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 32, + 16, + 16 + ], + [ + 32, + 16, + 16 + ], + [ + 32, + 16, + 16 + ], + [ + 32, + 16, + 16 + ], + [ + 32, + 16, + 16 + ], + [ + 32, + 16, + 16 + ], + [ + 64, + 8, + 8 + ], + [ + 64, + 8, + 8 + ], + [ + 64, + 8, + 8 + ], + [ + 64, + 8, + 8 + ], + [ + 64, + 8, + 8 + ], + [ + 64, + 8, + 8 + ] + ], + "normalization": "batchnorm", + "predictor_parameters": 376832, + "residual_scale": 1.0, + "vectorizer_mode": "spatial_template", + "vectorizer_parameters": 1884160 + }, + "args": { + "a_scale": 1.0, + "a_warmup_steps": 0, + "alignment_probe": 0, + "apical_seed": null, + "augment_train": 1, + "batch_size": 128, + "bn_eps": 1e-05, + "bn_momentum": 0.1, + "data_dir": "/home/yurenh2/sdrn/data", + "depth": 20, + "device": "cuda:0", + "epochs": 20, + "eta_A": 0.01, + "eta_P": 0.01, + "eval_every": 0, + "eval_split": "validation", + "learn_P": 0, + "loader_seed": 0, + "lr": 0.1, + "lr_gamma": 0.1, + "lr_milestones": "100,150", + "lr_schedule": "cosine", + "max_steps": 0, + "mode": "dfa", + "momentum": 0.9, + "normalization": "batchnorm", + "nuisance_scale": 0.0, + "out": "results/oral_a_short/dfa_lr0.1.json", + "output_lr": 0.1, + "pert_directions": 1, + "pert_every": 4, + "pert_sigma": 0.01, + "perturb_seed": 1000, + "predictor_warmup_steps": 0, + "residual_scale": null, + "seed": 0, + "split_seed": 2027, + "train_limit": 10000, + "use_residual": 1, + "val_examples": 5000, + "vectorizer_mode": "spatial_template", + "warmup_epochs": 0, + "weight_decay": 0.0001, + "weight_scale": 1.0, + "width": 16 + }, + "counters": { + "apical_warmup_examples": 0, + "calibration_event_examples": 0, + "causal_scalar_observations": 0, + "logical_batch_loss_queries": 0, + "ordinary_examples": 200000, + "per_example_loss_terms": 0, + "perturbation_events": 0, + "perturbation_forward_examples": 0, + "predictor_warmup_examples": 0 + }, + "diagnostics": null, + "epochs": [ + { + "epoch": 1, + "lr": 0.1, + "output_lr": 0.1, + "step": 79, + "train_examples": 10000, + "train_loss": 2.835949954223633 + }, + { + "epoch": 2, + "lr": 0.0993844170297569, + "output_lr": 0.0993844170297569, + "step": 158, + "train_examples": 10000, + "train_loss": 2.8182399749755858 + }, + { + "epoch": 3, + "lr": 0.09755282581475769, + "output_lr": 0.09755282581475769, + "step": 237, + "train_examples": 10000, + "train_loss": 4.338047203826904 + }, + { + "epoch": 4, + "lr": 0.09455032620941839, + "output_lr": 0.09455032620941839, + "step": 316, + "train_examples": 10000, + "train_loss": 4.830230680847168 + }, + { + "epoch": 5, + "lr": 0.09045084971874738, + "output_lr": 0.09045084971874738, + "step": 395, + "train_examples": 10000, + "train_loss": 4.160814493560791 + }, + { + "epoch": 6, + "lr": 0.08535533905932738, + "output_lr": 0.08535533905932738, + "step": 474, + "train_examples": 10000, + "train_loss": 7.872198279571533 + }, + { + "epoch": 7, + "lr": 0.07938926261462366, + "output_lr": 0.07938926261462366, + "step": 553, + "train_examples": 10000, + "train_loss": 5.239408417892456 + }, + { + "epoch": 8, + "lr": 0.07269952498697733, + "output_lr": 0.07269952498697733, + "step": 632, + "train_examples": 10000, + "train_loss": 7.5435514030456545 + }, + { + "epoch": 9, + "lr": 0.06545084971874737, + "output_lr": 0.06545084971874737, + "step": 711, + "train_examples": 10000, + "train_loss": 6.3907674743652345 + }, + { + "epoch": 10, + "lr": 0.05782172325201155, + "output_lr": 0.05782172325201155, + "step": 790, + "train_examples": 10000, + "train_loss": 8.492947927856445 + }, + { + "epoch": 11, + "lr": 0.05, + "output_lr": 0.05, + "step": 869, + "train_examples": 10000, + "train_loss": 5.571098100280762 + }, + { + "epoch": 12, + "lr": 0.04217827674798845, + "output_lr": 0.04217827674798845, + "step": 948, + "train_examples": 10000, + "train_loss": 5.536634136199951 + }, + { + "epoch": 13, + "lr": 0.03454915028125263, + "output_lr": 0.03454915028125263, + "step": 1027, + "train_examples": 10000, + "train_loss": 4.147188665771484 + }, + { + "epoch": 14, + "lr": 0.027300475013022664, + "output_lr": 0.027300475013022664, + "step": 1106, + "train_examples": 10000, + "train_loss": 2.4310934539794924 + }, + { + "epoch": 15, + "lr": 0.02061073738537635, + "output_lr": 0.02061073738537635, + "step": 1185, + "train_examples": 10000, + "train_loss": 2.091388729095459 + }, + { + "epoch": 16, + "lr": 0.014644660940672627, + "output_lr": 0.014644660940672627, + "step": 1264, + "train_examples": 10000, + "train_loss": 2.1787188812255858 + }, + { + "epoch": 17, + "lr": 0.009549150281252633, + "output_lr": 0.009549150281252633, + "step": 1343, + "train_examples": 10000, + "train_loss": 1.9502821924209595 + }, + { + "epoch": 18, + "lr": 0.005449673790581611, + "output_lr": 0.005449673790581611, + "step": 1422, + "train_examples": 10000, + "train_loss": 1.8884031147003173 + }, + { + "epoch": 19, + "lr": 0.0024471741852423235, + "output_lr": 0.0024471741852423235, + "step": 1501, + "train_examples": 10000, + "train_loss": 1.8296160148620606 + }, + { + "epoch": 20, + "lr": 0.0006155829702431171, + "output_lr": 0.0006155829702431171, + "step": 1580, + "train_examples": 10000, + "train_loss": 1.809558029937744 + } + ], + "evaluation_protocol": { + "test_evaluations": 0, + "test_used_for_selection": false, + "validation_evaluations": 1 + }, + "final": { + "accuracy": 0.3642, + "epoch": 20, + "evaluation_split": "validation", + "finite": true, + "loss": 1.7964451171875, + "step": 1580 + }, + "hardware": { + "cuda_device_name": "NVIDIA GeForce GTX 1080", + "cuda_visible_devices": "5", + "device": "cuda:0", + "device_total_memory_bytes": 8507949056, + "peak_memory_allocated_bytes": 1739555840, + "peak_memory_reserved_bytes": 2424307712, + "torch_version": "2.3.1+cu118" + }, + "protocol_family": "oral_a_cifar_local_resnet_development", + "provenance": { + "git_commit": "a1d060e6d96d4ab973d16cd75149dd61899c7c95", + "git_tracked_dirty": false + }, + "schema_version": 1, + "split": { + "cifar_source_files": [ + { + "bytes": 31035704, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/data_batch_1", + "sha256": "54636561a3ce25bd3e19253c6b0d8538147b0ae398331ac4a2d86c6d987368cd" + }, + { + "bytes": 31035320, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/data_batch_2", + "sha256": "766b2cef9fbc745cf056b3152224f7cf77163b330ea9a15f9392beb8b89bc5a8" + }, + { + "bytes": 31035999, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/data_batch_3", + "sha256": "0f00d98ebfb30b3ec0ad19f9756dc2630b89003e10525f5e148445e82aa6a1f9" + }, + { + "bytes": 31035696, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/data_batch_4", + "sha256": "3f7bb240661948b8f4d53e36ec720d8306f5668bd0071dcb4e6c947f78e9682b" + }, + { + "bytes": 31035623, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/data_batch_5", + "sha256": "d91802434d8376bbaeeadf58a737e3a1b12ac839077e931237e0dcd43adcb154" + }, + { + "bytes": 31035526, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/test_batch", + "sha256": "f53d8d457504f7cff4ea9e021afcf0e0ad8e24a91f3fc42091b8adef61157831" + } + ], + "dataset": "cifar10", + "evaluation_split": "validation", + "input_layout": "NCHW", + "input_shape": [ + 3, + 32, + 32 + ], + "loader_seed": 0, + "normalization_mean": [ + 0.49140000343322754, + 0.4821999967098236, + 0.4465000033378601 + ], + "normalization_std": [ + 0.24699999392032623, + 0.2434999942779541, + 0.26159998774528503 + ], + "split_from_training_only": true, + "split_seed": 2027, + "test_examples": 10000, + "train_examples": 10000, + "training_augmentation": "random_crop_32_padding_4_zero_then_horizontal_flip_p0.5", + "validation_class_counts": { + "0": 500, + "1": 500, + "2": 500, + "3": 500, + "4": 500, + "5": 500, + "6": 500, + "7": 500, + "8": 500, + "9": 500 + }, + "validation_examples": 5000, + "validation_index_sha256": "8328b206a97c420e49e54e3eca4abe3274c4756b084355784ea3fb8059e4515b" + }, + "timing": { + "apical_warmup_wall_s": 0.0, + "evaluation_wall_s": 0.36763930320739746, + "predictor_warmup_wall_s": 0.0, + "timing_excludes_data_loading_hashing_and_model_construction": true, + "total_timed_wall_s": 74.36485743522644, + "train_wall_s": 73.99453210830688 + }, + "work": { + "apical_macs_per_example": 1884160, + "causal_scalar_observations": 0, + "components": { + "apical_projection_macs": 376832000000, + "apical_regression_macs": 0, + "bp_reverse_macs_estimate": 0, + "local_weight_correlation_macs": 8110208000000, + "ordinary_forward_macs": 8110208000000, + "perturbation_forward_macs": 0, + "warmup_clean_forward_macs": 0 + }, + "definition": "multiply-accumulates in conv/linear maps; one local weight correlation equals one forward-weight MAC count; BP reverse is estimated as one weight-gradient plus one activation-gradient convolution per forward convolution; elementwise nonlinearities and optimizer arithmetic excluded", + "forward_macs_per_example": 40551040, + "logical_batch_loss_queries": 0, + "per_example_cross_entropy_terms": 0, + "total_clean_forward_examples": 200000, + "total_forward_equivalent_examples": 200000, + "total_macs_estimate": 16597248000000 + } +} diff --git a/results/oral_a_short/sdil_channel_gated_lr0.01.json b/results/oral_a_short/sdil_channel_gated_lr0.01.json new file mode 100644 index 0000000..d630417 --- /dev/null +++ b/results/oral_a_short/sdil_channel_gated_lr0.01.json @@ -0,0 +1,639 @@ +{ + "apical_warmup": { + "last": { + "calibration_mse": 518.2959050717561, + "prediction_target_cosine": 0.00016553175127441687, + "target_power": 518.2958984375 + }, + "steps": 100 + }, + "architecture": { + "adaptive_apical_parameters": 390592, + "base_width": 16, + "blocks_per_stage": 3, + "bn_eps": 1e-05, + "bn_momentum": 0.1, + "depth": 20, + "family": "CIFAR 6n+2 ResNet, option-A shortcuts", + "fixed_traffic_coefficients": 188416, + "forward_parameters": 269722, + "hidden_shapes": [ + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 32, + 16, + 16 + ], + [ + 32, + 16, + 16 + ], + [ + 32, + 16, + 16 + ], + [ + 32, + 16, + 16 + ], + [ + 32, + 16, + 16 + ], + [ + 32, + 16, + 16 + ], + [ + 64, + 8, + 8 + ], + [ + 64, + 8, + 8 + ], + [ + 64, + 8, + 8 + ], + [ + 64, + 8, + 8 + ], + [ + 64, + 8, + 8 + ], + [ + 64, + 8, + 8 + ] + ], + "normalization": "batchnorm", + "predictor_parameters": 376832, + "residual_scale": 1.0, + "vectorizer_mode": "channel_gated", + "vectorizer_parameters": 13760 + }, + "args": { + "a_scale": 1.0, + "a_warmup_steps": 100, + "alignment_probe": 32, + "apical_seed": null, + "augment_train": 1, + "batch_size": 128, + "bn_eps": 1e-05, + "bn_momentum": 0.1, + "data_dir": "/home/yurenh2/sdrn/data", + "depth": 20, + "device": "cuda:0", + "epochs": 20, + "eta_A": 0.001, + "eta_P": 0.01, + "eval_every": 0, + "eval_split": "validation", + "learn_P": 0, + "loader_seed": 0, + "lr": 0.01, + "lr_gamma": 0.1, + "lr_milestones": "100,150", + "lr_schedule": "cosine", + "max_steps": 0, + "mode": "sdil", + "momentum": 0.9, + "normalization": "batchnorm", + "nuisance_scale": 0.0, + "out": "results/oral_a_short/sdil_channel_gated_lr0.01.json", + "output_lr": 0.1, + "pert_directions": 1, + "pert_every": 4, + "pert_sigma": 0.01, + "perturb_seed": 1000, + "predictor_warmup_steps": 0, + "residual_scale": null, + "seed": 0, + "split_seed": 2027, + "train_limit": 10000, + "use_residual": 1, + "val_examples": 5000, + "vectorizer_mode": "channel_gated", + "warmup_epochs": 0, + "weight_decay": 0.0001, + "weight_scale": 1.0, + "width": 16 + }, + "counters": { + "apical_warmup_examples": 12688, + "calibration_event_examples": 62688, + "causal_scalar_observations": 990, + "logical_batch_loss_queries": 990, + "ordinary_examples": 200000, + "per_example_loss_terms": 125376, + "perturbation_events": 495, + "perturbation_forward_examples": 125376, + "predictor_warmup_examples": 0 + }, + "diagnostics": { + "early_third_mean": 0.018784026537711423, + "innovation_negative_gradient_cosine": [ + 0.0064247846603393555, + 0.04731021076440811, + 0.016317252069711685, + 0.0033069862984120846, + 0.021622221916913986, + 0.017722703516483307, + 0.027213476598262787, + 0.013780711218714714, + 0.07152771204710007, + 0.06467936187982559, + 0.05392073467373848, + 0.04071490839123726, + 0.02924099937081337, + 0.08977454900741577, + 0.16485495865345, + 0.19109880924224854, + 0.19917020201683044, + 0.2036983072757721, + 0.12616632878780365 + ], + "normalization_state": "training_batch_stats_without_running_update", + "raw_negative_gradient_cosine": [ + 0.0064247846603393555, + 0.04731021076440811, + 0.016317252069711685, + 0.0033069862984120846, + 0.021622221916913986, + 0.017722703516483307, + 0.027213476598262787, + 0.013780711218714714, + 0.07152771204710007, + 0.06467936187982559, + 0.05392073467373848, + 0.04071490839123726, + 0.02924099937081337, + 0.08977454900741577, + 0.16485495865345, + 0.19109880924224854, + 0.19917020201683044, + 0.2036983072757721, + 0.12616632878780365 + ], + "teaching_negative_gradient_cosine": [ + 0.0064247846603393555, + 0.04731021076440811, + 0.016317252069711685, + 0.0033069862984120846, + 0.021622221916913986, + 0.017722703516483307, + 0.027213476598262787, + 0.013780711218714714, + 0.07152771204710007, + 0.06467936187982559, + 0.05392073467373848, + 0.04071490839123726, + 0.02924099937081337, + 0.08977454900741577, + 0.16485495865345, + 0.19109880924224854, + 0.19917020201683044, + 0.2036983072757721, + 0.12616632878780365 + ], + "wall_s": 0.1185142993927002 + }, + "epochs": [ + { + "calibration": { + "calibration_mse": 303.60023584691606, + "prediction_target_cosine": -7.053524919744465e-05, + "target_power": 303.60024019551014 + }, + "epoch": 1, + "lr": 0.01, + "output_lr": 0.1, + "step": 79, + "train_examples": 10000, + "train_loss": 3.0403768768310546 + }, + { + "calibration": { + "calibration_mse": 785.7472175463388, + "prediction_target_cosine": 2.4688466785111376e-05, + "target_power": 785.7471995187842 + }, + "epoch": 2, + "lr": 0.009938441702975689, + "output_lr": 0.0993844170297569, + "step": 158, + "train_examples": 10000, + "train_loss": 2.47521697807312 + }, + { + "calibration": { + "calibration_mse": 926.6549590877864, + "prediction_target_cosine": -8.481026661355656e-05, + "target_power": 926.6549154965772 + }, + "epoch": 3, + "lr": 0.009755282581475769, + "output_lr": 0.09755282581475769, + "step": 237, + "train_examples": 10000, + "train_loss": 2.327832473754883 + }, + { + "calibration": { + "calibration_mse": 524.3671950681531, + "prediction_target_cosine": -9.424448535305888e-06, + "target_power": 524.3671919719851 + }, + "epoch": 4, + "lr": 0.00945503262094184, + "output_lr": 0.09455032620941839, + "step": 316, + "train_examples": 10000, + "train_loss": 2.2477327514648437 + }, + { + "calibration": { + "calibration_mse": 906.0465460537567, + "prediction_target_cosine": 5.689712272068958e-05, + "target_power": 906.0465624182121 + }, + "epoch": 5, + "lr": 0.009045084971874737, + "output_lr": 0.09045084971874738, + "step": 395, + "train_examples": 10000, + "train_loss": 2.186990054321289 + }, + { + "calibration": { + "calibration_mse": 1382.679324004961, + "prediction_target_cosine": -1.069098398475288e-05, + "target_power": 1382.6792525332908 + }, + "epoch": 6, + "lr": 0.008535533905932738, + "output_lr": 0.08535533905932738, + "step": 474, + "train_examples": 10000, + "train_loss": 2.075714948654175 + }, + { + "calibration": { + "calibration_mse": 462.8764529534434, + "prediction_target_cosine": -1.6088110219793693e-05, + "target_power": 462.876455851375 + }, + "epoch": 7, + "lr": 0.007938926261462366, + "output_lr": 0.07938926261462366, + "step": 553, + "train_examples": 10000, + "train_loss": 2.0236370288848877 + }, + { + "calibration": { + "calibration_mse": 1054.5301591010266, + "prediction_target_cosine": 4.050277694471746e-05, + "target_power": 1054.5301153976257 + }, + "epoch": 8, + "lr": 0.007269952498697734, + "output_lr": 0.07269952498697733, + "step": 632, + "train_examples": 10000, + "train_loss": 2.0253488752365114 + }, + { + "calibration": { + "calibration_mse": 1057.2615307163528, + "prediction_target_cosine": 5.20554017652448e-05, + "target_power": 1057.2614973764055 + }, + "epoch": 9, + "lr": 0.006545084971874737, + "output_lr": 0.06545084971874737, + "step": 711, + "train_examples": 10000, + "train_loss": 1.9688177600860595 + }, + { + "calibration": { + "calibration_mse": 1072.6804250551306, + "prediction_target_cosine": -6.634325282023175e-05, + "target_power": 1072.6804443732553 + }, + "epoch": 10, + "lr": 0.0057821723252011546, + "output_lr": 0.05782172325201155, + "step": 790, + "train_examples": 10000, + "train_loss": 1.8554277042388916 + }, + { + "calibration": { + "calibration_mse": 978.8598406502088, + "prediction_target_cosine": 3.40096549432178e-05, + "target_power": 978.8598520575655 + }, + "epoch": 11, + "lr": 0.005, + "output_lr": 0.05, + "step": 869, + "train_examples": 10000, + "train_loss": 1.8643826206207275 + }, + { + "calibration": { + "calibration_mse": 600.056727797202, + "prediction_target_cosine": -1.9106017742333596e-05, + "target_power": 600.0567101223791 + }, + "epoch": 12, + "lr": 0.004217827674798845, + "output_lr": 0.04217827674798845, + "step": 948, + "train_examples": 10000, + "train_loss": 1.7651144916534425 + }, + { + "calibration": { + "calibration_mse": 732.2157620286812, + "prediction_target_cosine": 4.7248998125412554e-05, + "target_power": 732.2157623713431 + }, + "epoch": 13, + "lr": 0.003454915028125263, + "output_lr": 0.03454915028125263, + "step": 1027, + "train_examples": 10000, + "train_loss": 1.7445065725326538 + }, + { + "calibration": { + "calibration_mse": 976.8885286455568, + "prediction_target_cosine": 6.757763090613125e-05, + "target_power": 976.8885265516197 + }, + "epoch": 14, + "lr": 0.0027300475013022664, + "output_lr": 0.027300475013022664, + "step": 1106, + "train_examples": 10000, + "train_loss": 1.7122911861419678 + }, + { + "calibration": { + "calibration_mse": 668.0147250166406, + "prediction_target_cosine": 4.402830101716635e-05, + "target_power": 668.0147206327189 + }, + "epoch": 15, + "lr": 0.0020610737385376348, + "output_lr": 0.02061073738537635, + "step": 1185, + "train_examples": 10000, + "train_loss": 1.6635886901855468 + }, + { + "calibration": { + "calibration_mse": 980.3828986875064, + "prediction_target_cosine": 4.302361982883913e-05, + "target_power": 980.382911251547 + }, + "epoch": 16, + "lr": 0.0014644660940672626, + "output_lr": 0.014644660940672627, + "step": 1264, + "train_examples": 10000, + "train_loss": 1.6665239530563354 + }, + { + "calibration": { + "calibration_mse": 675.8823695696565, + "prediction_target_cosine": -6.293169594075907e-05, + "target_power": 675.882374613732 + }, + "epoch": 17, + "lr": 0.0009549150281252633, + "output_lr": 0.009549150281252633, + "step": 1343, + "train_examples": 10000, + "train_loss": 1.6579075477600098 + }, + { + "calibration": { + "calibration_mse": 1099.5418591521516, + "prediction_target_cosine": 3.818202317773011e-05, + "target_power": 1099.5419457962148 + }, + "epoch": 18, + "lr": 0.0005449673790581611, + "output_lr": 0.005449673790581611, + "step": 1422, + "train_examples": 10000, + "train_loss": 1.6407965618133544 + }, + { + "calibration": { + "calibration_mse": 371.8435573219928, + "prediction_target_cosine": -4.679937512933992e-05, + "target_power": 371.843570496725 + }, + "epoch": 19, + "lr": 0.00024471741852423234, + "output_lr": 0.0024471741852423235, + "step": 1501, + "train_examples": 10000, + "train_loss": 1.6256537380218505 + }, + { + "calibration": { + "calibration_mse": 1096.4412992398852, + "prediction_target_cosine": 2.999884061467783e-05, + "target_power": 1096.44131298087 + }, + "epoch": 20, + "lr": 6.15582970243117e-05, + "output_lr": 0.0006155829702431171, + "step": 1580, + "train_examples": 10000, + "train_loss": 1.6222436019897462 + } + ], + "evaluation_protocol": { + "test_evaluations": 0, + "test_used_for_selection": false, + "validation_evaluations": 1 + }, + "final": { + "accuracy": 0.4054, + "epoch": 20, + "evaluation_split": "validation", + "finite": true, + "loss": 1.6269997802734375, + "step": 1580 + }, + "hardware": { + "cuda_device_name": "NVIDIA GeForce GTX 1080", + "cuda_visible_devices": "5", + "device": "cuda:0", + "device_total_memory_bytes": 8507949056, + "peak_memory_allocated_bytes": 1733492736, + "peak_memory_reserved_bytes": 2858418176, + "torch_version": "2.3.1+cu118" + }, + "protocol_family": "oral_a_cifar_local_resnet_development", + "provenance": { + "git_commit": "a1d060e6d96d4ab973d16cd75149dd61899c7c95", + "git_tracked_dirty": false + }, + "schema_version": 1, + "split": { + "cifar_source_files": [ + { + "bytes": 31035704, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/data_batch_1", + "sha256": "54636561a3ce25bd3e19253c6b0d8538147b0ae398331ac4a2d86c6d987368cd" + }, + { + "bytes": 31035320, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/data_batch_2", + "sha256": "766b2cef9fbc745cf056b3152224f7cf77163b330ea9a15f9392beb8b89bc5a8" + }, + { + "bytes": 31035999, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/data_batch_3", + "sha256": "0f00d98ebfb30b3ec0ad19f9756dc2630b89003e10525f5e148445e82aa6a1f9" + }, + { + "bytes": 31035696, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/data_batch_4", + "sha256": "3f7bb240661948b8f4d53e36ec720d8306f5668bd0071dcb4e6c947f78e9682b" + }, + { + "bytes": 31035623, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/data_batch_5", + "sha256": "d91802434d8376bbaeeadf58a737e3a1b12ac839077e931237e0dcd43adcb154" + }, + { + "bytes": 31035526, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/test_batch", + "sha256": "f53d8d457504f7cff4ea9e021afcf0e0ad8e24a91f3fc42091b8adef61157831" + } + ], + "dataset": "cifar10", + "evaluation_split": "validation", + "input_layout": "NCHW", + "input_shape": [ + 3, + 32, + 32 + ], + "loader_seed": 0, + "normalization_mean": [ + 0.49140000343322754, + 0.4821999967098236, + 0.4465000033378601 + ], + "normalization_std": [ + 0.24699999392032623, + 0.2434999942779541, + 0.26159998774528503 + ], + "split_from_training_only": true, + "split_seed": 2027, + "test_examples": 10000, + "train_examples": 10000, + "training_augmentation": "random_crop_32_padding_4_zero_then_horizontal_flip_p0.5", + "validation_class_counts": { + "0": 500, + "1": 500, + "2": 500, + "3": 500, + "4": 500, + "5": 500, + "6": 500, + "7": 500, + "8": 500, + "9": 500 + }, + "validation_examples": 5000, + "validation_index_sha256": "8328b206a97c420e49e54e3eca4abe3274c4756b084355784ea3fb8059e4515b" + }, + "timing": { + "apical_warmup_wall_s": 6.557543039321899, + "evaluation_wall_s": 0.36567163467407227, + "predictor_warmup_wall_s": 0.0, + "timing_excludes_data_loading_hashing_and_model_construction": true, + "total_timed_wall_s": 101.1947500705719, + "train_wall_s": 94.26846575737 + }, + "work": { + "apical_macs_per_example": 202176, + "causal_scalar_observations": 990, + "components": { + "apical_projection_macs": 43000409088, + "apical_regression_macs": 12674009088, + "bp_reverse_macs_estimate": 0, + "local_weight_correlation_macs": 8110208000000, + "ordinary_forward_macs": 8110208000000, + "perturbation_forward_macs": 5084127191040, + "warmup_clean_forward_macs": 514511595520 + }, + "definition": "multiply-accumulates in conv/linear maps; one local weight correlation equals one forward-weight MAC count; BP reverse is estimated as one weight-gradient plus one activation-gradient convolution per forward convolution; elementwise nonlinearities and optimizer arithmetic excluded", + "forward_macs_per_example": 40551040, + "logical_batch_loss_queries": 990, + "per_example_cross_entropy_terms": 125376, + "total_clean_forward_examples": 212688, + "total_forward_equivalent_examples": 338064, + "total_macs_estimate": 21874729204736 + } +} diff --git a/results/oral_a_short/sdil_channel_gated_lr0.03.json b/results/oral_a_short/sdil_channel_gated_lr0.03.json new file mode 100644 index 0000000..3932283 --- /dev/null +++ b/results/oral_a_short/sdil_channel_gated_lr0.03.json @@ -0,0 +1,639 @@ +{ + "apical_warmup": { + "last": { + "calibration_mse": 518.2959050717561, + "prediction_target_cosine": 0.00016553175127441687, + "target_power": 518.2958984375 + }, + "steps": 100 + }, + "architecture": { + "adaptive_apical_parameters": 390592, + "base_width": 16, + "blocks_per_stage": 3, + "bn_eps": 1e-05, + "bn_momentum": 0.1, + "depth": 20, + "family": "CIFAR 6n+2 ResNet, option-A shortcuts", + "fixed_traffic_coefficients": 188416, + "forward_parameters": 269722, + "hidden_shapes": [ + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 32, + 16, + 16 + ], + [ + 32, + 16, + 16 + ], + [ + 32, + 16, + 16 + ], + [ + 32, + 16, + 16 + ], + [ + 32, + 16, + 16 + ], + [ + 32, + 16, + 16 + ], + [ + 64, + 8, + 8 + ], + [ + 64, + 8, + 8 + ], + [ + 64, + 8, + 8 + ], + [ + 64, + 8, + 8 + ], + [ + 64, + 8, + 8 + ], + [ + 64, + 8, + 8 + ] + ], + "normalization": "batchnorm", + "predictor_parameters": 376832, + "residual_scale": 1.0, + "vectorizer_mode": "channel_gated", + "vectorizer_parameters": 13760 + }, + "args": { + "a_scale": 1.0, + "a_warmup_steps": 100, + "alignment_probe": 32, + "apical_seed": null, + "augment_train": 1, + "batch_size": 128, + "bn_eps": 1e-05, + "bn_momentum": 0.1, + "data_dir": "/home/yurenh2/sdrn/data", + "depth": 20, + "device": "cuda:0", + "epochs": 20, + "eta_A": 0.001, + "eta_P": 0.01, + "eval_every": 0, + "eval_split": "validation", + "learn_P": 0, + "loader_seed": 0, + "lr": 0.03, + "lr_gamma": 0.1, + "lr_milestones": "100,150", + "lr_schedule": "cosine", + "max_steps": 0, + "mode": "sdil", + "momentum": 0.9, + "normalization": "batchnorm", + "nuisance_scale": 0.0, + "out": "results/oral_a_short/sdil_channel_gated_lr0.03.json", + "output_lr": 0.1, + "pert_directions": 1, + "pert_every": 4, + "pert_sigma": 0.01, + "perturb_seed": 1000, + "predictor_warmup_steps": 0, + "residual_scale": null, + "seed": 0, + "split_seed": 2027, + "train_limit": 10000, + "use_residual": 1, + "val_examples": 5000, + "vectorizer_mode": "channel_gated", + "warmup_epochs": 0, + "weight_decay": 0.0001, + "weight_scale": 1.0, + "width": 16 + }, + "counters": { + "apical_warmup_examples": 12688, + "calibration_event_examples": 62688, + "causal_scalar_observations": 990, + "logical_batch_loss_queries": 990, + "ordinary_examples": 200000, + "per_example_loss_terms": 125376, + "perturbation_events": 495, + "perturbation_forward_examples": 125376, + "predictor_warmup_examples": 0 + }, + "diagnostics": { + "early_third_mean": 0.08889565182228883, + "innovation_negative_gradient_cosine": [ + 0.05685567483305931, + 0.13134989142417908, + 0.08806467056274414, + 0.09720344841480255, + 0.06525854766368866, + 0.09464167803525925, + 0.02397942543029785, + 0.06854601949453354, + 0.07409851253032684, + 0.06666077673435211, + 0.07716609537601471, + 0.09574130177497864, + 0.034181758761405945, + 0.07528112828731537, + 0.1363137811422348, + 0.22182485461235046, + 0.13769981265068054, + 0.27749329805374146, + 0.11678922176361084 + ], + "normalization_state": "training_batch_stats_without_running_update", + "raw_negative_gradient_cosine": [ + 0.05685567483305931, + 0.13134989142417908, + 0.08806467056274414, + 0.09720344841480255, + 0.06525854766368866, + 0.09464167803525925, + 0.02397942543029785, + 0.06854601949453354, + 0.07409851253032684, + 0.06666077673435211, + 0.07716609537601471, + 0.09574130177497864, + 0.034181758761405945, + 0.07528112828731537, + 0.1363137811422348, + 0.22182485461235046, + 0.13769981265068054, + 0.27749329805374146, + 0.11678922176361084 + ], + "teaching_negative_gradient_cosine": [ + 0.05685567483305931, + 0.13134989142417908, + 0.08806467056274414, + 0.09720344841480255, + 0.06525854766368866, + 0.09464167803525925, + 0.02397942543029785, + 0.06854601949453354, + 0.07409851253032684, + 0.06666077673435211, + 0.07716609537601471, + 0.09574130177497864, + 0.034181758761405945, + 0.07528112828731537, + 0.1363137811422348, + 0.22182485461235046, + 0.13769981265068054, + 0.27749329805374146, + 0.11678922176361084 + ], + "wall_s": 0.10262227058410645 + }, + "epochs": [ + { + "calibration": { + "calibration_mse": 120.94099242469682, + "prediction_target_cosine": 1.506113768211827e-05, + "target_power": 120.94099207731854 + }, + "epoch": 1, + "lr": 0.03, + "output_lr": 0.1, + "step": 79, + "train_examples": 10000, + "train_loss": 2.7430426933288574 + }, + { + "calibration": { + "calibration_mse": 90.8829443671228, + "prediction_target_cosine": 4.748458338215727e-05, + "target_power": 90.88294743040333 + }, + "epoch": 2, + "lr": 0.029815325108927065, + "output_lr": 0.0993844170297569, + "step": 158, + "train_examples": 10000, + "train_loss": 2.6882332107543947 + }, + { + "calibration": { + "calibration_mse": 129.28391048064168, + "prediction_target_cosine": 2.1492919926159852e-05, + "target_power": 129.2838982463207 + }, + "epoch": 3, + "lr": 0.029265847744427302, + "output_lr": 0.09755282581475769, + "step": 237, + "train_examples": 10000, + "train_loss": 2.7864545795440674 + }, + { + "calibration": { + "calibration_mse": 114.01519549503767, + "prediction_target_cosine": 7.391323251578894e-05, + "target_power": 114.01519008284303 + }, + "epoch": 4, + "lr": 0.028365097862825515, + "output_lr": 0.09455032620941839, + "step": 316, + "train_examples": 10000, + "train_loss": 2.278718226623535 + }, + { + "calibration": { + "calibration_mse": 227.3724495573293, + "prediction_target_cosine": 4.533821080774298e-05, + "target_power": 227.37244714005323 + }, + "epoch": 5, + "lr": 0.02713525491562421, + "output_lr": 0.09045084971874738, + "step": 395, + "train_examples": 10000, + "train_loss": 2.3870798393249513 + }, + { + "calibration": { + "calibration_mse": 166.20845720914784, + "prediction_target_cosine": 6.765680817807511e-05, + "target_power": 166.20845642938562 + }, + "epoch": 6, + "lr": 0.02560660171779821, + "output_lr": 0.08535533905932738, + "step": 474, + "train_examples": 10000, + "train_loss": 2.2577471870422365 + }, + { + "calibration": { + "calibration_mse": 170.727521433827, + "prediction_target_cosine": -5.7050105486161436e-05, + "target_power": 170.72752845326198 + }, + "epoch": 7, + "lr": 0.023816778784387097, + "output_lr": 0.07938926261462366, + "step": 553, + "train_examples": 10000, + "train_loss": 2.12043060874939 + }, + { + "calibration": { + "calibration_mse": 209.52628865981265, + "prediction_target_cosine": -2.0228797603081682e-05, + "target_power": 209.52628848590894 + }, + "epoch": 8, + "lr": 0.0218098574960932, + "output_lr": 0.07269952498697733, + "step": 632, + "train_examples": 10000, + "train_loss": 2.059708542060852 + }, + { + "calibration": { + "calibration_mse": 141.29861236396044, + "prediction_target_cosine": -5.216942692074475e-06, + "target_power": 141.2986095050107 + }, + "epoch": 9, + "lr": 0.01963525491562421, + "output_lr": 0.06545084971874737, + "step": 711, + "train_examples": 10000, + "train_loss": 1.9873547559738158 + }, + { + "calibration": { + "calibration_mse": 252.29498001822952, + "prediction_target_cosine": 6.647677482561257e-05, + "target_power": 252.29497267383599 + }, + "epoch": 10, + "lr": 0.017346516975603462, + "output_lr": 0.05782172325201155, + "step": 790, + "train_examples": 10000, + "train_loss": 1.9427831512451172 + }, + { + "calibration": { + "calibration_mse": 112.07356150142402, + "prediction_target_cosine": -7.042171144742458e-05, + "target_power": 112.0735553999958 + }, + "epoch": 11, + "lr": 0.015, + "output_lr": 0.05, + "step": 869, + "train_examples": 10000, + "train_loss": 1.8960050271987916 + }, + { + "calibration": { + "calibration_mse": 275.5549939046497, + "prediction_target_cosine": 2.7840606439390747e-05, + "target_power": 275.55498504243127 + }, + "epoch": 12, + "lr": 0.012653483024396533, + "output_lr": 0.04217827674798845, + "step": 948, + "train_examples": 10000, + "train_loss": 1.7632465076446533 + }, + { + "calibration": { + "calibration_mse": 391.5784136378246, + "prediction_target_cosine": -4.5330952303582855e-05, + "target_power": 391.57838901851494 + }, + "epoch": 13, + "lr": 0.01036474508437579, + "output_lr": 0.03454915028125263, + "step": 1027, + "train_examples": 10000, + "train_loss": 1.7816886276245116 + }, + { + "calibration": { + "calibration_mse": 288.2343263507861, + "prediction_target_cosine": 9.16609312615192e-05, + "target_power": 288.23433236402013 + }, + "epoch": 14, + "lr": 0.0081901425039068, + "output_lr": 0.027300475013022664, + "step": 1106, + "train_examples": 10000, + "train_loss": 1.7592916584014893 + }, + { + "calibration": { + "calibration_mse": 162.82111501402179, + "prediction_target_cosine": 5.7469316424673235e-05, + "target_power": 162.82111272915546 + }, + "epoch": 15, + "lr": 0.006183221215612904, + "output_lr": 0.02061073738537635, + "step": 1185, + "train_examples": 10000, + "train_loss": 1.683985795021057 + }, + { + "calibration": { + "calibration_mse": 177.96064978913256, + "prediction_target_cosine": -4.3211184289433225e-05, + "target_power": 177.96064038461319 + }, + "epoch": 16, + "lr": 0.004393398282201788, + "output_lr": 0.014644660940672627, + "step": 1264, + "train_examples": 10000, + "train_loss": 1.6693348972320556 + }, + { + "calibration": { + "calibration_mse": 116.30543024762251, + "prediction_target_cosine": 5.207735526040399e-05, + "target_power": 116.30542734543103 + }, + "epoch": 17, + "lr": 0.00286474508437579, + "output_lr": 0.009549150281252633, + "step": 1343, + "train_examples": 10000, + "train_loss": 1.6360828407287598 + }, + { + "calibration": { + "calibration_mse": 224.51228453390218, + "prediction_target_cosine": 8.341093247343407e-05, + "target_power": 224.5122794786468 + }, + "epoch": 18, + "lr": 0.001634902137174483, + "output_lr": 0.005449673790581611, + "step": 1422, + "train_examples": 10000, + "train_loss": 1.614950588989258 + }, + { + "calibration": { + "calibration_mse": 227.45811103251967, + "prediction_target_cosine": 7.730374096652188e-05, + "target_power": 227.45810761075955 + }, + "epoch": 19, + "lr": 0.000734152255572697, + "output_lr": 0.0024471741852423235, + "step": 1501, + "train_examples": 10000, + "train_loss": 1.5980025550842285 + }, + { + "calibration": { + "calibration_mse": 233.88802596526028, + "prediction_target_cosine": 6.816316473038772e-06, + "target_power": 233.88802621876758 + }, + "epoch": 20, + "lr": 0.0001846748910729351, + "output_lr": 0.0006155829702431171, + "step": 1580, + "train_examples": 10000, + "train_loss": 1.598771397781372 + } + ], + "evaluation_protocol": { + "test_evaluations": 0, + "test_used_for_selection": false, + "validation_evaluations": 1 + }, + "final": { + "accuracy": 0.4198, + "epoch": 20, + "evaluation_split": "validation", + "finite": true, + "loss": 1.60240986328125, + "step": 1580 + }, + "hardware": { + "cuda_device_name": "NVIDIA GeForce GTX 1080", + "cuda_visible_devices": "5", + "device": "cuda:0", + "device_total_memory_bytes": 8507949056, + "peak_memory_allocated_bytes": 1733492736, + "peak_memory_reserved_bytes": 2858418176, + "torch_version": "2.3.1+cu118" + }, + "protocol_family": "oral_a_cifar_local_resnet_development", + "provenance": { + "git_commit": "a1d060e6d96d4ab973d16cd75149dd61899c7c95", + "git_tracked_dirty": false + }, + "schema_version": 1, + "split": { + "cifar_source_files": [ + { + "bytes": 31035704, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/data_batch_1", + "sha256": "54636561a3ce25bd3e19253c6b0d8538147b0ae398331ac4a2d86c6d987368cd" + }, + { + "bytes": 31035320, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/data_batch_2", + "sha256": "766b2cef9fbc745cf056b3152224f7cf77163b330ea9a15f9392beb8b89bc5a8" + }, + { + "bytes": 31035999, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/data_batch_3", + "sha256": "0f00d98ebfb30b3ec0ad19f9756dc2630b89003e10525f5e148445e82aa6a1f9" + }, + { + "bytes": 31035696, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/data_batch_4", + "sha256": "3f7bb240661948b8f4d53e36ec720d8306f5668bd0071dcb4e6c947f78e9682b" + }, + { + "bytes": 31035623, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/data_batch_5", + "sha256": "d91802434d8376bbaeeadf58a737e3a1b12ac839077e931237e0dcd43adcb154" + }, + { + "bytes": 31035526, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/test_batch", + "sha256": "f53d8d457504f7cff4ea9e021afcf0e0ad8e24a91f3fc42091b8adef61157831" + } + ], + "dataset": "cifar10", + "evaluation_split": "validation", + "input_layout": "NCHW", + "input_shape": [ + 3, + 32, + 32 + ], + "loader_seed": 0, + "normalization_mean": [ + 0.49140000343322754, + 0.4821999967098236, + 0.4465000033378601 + ], + "normalization_std": [ + 0.24699999392032623, + 0.2434999942779541, + 0.26159998774528503 + ], + "split_from_training_only": true, + "split_seed": 2027, + "test_examples": 10000, + "train_examples": 10000, + "training_augmentation": "random_crop_32_padding_4_zero_then_horizontal_flip_p0.5", + "validation_class_counts": { + "0": 500, + "1": 500, + "2": 500, + "3": 500, + "4": 500, + "5": 500, + "6": 500, + "7": 500, + "8": 500, + "9": 500 + }, + "validation_examples": 5000, + "validation_index_sha256": "8328b206a97c420e49e54e3eca4abe3274c4756b084355784ea3fb8059e4515b" + }, + "timing": { + "apical_warmup_wall_s": 6.487434387207031, + "evaluation_wall_s": 0.372774600982666, + "predictor_warmup_wall_s": 0.0, + "timing_excludes_data_loading_hashing_and_model_construction": true, + "total_timed_wall_s": 100.67689514160156, + "train_wall_s": 93.81352663040161 + }, + "work": { + "apical_macs_per_example": 202176, + "causal_scalar_observations": 990, + "components": { + "apical_projection_macs": 43000409088, + "apical_regression_macs": 12674009088, + "bp_reverse_macs_estimate": 0, + "local_weight_correlation_macs": 8110208000000, + "ordinary_forward_macs": 8110208000000, + "perturbation_forward_macs": 5084127191040, + "warmup_clean_forward_macs": 514511595520 + }, + "definition": "multiply-accumulates in conv/linear maps; one local weight correlation equals one forward-weight MAC count; BP reverse is estimated as one weight-gradient plus one activation-gradient convolution per forward convolution; elementwise nonlinearities and optimizer arithmetic excluded", + "forward_macs_per_example": 40551040, + "logical_batch_loss_queries": 990, + "per_example_cross_entropy_terms": 125376, + "total_clean_forward_examples": 212688, + "total_forward_equivalent_examples": 338064, + "total_macs_estimate": 21874729204736 + } +} diff --git a/results/oral_a_short/sdil_channel_gated_lr0.1.json b/results/oral_a_short/sdil_channel_gated_lr0.1.json new file mode 100644 index 0000000..a546cab --- /dev/null +++ b/results/oral_a_short/sdil_channel_gated_lr0.1.json @@ -0,0 +1,639 @@ +{ + "apical_warmup": { + "last": { + "calibration_mse": 518.2959050717561, + "prediction_target_cosine": 0.00016553175127441687, + "target_power": 518.2958984375 + }, + "steps": 100 + }, + "architecture": { + "adaptive_apical_parameters": 390592, + "base_width": 16, + "blocks_per_stage": 3, + "bn_eps": 1e-05, + "bn_momentum": 0.1, + "depth": 20, + "family": "CIFAR 6n+2 ResNet, option-A shortcuts", + "fixed_traffic_coefficients": 188416, + "forward_parameters": 269722, + "hidden_shapes": [ + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 32, + 16, + 16 + ], + [ + 32, + 16, + 16 + ], + [ + 32, + 16, + 16 + ], + [ + 32, + 16, + 16 + ], + [ + 32, + 16, + 16 + ], + [ + 32, + 16, + 16 + ], + [ + 64, + 8, + 8 + ], + [ + 64, + 8, + 8 + ], + [ + 64, + 8, + 8 + ], + [ + 64, + 8, + 8 + ], + [ + 64, + 8, + 8 + ], + [ + 64, + 8, + 8 + ] + ], + "normalization": "batchnorm", + "predictor_parameters": 376832, + "residual_scale": 1.0, + "vectorizer_mode": "channel_gated", + "vectorizer_parameters": 13760 + }, + "args": { + "a_scale": 1.0, + "a_warmup_steps": 100, + "alignment_probe": 32, + "apical_seed": null, + "augment_train": 1, + "batch_size": 128, + "bn_eps": 1e-05, + "bn_momentum": 0.1, + "data_dir": "/home/yurenh2/sdrn/data", + "depth": 20, + "device": "cuda:0", + "epochs": 20, + "eta_A": 0.001, + "eta_P": 0.01, + "eval_every": 0, + "eval_split": "validation", + "learn_P": 0, + "loader_seed": 0, + "lr": 0.1, + "lr_gamma": 0.1, + "lr_milestones": "100,150", + "lr_schedule": "cosine", + "max_steps": 0, + "mode": "sdil", + "momentum": 0.9, + "normalization": "batchnorm", + "nuisance_scale": 0.0, + "out": "results/oral_a_short/sdil_channel_gated_lr0.1.json", + "output_lr": 0.1, + "pert_directions": 1, + "pert_every": 4, + "pert_sigma": 0.01, + "perturb_seed": 1000, + "predictor_warmup_steps": 0, + "residual_scale": null, + "seed": 0, + "split_seed": 2027, + "train_limit": 10000, + "use_residual": 1, + "val_examples": 5000, + "vectorizer_mode": "channel_gated", + "warmup_epochs": 0, + "weight_decay": 0.0001, + "weight_scale": 1.0, + "width": 16 + }, + "counters": { + "apical_warmup_examples": 12688, + "calibration_event_examples": 62688, + "causal_scalar_observations": 990, + "logical_batch_loss_queries": 990, + "ordinary_examples": 200000, + "per_example_loss_terms": 125376, + "perturbation_events": 495, + "perturbation_forward_examples": 125376, + "predictor_warmup_examples": 0 + }, + "diagnostics": { + "early_third_mean": 0.08000707471122344, + "innovation_negative_gradient_cosine": [ + 0.1133790984749794, + 0.12308487296104431, + 0.09794244170188904, + 0.07138215750455856, + 0.05488227307796478, + 0.019371604546904564, + 0.015585479326546192, + 0.04312252625823021, + 0.006569968536496162, + 0.048957690596580505, + 0.041394077241420746, + -0.03040825016796589, + 0.023677144199609756, + 0.11618378013372421, + 0.185902938246727, + 0.21603626012802124, + 0.1565398871898651, + 0.16705910861492157, + -0.004543994553387165 + ], + "normalization_state": "training_batch_stats_without_running_update", + "raw_negative_gradient_cosine": [ + 0.1133790984749794, + 0.12308487296104431, + 0.09794244170188904, + 0.07138215750455856, + 0.05488227307796478, + 0.019371604546904564, + 0.015585479326546192, + 0.04312252625823021, + 0.006569968536496162, + 0.048957690596580505, + 0.041394077241420746, + -0.03040825016796589, + 0.023677144199609756, + 0.11618378013372421, + 0.185902938246727, + 0.21603626012802124, + 0.1565398871898651, + 0.16705910861492157, + -0.004543994553387165 + ], + "teaching_negative_gradient_cosine": [ + 0.1133790984749794, + 0.12308487296104431, + 0.09794244170188904, + 0.07138215750455856, + 0.05488227307796478, + 0.019371604546904564, + 0.015585479326546192, + 0.04312252625823021, + 0.006569968536496162, + 0.048957690596580505, + 0.041394077241420746, + -0.03040825016796589, + 0.023677144199609756, + 0.11618378013372421, + 0.185902938246727, + 0.21603626012802124, + 0.1565398871898651, + 0.16705910861492157, + -0.004543994553387165 + ], + "wall_s": 0.10671830177307129 + }, + "epochs": [ + { + "calibration": { + "calibration_mse": 60.43777185808058, + "prediction_target_cosine": -6.181264794266107e-05, + "target_power": 60.43776736048902 + }, + "epoch": 1, + "lr": 0.1, + "output_lr": 0.1, + "step": 79, + "train_examples": 10000, + "train_loss": 2.5334342124938964 + }, + { + "calibration": { + "calibration_mse": 23.54987083374619, + "prediction_target_cosine": -6.96036363511297e-05, + "target_power": 23.549866315204163 + }, + "epoch": 2, + "lr": 0.0993844170297569, + "output_lr": 0.0993844170297569, + "step": 158, + "train_examples": 10000, + "train_loss": 2.5487233268737794 + }, + { + "calibration": { + "calibration_mse": 11.827637325330516, + "prediction_target_cosine": -3.0372441458224223e-06, + "target_power": 11.827633745229358 + }, + "epoch": 3, + "lr": 0.09755282581475769, + "output_lr": 0.09755282581475769, + "step": 237, + "train_examples": 10000, + "train_loss": 2.461765655517578 + }, + { + "calibration": { + "calibration_mse": 30.754039637663446, + "prediction_target_cosine": 1.8163386607575518e-05, + "target_power": 30.754037120982627 + }, + "epoch": 4, + "lr": 0.09455032620941839, + "output_lr": 0.09455032620941839, + "step": 316, + "train_examples": 10000, + "train_loss": 2.3639930412292482 + }, + { + "calibration": { + "calibration_mse": 52.041269661478054, + "prediction_target_cosine": 7.097774633237941e-05, + "target_power": 52.041270009258184 + }, + "epoch": 5, + "lr": 0.09045084971874738, + "output_lr": 0.09045084971874738, + "step": 395, + "train_examples": 10000, + "train_loss": 2.3211933601379395 + }, + { + "calibration": { + "calibration_mse": 42.16324888252278, + "prediction_target_cosine": 7.073739299367894e-05, + "target_power": 42.16324767635568 + }, + "epoch": 6, + "lr": 0.08535533905932738, + "output_lr": 0.08535533905932738, + "step": 474, + "train_examples": 10000, + "train_loss": 2.2405354804992674 + }, + { + "calibration": { + "calibration_mse": 96.60961034550614, + "prediction_target_cosine": -2.6760619008438326e-05, + "target_power": 96.60960379281771 + }, + "epoch": 7, + "lr": 0.07938926261462366, + "output_lr": 0.07938926261462366, + "step": 553, + "train_examples": 10000, + "train_loss": 2.1257784160614013 + }, + { + "calibration": { + "calibration_mse": 95.43317693296298, + "prediction_target_cosine": -1.0208342923692714e-05, + "target_power": 95.43316866958713 + }, + "epoch": 8, + "lr": 0.07269952498697733, + "output_lr": 0.07269952498697733, + "step": 632, + "train_examples": 10000, + "train_loss": 2.2211479858398437 + }, + { + "calibration": { + "calibration_mse": 66.98153618496397, + "prediction_target_cosine": -1.2381250456731113e-05, + "target_power": 66.98153039761213 + }, + "epoch": 9, + "lr": 0.06545084971874737, + "output_lr": 0.06545084971874737, + "step": 711, + "train_examples": 10000, + "train_loss": 2.083836813926697 + }, + { + "calibration": { + "calibration_mse": 146.6930261971186, + "prediction_target_cosine": -5.327456506330552e-07, + "target_power": 146.69302486945753 + }, + "epoch": 10, + "lr": 0.05782172325201155, + "output_lr": 0.05782172325201155, + "step": 790, + "train_examples": 10000, + "train_loss": 1.9434806201934816 + }, + { + "calibration": { + "calibration_mse": 80.2759034222883, + "prediction_target_cosine": 4.8706224176594515e-05, + "target_power": 80.27590192608213 + }, + "epoch": 11, + "lr": 0.05, + "output_lr": 0.05, + "step": 869, + "train_examples": 10000, + "train_loss": 1.9414970417022706 + }, + { + "calibration": { + "calibration_mse": 96.45602903597789, + "prediction_target_cosine": -4.129046053586153e-05, + "target_power": 96.45602758309765 + }, + "epoch": 12, + "lr": 0.04217827674798845, + "output_lr": 0.04217827674798845, + "step": 948, + "train_examples": 10000, + "train_loss": 1.8372822145462036 + }, + { + "calibration": { + "calibration_mse": 176.84556970363073, + "prediction_target_cosine": -1.9519702580488654e-05, + "target_power": 176.84556127799596 + }, + "epoch": 13, + "lr": 0.03454915028125263, + "output_lr": 0.03454915028125263, + "step": 1027, + "train_examples": 10000, + "train_loss": 1.7941364185333253 + }, + { + "calibration": { + "calibration_mse": 75.1211485477286, + "prediction_target_cosine": 0.00010757188693459416, + "target_power": 75.12114817083045 + }, + "epoch": 14, + "lr": 0.027300475013022664, + "output_lr": 0.027300475013022664, + "step": 1106, + "train_examples": 10000, + "train_loss": 1.7450150983810424 + }, + { + "calibration": { + "calibration_mse": 114.3603014383627, + "prediction_target_cosine": -9.490821681384313e-07, + "target_power": 114.36030431791491 + }, + "epoch": 15, + "lr": 0.02061073738537635, + "output_lr": 0.02061073738537635, + "step": 1185, + "train_examples": 10000, + "train_loss": 1.6714060207366943 + }, + { + "calibration": { + "calibration_mse": 122.73489581566791, + "prediction_target_cosine": 4.2819634654651794e-05, + "target_power": 122.7348966235018 + }, + "epoch": 16, + "lr": 0.014644660940672627, + "output_lr": 0.014644660940672627, + "step": 1264, + "train_examples": 10000, + "train_loss": 1.6516524749755859 + }, + { + "calibration": { + "calibration_mse": 42.00022924316126, + "prediction_target_cosine": -4.6748324469624814e-05, + "target_power": 42.000227053771205 + }, + "epoch": 17, + "lr": 0.009549150281252633, + "output_lr": 0.009549150281252633, + "step": 1343, + "train_examples": 10000, + "train_loss": 1.6312512542724609 + }, + { + "calibration": { + "calibration_mse": 146.15481284242125, + "prediction_target_cosine": -1.823520416534038e-05, + "target_power": 146.15481366035732 + }, + "epoch": 18, + "lr": 0.005449673790581611, + "output_lr": 0.005449673790581611, + "step": 1422, + "train_examples": 10000, + "train_loss": 1.592088447189331 + }, + { + "calibration": { + "calibration_mse": 50.12404485339623, + "prediction_target_cosine": 7.522739697549936e-05, + "target_power": 50.12404224670939 + }, + "epoch": 19, + "lr": 0.0024471741852423235, + "output_lr": 0.0024471741852423235, + "step": 1501, + "train_examples": 10000, + "train_loss": 1.5661032293319703 + }, + { + "calibration": { + "calibration_mse": 81.33399665307944, + "prediction_target_cosine": 8.698869592416884e-05, + "target_power": 81.33399560536073 + }, + "epoch": 20, + "lr": 0.0006155829702431171, + "output_lr": 0.0006155829702431171, + "step": 1580, + "train_examples": 10000, + "train_loss": 1.5665943405151368 + } + ], + "evaluation_protocol": { + "test_evaluations": 0, + "test_used_for_selection": false, + "validation_evaluations": 1 + }, + "final": { + "accuracy": 0.414, + "epoch": 20, + "evaluation_split": "validation", + "finite": true, + "loss": 1.597408447265625, + "step": 1580 + }, + "hardware": { + "cuda_device_name": "NVIDIA GeForce GTX 1080", + "cuda_visible_devices": "5", + "device": "cuda:0", + "device_total_memory_bytes": 8507949056, + "peak_memory_allocated_bytes": 1733492736, + "peak_memory_reserved_bytes": 2858418176, + "torch_version": "2.3.1+cu118" + }, + "protocol_family": "oral_a_cifar_local_resnet_development", + "provenance": { + "git_commit": "a1d060e6d96d4ab973d16cd75149dd61899c7c95", + "git_tracked_dirty": false + }, + "schema_version": 1, + "split": { + "cifar_source_files": [ + { + "bytes": 31035704, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/data_batch_1", + "sha256": "54636561a3ce25bd3e19253c6b0d8538147b0ae398331ac4a2d86c6d987368cd" + }, + { + "bytes": 31035320, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/data_batch_2", + "sha256": "766b2cef9fbc745cf056b3152224f7cf77163b330ea9a15f9392beb8b89bc5a8" + }, + { + "bytes": 31035999, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/data_batch_3", + "sha256": "0f00d98ebfb30b3ec0ad19f9756dc2630b89003e10525f5e148445e82aa6a1f9" + }, + { + "bytes": 31035696, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/data_batch_4", + "sha256": "3f7bb240661948b8f4d53e36ec720d8306f5668bd0071dcb4e6c947f78e9682b" + }, + { + "bytes": 31035623, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/data_batch_5", + "sha256": "d91802434d8376bbaeeadf58a737e3a1b12ac839077e931237e0dcd43adcb154" + }, + { + "bytes": 31035526, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/test_batch", + "sha256": "f53d8d457504f7cff4ea9e021afcf0e0ad8e24a91f3fc42091b8adef61157831" + } + ], + "dataset": "cifar10", + "evaluation_split": "validation", + "input_layout": "NCHW", + "input_shape": [ + 3, + 32, + 32 + ], + "loader_seed": 0, + "normalization_mean": [ + 0.49140000343322754, + 0.4821999967098236, + 0.4465000033378601 + ], + "normalization_std": [ + 0.24699999392032623, + 0.2434999942779541, + 0.26159998774528503 + ], + "split_from_training_only": true, + "split_seed": 2027, + "test_examples": 10000, + "train_examples": 10000, + "training_augmentation": "random_crop_32_padding_4_zero_then_horizontal_flip_p0.5", + "validation_class_counts": { + "0": 500, + "1": 500, + "2": 500, + "3": 500, + "4": 500, + "5": 500, + "6": 500, + "7": 500, + "8": 500, + "9": 500 + }, + "validation_examples": 5000, + "validation_index_sha256": "8328b206a97c420e49e54e3eca4abe3274c4756b084355784ea3fb8059e4515b" + }, + "timing": { + "apical_warmup_wall_s": 6.4902708530426025, + "evaluation_wall_s": 0.36696362495422363, + "predictor_warmup_wall_s": 0.0, + "timing_excludes_data_loading_hashing_and_model_construction": true, + "total_timed_wall_s": 100.41240739822388, + "train_wall_s": 93.55240631103516 + }, + "work": { + "apical_macs_per_example": 202176, + "causal_scalar_observations": 990, + "components": { + "apical_projection_macs": 43000409088, + "apical_regression_macs": 12674009088, + "bp_reverse_macs_estimate": 0, + "local_weight_correlation_macs": 8110208000000, + "ordinary_forward_macs": 8110208000000, + "perturbation_forward_macs": 5084127191040, + "warmup_clean_forward_macs": 514511595520 + }, + "definition": "multiply-accumulates in conv/linear maps; one local weight correlation equals one forward-weight MAC count; BP reverse is estimated as one weight-gradient plus one activation-gradient convolution per forward convolution; elementwise nonlinearities and optimizer arithmetic excluded", + "forward_macs_per_example": 40551040, + "logical_batch_loss_queries": 990, + "per_example_cross_entropy_terms": 125376, + "total_clean_forward_examples": 212688, + "total_forward_equivalent_examples": 338064, + "total_macs_estimate": 21874729204736 + } +} diff --git a/results/oral_a_short/sdil_spatial_template_lr0.01.json b/results/oral_a_short/sdil_spatial_template_lr0.01.json new file mode 100644 index 0000000..f040658 --- /dev/null +++ b/results/oral_a_short/sdil_spatial_template_lr0.01.json @@ -0,0 +1,639 @@ +{ + "apical_warmup": { + "last": { + "calibration_mse": 518.2958984375, + "prediction_target_cosine": 0.0001586056141805217, + "target_power": 518.2958984375 + }, + "steps": 100 + }, + "architecture": { + "adaptive_apical_parameters": 2260992, + "base_width": 16, + "blocks_per_stage": 3, + "bn_eps": 1e-05, + "bn_momentum": 0.1, + "depth": 20, + "family": "CIFAR 6n+2 ResNet, option-A shortcuts", + "fixed_traffic_coefficients": 188416, + "forward_parameters": 269722, + "hidden_shapes": [ + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 32, + 16, + 16 + ], + [ + 32, + 16, + 16 + ], + [ + 32, + 16, + 16 + ], + [ + 32, + 16, + 16 + ], + [ + 32, + 16, + 16 + ], + [ + 32, + 16, + 16 + ], + [ + 64, + 8, + 8 + ], + [ + 64, + 8, + 8 + ], + [ + 64, + 8, + 8 + ], + [ + 64, + 8, + 8 + ], + [ + 64, + 8, + 8 + ], + [ + 64, + 8, + 8 + ] + ], + "normalization": "batchnorm", + "predictor_parameters": 376832, + "residual_scale": 1.0, + "vectorizer_mode": "spatial_template", + "vectorizer_parameters": 1884160 + }, + "args": { + "a_scale": 0.0, + "a_warmup_steps": 100, + "alignment_probe": 32, + "apical_seed": null, + "augment_train": 1, + "batch_size": 128, + "bn_eps": 1e-05, + "bn_momentum": 0.1, + "data_dir": "/home/yurenh2/sdrn/data", + "depth": 20, + "device": "cuda:0", + "epochs": 20, + "eta_A": 0.001, + "eta_P": 0.01, + "eval_every": 0, + "eval_split": "validation", + "learn_P": 0, + "loader_seed": 0, + "lr": 0.01, + "lr_gamma": 0.1, + "lr_milestones": "100,150", + "lr_schedule": "cosine", + "max_steps": 0, + "mode": "sdil", + "momentum": 0.9, + "normalization": "batchnorm", + "nuisance_scale": 0.0, + "out": "results/oral_a_short/sdil_spatial_template_lr0.01.json", + "output_lr": 0.1, + "pert_directions": 1, + "pert_every": 4, + "pert_sigma": 0.01, + "perturb_seed": 1000, + "predictor_warmup_steps": 0, + "residual_scale": null, + "seed": 0, + "split_seed": 2027, + "train_limit": 10000, + "use_residual": 1, + "val_examples": 5000, + "vectorizer_mode": "spatial_template", + "warmup_epochs": 0, + "weight_decay": 0.0001, + "weight_scale": 1.0, + "width": 16 + }, + "counters": { + "apical_warmup_examples": 12688, + "calibration_event_examples": 62688, + "causal_scalar_observations": 990, + "logical_batch_loss_queries": 990, + "ordinary_examples": 200000, + "per_example_loss_terms": 125376, + "perturbation_events": 495, + "perturbation_forward_examples": 125376, + "predictor_warmup_examples": 0 + }, + "diagnostics": { + "early_third_mean": 0.001612375567977627, + "innovation_negative_gradient_cosine": [ + 0.001027052290737629, + -0.0022626847494393587, + 0.003048800863325596, + 0.0009945780038833618, + 0.003387338714674115, + 0.0034791682846844196, + 0.0036151609383523464, + 0.0016754134558141232, + -0.00018921078299172223, + 0.0040879761800169945, + 0.006074178963899612, + 0.0037323604337871075, + 0.002060003811493516, + 0.007457074709236622, + 0.008745873346924782, + 0.01761583611369133, + 0.004088421352207661, + 0.02299105003476143, + 0.015363558195531368 + ], + "normalization_state": "training_batch_stats_without_running_update", + "raw_negative_gradient_cosine": [ + 0.001027052290737629, + -0.0022626847494393587, + 0.003048800863325596, + 0.0009945780038833618, + 0.003387338714674115, + 0.0034791682846844196, + 0.0036151609383523464, + 0.0016754134558141232, + -0.00018921078299172223, + 0.0040879761800169945, + 0.006074178963899612, + 0.0037323604337871075, + 0.002060003811493516, + 0.007457074709236622, + 0.008745873346924782, + 0.01761583611369133, + 0.004088421352207661, + 0.02299105003476143, + 0.015363558195531368 + ], + "teaching_negative_gradient_cosine": [ + 0.001027052290737629, + -0.0022626847494393587, + 0.003048800863325596, + 0.0009945780038833618, + 0.003387338714674115, + 0.0034791682846844196, + 0.0036151609383523464, + 0.0016754134558141232, + -0.00018921078299172223, + 0.0040879761800169945, + 0.006074178963899612, + 0.0037323604337871075, + 0.002060003811493516, + 0.007457074709236622, + 0.008745873346924782, + 0.01761583611369133, + 0.004088421352207661, + 0.02299105003476143, + 0.015363558195531368 + ], + "wall_s": 0.1027383804321289 + }, + "epochs": [ + { + "calibration": { + "calibration_mse": 311.4595984723095, + "prediction_target_cosine": -2.2504479337353996e-05, + "target_power": 311.4595591177434 + }, + "epoch": 1, + "lr": 0.01, + "output_lr": 0.1, + "step": 79, + "train_examples": 10000, + "train_loss": 2.774099029159546 + }, + { + "calibration": { + "calibration_mse": 1526.5578844609468, + "prediction_target_cosine": 2.554489113023116e-06, + "target_power": 1526.557862022649 + }, + "epoch": 2, + "lr": 0.009938441702975689, + "output_lr": 0.0993844170297569, + "step": 158, + "train_examples": 10000, + "train_loss": 2.431015293121338 + }, + { + "calibration": { + "calibration_mse": 1242.6608512818873, + "prediction_target_cosine": -3.991147778175767e-05, + "target_power": 1242.6608249527821 + }, + "epoch": 3, + "lr": 0.009755282581475769, + "output_lr": 0.09755282581475769, + "step": 237, + "train_examples": 10000, + "train_loss": 2.466135598754883 + }, + { + "calibration": { + "calibration_mse": 634.9810205836453, + "prediction_target_cosine": -3.91370314985448e-06, + "target_power": 634.980954601372 + }, + "epoch": 4, + "lr": 0.00945503262094184, + "output_lr": 0.09455032620941839, + "step": 316, + "train_examples": 10000, + "train_loss": 2.6161225547790528 + }, + { + "calibration": { + "calibration_mse": 250.2808392957501, + "prediction_target_cosine": 4.677621205119292e-05, + "target_power": 250.28076969955282 + }, + "epoch": 5, + "lr": 0.009045084971874737, + "output_lr": 0.09045084971874738, + "step": 395, + "train_examples": 10000, + "train_loss": 2.443944037628174 + }, + { + "calibration": { + "calibration_mse": 280.3610884783708, + "prediction_target_cosine": 3.712464558796983e-08, + "target_power": 280.3610162495271 + }, + "epoch": 6, + "lr": 0.008535533905932738, + "output_lr": 0.08535533905932738, + "step": 474, + "train_examples": 10000, + "train_loss": 2.4743158672332766 + }, + { + "calibration": { + "calibration_mse": 397.6820018188137, + "prediction_target_cosine": -3.488966965419155e-05, + "target_power": 397.68190724253657 + }, + "epoch": 7, + "lr": 0.007938926261462366, + "output_lr": 0.07938926261462366, + "step": 553, + "train_examples": 10000, + "train_loss": 2.401729510116577 + }, + { + "calibration": { + "calibration_mse": 694.6581847718459, + "prediction_target_cosine": -1.5154622745509205e-05, + "target_power": 694.6580754621829 + }, + "epoch": 8, + "lr": 0.007269952498697734, + "output_lr": 0.07269952498697733, + "step": 632, + "train_examples": 10000, + "train_loss": 2.340212138366699 + }, + { + "calibration": { + "calibration_mse": 447.4696043644261, + "prediction_target_cosine": -5.656511341783755e-06, + "target_power": 447.4694925629574 + }, + "epoch": 9, + "lr": 0.006545084971874737, + "output_lr": 0.06545084971874737, + "step": 711, + "train_examples": 10000, + "train_loss": 2.2473524238586426 + }, + { + "calibration": { + "calibration_mse": 225.01660751981012, + "prediction_target_cosine": 2.417077360226098e-05, + "target_power": 225.016517061643 + }, + "epoch": 10, + "lr": 0.0057821723252011546, + "output_lr": 0.05782172325201155, + "step": 790, + "train_examples": 10000, + "train_loss": 2.1798657495498657 + }, + { + "calibration": { + "calibration_mse": 518.3563566866129, + "prediction_target_cosine": 1.0120506274893608e-05, + "target_power": 518.3562297463417 + }, + "epoch": 11, + "lr": 0.005, + "output_lr": 0.05, + "step": 869, + "train_examples": 10000, + "train_loss": 2.142610039138794 + }, + { + "calibration": { + "calibration_mse": 298.18500603163864, + "prediction_target_cosine": 2.516028858223128e-06, + "target_power": 298.1849143187552 + }, + "epoch": 12, + "lr": 0.004217827674798845, + "output_lr": 0.04217827674798845, + "step": 948, + "train_examples": 10000, + "train_loss": 2.0082340023040772 + }, + { + "calibration": { + "calibration_mse": 816.293761626534, + "prediction_target_cosine": -3.1195302557482935e-05, + "target_power": 816.2936095196268 + }, + "epoch": 13, + "lr": 0.003454915028125263, + "output_lr": 0.03454915028125263, + "step": 1027, + "train_examples": 10000, + "train_loss": 1.9961866901397705 + }, + { + "calibration": { + "calibration_mse": 557.8168947697982, + "prediction_target_cosine": 1.1375643767241642e-05, + "target_power": 557.8167687418669 + }, + "epoch": 14, + "lr": 0.0027300475013022664, + "output_lr": 0.027300475013022664, + "step": 1106, + "train_examples": 10000, + "train_loss": 1.9453593284606934 + }, + { + "calibration": { + "calibration_mse": 184.58966761069047, + "prediction_target_cosine": -4.041631141428753e-05, + "target_power": 184.58953192374145 + }, + "epoch": 15, + "lr": 0.0020610737385376348, + "output_lr": 0.02061073738537635, + "step": 1185, + "train_examples": 10000, + "train_loss": 1.9042009094238281 + }, + { + "calibration": { + "calibration_mse": 281.9687953793978, + "prediction_target_cosine": 2.328571111638906e-05, + "target_power": 281.9686650505601 + }, + "epoch": 16, + "lr": 0.0014644660940672626, + "output_lr": 0.014644660940672627, + "step": 1264, + "train_examples": 10000, + "train_loss": 1.8901697011947631 + }, + { + "calibration": { + "calibration_mse": 325.46421335831934, + "prediction_target_cosine": 5.358104560800165e-05, + "target_power": 325.4640667039415 + }, + "epoch": 17, + "lr": 0.0009549150281252633, + "output_lr": 0.009549150281252633, + "step": 1343, + "train_examples": 10000, + "train_loss": 1.8364084032058716 + }, + { + "calibration": { + "calibration_mse": 344.07036824645877, + "prediction_target_cosine": -6.079723779001693e-05, + "target_power": 344.07023295721285 + }, + "epoch": 18, + "lr": 0.0005449673790581611, + "output_lr": 0.005449673790581611, + "step": 1422, + "train_examples": 10000, + "train_loss": 1.816278349876404 + }, + { + "calibration": { + "calibration_mse": 371.9894568442003, + "prediction_target_cosine": -2.267388030846593e-05, + "target_power": 371.9893169559863 + }, + "epoch": 19, + "lr": 0.00024471741852423234, + "output_lr": 0.0024471741852423235, + "step": 1501, + "train_examples": 10000, + "train_loss": 1.7986092542648315 + }, + { + "calibration": { + "calibration_mse": 531.5486738582225, + "prediction_target_cosine": -2.485824704439809e-05, + "target_power": 531.548502974843 + }, + "epoch": 20, + "lr": 6.15582970243117e-05, + "output_lr": 0.0006155829702431171, + "step": 1580, + "train_examples": 10000, + "train_loss": 1.7901613536834717 + } + ], + "evaluation_protocol": { + "test_evaluations": 0, + "test_used_for_selection": false, + "validation_evaluations": 1 + }, + "final": { + "accuracy": 0.3572, + "epoch": 20, + "evaluation_split": "validation", + "finite": true, + "loss": 1.7704051513671875, + "step": 1580 + }, + "hardware": { + "cuda_device_name": "NVIDIA GeForce GTX 1080", + "cuda_visible_devices": "5", + "device": "cuda:0", + "device_total_memory_bytes": 8507949056, + "peak_memory_allocated_bytes": 1740965888, + "peak_memory_reserved_bytes": 2866806784, + "torch_version": "2.3.1+cu118" + }, + "protocol_family": "oral_a_cifar_local_resnet_development", + "provenance": { + "git_commit": "a1d060e6d96d4ab973d16cd75149dd61899c7c95", + "git_tracked_dirty": false + }, + "schema_version": 1, + "split": { + "cifar_source_files": [ + { + "bytes": 31035704, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/data_batch_1", + "sha256": "54636561a3ce25bd3e19253c6b0d8538147b0ae398331ac4a2d86c6d987368cd" + }, + { + "bytes": 31035320, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/data_batch_2", + "sha256": "766b2cef9fbc745cf056b3152224f7cf77163b330ea9a15f9392beb8b89bc5a8" + }, + { + "bytes": 31035999, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/data_batch_3", + "sha256": "0f00d98ebfb30b3ec0ad19f9756dc2630b89003e10525f5e148445e82aa6a1f9" + }, + { + "bytes": 31035696, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/data_batch_4", + "sha256": "3f7bb240661948b8f4d53e36ec720d8306f5668bd0071dcb4e6c947f78e9682b" + }, + { + "bytes": 31035623, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/data_batch_5", + "sha256": "d91802434d8376bbaeeadf58a737e3a1b12ac839077e931237e0dcd43adcb154" + }, + { + "bytes": 31035526, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/test_batch", + "sha256": "f53d8d457504f7cff4ea9e021afcf0e0ad8e24a91f3fc42091b8adef61157831" + } + ], + "dataset": "cifar10", + "evaluation_split": "validation", + "input_layout": "NCHW", + "input_shape": [ + 3, + 32, + 32 + ], + "loader_seed": 0, + "normalization_mean": [ + 0.49140000343322754, + 0.4821999967098236, + 0.4465000033378601 + ], + "normalization_std": [ + 0.24699999392032623, + 0.2434999942779541, + 0.26159998774528503 + ], + "split_from_training_only": true, + "split_seed": 2027, + "test_examples": 10000, + "train_examples": 10000, + "training_augmentation": "random_crop_32_padding_4_zero_then_horizontal_flip_p0.5", + "validation_class_counts": { + "0": 500, + "1": 500, + "2": 500, + "3": 500, + "4": 500, + "5": 500, + "6": 500, + "7": 500, + "8": 500, + "9": 500 + }, + "validation_examples": 5000, + "validation_index_sha256": "8328b206a97c420e49e54e3eca4abe3274c4756b084355784ea3fb8059e4515b" + }, + "timing": { + "apical_warmup_wall_s": 6.035719156265259, + "evaluation_wall_s": 0.36167001724243164, + "predictor_warmup_wall_s": 0.0, + "timing_excludes_data_loading_hashing_and_model_construction": true, + "total_timed_wall_s": 95.64743661880493, + "train_wall_s": 89.2470269203186 + }, + "work": { + "apical_macs_per_example": 1884160, + "causal_scalar_observations": 990, + "components": { + "apical_projection_macs": 400738222080, + "apical_regression_macs": 118114222080, + "bp_reverse_macs_estimate": 0, + "local_weight_correlation_macs": 8110208000000, + "ordinary_forward_macs": 8110208000000, + "perturbation_forward_macs": 5084127191040, + "warmup_clean_forward_macs": 514511595520 + }, + "definition": "multiply-accumulates in conv/linear maps; one local weight correlation equals one forward-weight MAC count; BP reverse is estimated as one weight-gradient plus one activation-gradient convolution per forward convolution; elementwise nonlinearities and optimizer arithmetic excluded", + "forward_macs_per_example": 40551040, + "logical_batch_loss_queries": 990, + "per_example_cross_entropy_terms": 125376, + "total_clean_forward_examples": 212688, + "total_forward_equivalent_examples": 338064, + "total_macs_estimate": 22337907230720 + } +} diff --git a/results/oral_a_short/sdil_spatial_template_lr0.03.json b/results/oral_a_short/sdil_spatial_template_lr0.03.json new file mode 100644 index 0000000..53ce75a --- /dev/null +++ b/results/oral_a_short/sdil_spatial_template_lr0.03.json @@ -0,0 +1,639 @@ +{ + "apical_warmup": { + "last": { + "calibration_mse": 518.2958984375, + "prediction_target_cosine": 0.0001586056141805217, + "target_power": 518.2958984375 + }, + "steps": 100 + }, + "architecture": { + "adaptive_apical_parameters": 2260992, + "base_width": 16, + "blocks_per_stage": 3, + "bn_eps": 1e-05, + "bn_momentum": 0.1, + "depth": 20, + "family": "CIFAR 6n+2 ResNet, option-A shortcuts", + "fixed_traffic_coefficients": 188416, + "forward_parameters": 269722, + "hidden_shapes": [ + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 32, + 16, + 16 + ], + [ + 32, + 16, + 16 + ], + [ + 32, + 16, + 16 + ], + [ + 32, + 16, + 16 + ], + [ + 32, + 16, + 16 + ], + [ + 32, + 16, + 16 + ], + [ + 64, + 8, + 8 + ], + [ + 64, + 8, + 8 + ], + [ + 64, + 8, + 8 + ], + [ + 64, + 8, + 8 + ], + [ + 64, + 8, + 8 + ], + [ + 64, + 8, + 8 + ] + ], + "normalization": "batchnorm", + "predictor_parameters": 376832, + "residual_scale": 1.0, + "vectorizer_mode": "spatial_template", + "vectorizer_parameters": 1884160 + }, + "args": { + "a_scale": 0.0, + "a_warmup_steps": 100, + "alignment_probe": 32, + "apical_seed": null, + "augment_train": 1, + "batch_size": 128, + "bn_eps": 1e-05, + "bn_momentum": 0.1, + "data_dir": "/home/yurenh2/sdrn/data", + "depth": 20, + "device": "cuda:0", + "epochs": 20, + "eta_A": 0.001, + "eta_P": 0.01, + "eval_every": 0, + "eval_split": "validation", + "learn_P": 0, + "loader_seed": 0, + "lr": 0.03, + "lr_gamma": 0.1, + "lr_milestones": "100,150", + "lr_schedule": "cosine", + "max_steps": 0, + "mode": "sdil", + "momentum": 0.9, + "normalization": "batchnorm", + "nuisance_scale": 0.0, + "out": "results/oral_a_short/sdil_spatial_template_lr0.03.json", + "output_lr": 0.1, + "pert_directions": 1, + "pert_every": 4, + "pert_sigma": 0.01, + "perturb_seed": 1000, + "predictor_warmup_steps": 0, + "residual_scale": null, + "seed": 0, + "split_seed": 2027, + "train_limit": 10000, + "use_residual": 1, + "val_examples": 5000, + "vectorizer_mode": "spatial_template", + "warmup_epochs": 0, + "weight_decay": 0.0001, + "weight_scale": 1.0, + "width": 16 + }, + "counters": { + "apical_warmup_examples": 12688, + "calibration_event_examples": 62688, + "causal_scalar_observations": 990, + "logical_batch_loss_queries": 990, + "ordinary_examples": 200000, + "per_example_loss_terms": 125376, + "perturbation_events": 495, + "perturbation_forward_examples": 125376, + "predictor_warmup_examples": 0 + }, + "diagnostics": { + "early_third_mean": 0.0016261294658761472, + "innovation_negative_gradient_cosine": [ + 0.0026925508864223957, + 0.002747426275163889, + 0.0011170203797519207, + 2.4994893465191126e-05, + 0.002543856156989932, + 0.0006309282034635544, + 0.0015769501915201545, + -0.0002410386805422604, + 0.002343986416235566, + 0.0025179297663271427, + 0.0012802392011508346, + 0.0014369012787938118, + 0.001235027564689517, + 0.01105789840221405, + 0.01401814166456461, + 0.020832587033510208, + 0.011961638927459717, + 0.017622990533709526, + 0.012485330924391747 + ], + "normalization_state": "training_batch_stats_without_running_update", + "raw_negative_gradient_cosine": [ + 0.0026925508864223957, + 0.002747426275163889, + 0.0011170203797519207, + 2.4994893465191126e-05, + 0.002543856156989932, + 0.0006309282034635544, + 0.0015769501915201545, + -0.0002410386805422604, + 0.002343986416235566, + 0.0025179297663271427, + 0.0012802392011508346, + 0.0014369012787938118, + 0.001235027564689517, + 0.01105789840221405, + 0.01401814166456461, + 0.020832587033510208, + 0.011961638927459717, + 0.017622990533709526, + 0.012485330924391747 + ], + "teaching_negative_gradient_cosine": [ + 0.0026925508864223957, + 0.002747426275163889, + 0.0011170203797519207, + 2.4994893465191126e-05, + 0.002543856156989932, + 0.0006309282034635544, + 0.0015769501915201545, + -0.0002410386805422604, + 0.002343986416235566, + 0.0025179297663271427, + 0.0012802392011508346, + 0.0014369012787938118, + 0.001235027564689517, + 0.01105789840221405, + 0.01401814166456461, + 0.020832587033510208, + 0.011961638927459717, + 0.017622990533709526, + 0.012485330924391747 + ], + "wall_s": 0.10700726509094238 + }, + "epochs": [ + { + "calibration": { + "calibration_mse": 225.0854659847186, + "prediction_target_cosine": -1.1367990201770532e-05, + "target_power": 225.08545032376824 + }, + "epoch": 1, + "lr": 0.03, + "output_lr": 0.1, + "step": 79, + "train_examples": 10000, + "train_loss": 2.779825601196289 + }, + { + "calibration": { + "calibration_mse": 224.05881381941876, + "prediction_target_cosine": 7.364900904852228e-05, + "target_power": 224.05880134805392 + }, + "epoch": 2, + "lr": 0.029815325108927065, + "output_lr": 0.0993844170297569, + "step": 158, + "train_examples": 10000, + "train_loss": 2.464528601074219 + }, + { + "calibration": { + "calibration_mse": 189.00379390978944, + "prediction_target_cosine": -4.6249213236814e-05, + "target_power": 189.00376264984192 + }, + "epoch": 3, + "lr": 0.029265847744427302, + "output_lr": 0.09755282581475769, + "step": 237, + "train_examples": 10000, + "train_loss": 2.608198607635498 + }, + { + "calibration": { + "calibration_mse": 252.0002302787942, + "prediction_target_cosine": 2.8914304700664197e-05, + "target_power": 252.00018578426793 + }, + "epoch": 4, + "lr": 0.028365097862825515, + "output_lr": 0.09455032620941839, + "step": 316, + "train_examples": 10000, + "train_loss": 2.8782430679321287 + }, + { + "calibration": { + "calibration_mse": 162.07468072448088, + "prediction_target_cosine": -9.906694745418392e-05, + "target_power": 162.07462500592936 + }, + "epoch": 5, + "lr": 0.02713525491562421, + "output_lr": 0.09045084971874738, + "step": 395, + "train_examples": 10000, + "train_loss": 3.600461712646484 + }, + { + "calibration": { + "calibration_mse": 339.4159528536642, + "prediction_target_cosine": -3.538781021778639e-05, + "target_power": 339.415888260888 + }, + "epoch": 6, + "lr": 0.02560660171779821, + "output_lr": 0.08535533905932738, + "step": 474, + "train_examples": 10000, + "train_loss": 4.972540879821778 + }, + { + "calibration": { + "calibration_mse": 320.14482944957587, + "prediction_target_cosine": -5.400665091043201e-05, + "target_power": 320.14475205922895 + }, + "epoch": 7, + "lr": 0.023816778784387097, + "output_lr": 0.07938926261462366, + "step": 553, + "train_examples": 10000, + "train_loss": 6.406736657714844 + }, + { + "calibration": { + "calibration_mse": 239.223344233768, + "prediction_target_cosine": 1.9879424781897673e-05, + "target_power": 239.2232916822968 + }, + "epoch": 8, + "lr": 0.0218098574960932, + "output_lr": 0.07269952498697733, + "step": 632, + "train_examples": 10000, + "train_loss": 6.1722028503417965 + }, + { + "calibration": { + "calibration_mse": 463.84863981811895, + "prediction_target_cosine": -1.875769559546624e-05, + "target_power": 463.8485612288765 + }, + "epoch": 9, + "lr": 0.01963525491562421, + "output_lr": 0.06545084971874737, + "step": 711, + "train_examples": 10000, + "train_loss": 6.8595925750732425 + }, + { + "calibration": { + "calibration_mse": 453.42677381142323, + "prediction_target_cosine": -3.32086027929901e-05, + "target_power": 453.42670377959377 + }, + "epoch": 10, + "lr": 0.017346516975603462, + "output_lr": 0.05782172325201155, + "step": 790, + "train_examples": 10000, + "train_loss": 5.333097433471679 + }, + { + "calibration": { + "calibration_mse": 309.16671639810437, + "prediction_target_cosine": -5.810984922454561e-05, + "target_power": 309.1665808511818 + }, + "epoch": 11, + "lr": 0.015, + "output_lr": 0.05, + "step": 869, + "train_examples": 10000, + "train_loss": 5.10923678855896 + }, + { + "calibration": { + "calibration_mse": 159.81345182357174, + "prediction_target_cosine": 5.906782660362616e-06, + "target_power": 159.8133521187769 + }, + "epoch": 12, + "lr": 0.012653483024396533, + "output_lr": 0.04217827674798845, + "step": 948, + "train_examples": 10000, + "train_loss": 3.1977948154449463 + }, + { + "calibration": { + "calibration_mse": 228.8384463015944, + "prediction_target_cosine": 4.724701258411892e-05, + "target_power": 228.83836178986923 + }, + "epoch": 13, + "lr": 0.01036474508437579, + "output_lr": 0.03454915028125263, + "step": 1027, + "train_examples": 10000, + "train_loss": 2.952687503814697 + }, + { + "calibration": { + "calibration_mse": 138.64461772111446, + "prediction_target_cosine": -4.9393437286832497e-05, + "target_power": 138.6445097280585 + }, + "epoch": 14, + "lr": 0.0081901425039068, + "output_lr": 0.027300475013022664, + "step": 1106, + "train_examples": 10000, + "train_loss": 2.335374161529541 + }, + { + "calibration": { + "calibration_mse": 162.95293996288723, + "prediction_target_cosine": -0.00010373396979351931, + "target_power": 162.9528389635293 + }, + "epoch": 15, + "lr": 0.006183221215612904, + "output_lr": 0.02061073738537635, + "step": 1185, + "train_examples": 10000, + "train_loss": 2.05778305015564 + }, + { + "calibration": { + "calibration_mse": 296.1335524799349, + "prediction_target_cosine": 5.441280496664627e-05, + "target_power": 296.13348323311357 + }, + "epoch": 16, + "lr": 0.004393398282201788, + "output_lr": 0.014644660940672627, + "step": 1264, + "train_examples": 10000, + "train_loss": 2.0930088500976565 + }, + { + "calibration": { + "calibration_mse": 151.49618704825153, + "prediction_target_cosine": -6.845286494395057e-05, + "target_power": 151.4960966487294 + }, + "epoch": 17, + "lr": 0.00286474508437579, + "output_lr": 0.009549150281252633, + "step": 1343, + "train_examples": 10000, + "train_loss": 1.9203613328933715 + }, + { + "calibration": { + "calibration_mse": 172.30423948797198, + "prediction_target_cosine": -4.908389648638358e-05, + "target_power": 172.3041283291319 + }, + "epoch": 18, + "lr": 0.001634902137174483, + "output_lr": 0.005449673790581611, + "step": 1422, + "train_examples": 10000, + "train_loss": 1.8738044273376464 + }, + { + "calibration": { + "calibration_mse": 204.4578604910361, + "prediction_target_cosine": -6.160888634974665e-05, + "target_power": 204.45776410757202 + }, + "epoch": 19, + "lr": 0.000734152255572697, + "output_lr": 0.0024471741852423235, + "step": 1501, + "train_examples": 10000, + "train_loss": 1.8021988306045533 + }, + { + "calibration": { + "calibration_mse": 258.4718296773374, + "prediction_target_cosine": 5.921584174947326e-05, + "target_power": 258.47175269814323 + }, + "epoch": 20, + "lr": 0.0001846748910729351, + "output_lr": 0.0006155829702431171, + "step": 1580, + "train_examples": 10000, + "train_loss": 1.787876174545288 + } + ], + "evaluation_protocol": { + "test_evaluations": 0, + "test_used_for_selection": false, + "validation_evaluations": 1 + }, + "final": { + "accuracy": 0.3596, + "epoch": 20, + "evaluation_split": "validation", + "finite": true, + "loss": 1.777641943359375, + "step": 1580 + }, + "hardware": { + "cuda_device_name": "NVIDIA GeForce GTX 1080", + "cuda_visible_devices": "5", + "device": "cuda:0", + "device_total_memory_bytes": 8507949056, + "peak_memory_allocated_bytes": 1740965888, + "peak_memory_reserved_bytes": 2866806784, + "torch_version": "2.3.1+cu118" + }, + "protocol_family": "oral_a_cifar_local_resnet_development", + "provenance": { + "git_commit": "a1d060e6d96d4ab973d16cd75149dd61899c7c95", + "git_tracked_dirty": false + }, + "schema_version": 1, + "split": { + "cifar_source_files": [ + { + "bytes": 31035704, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/data_batch_1", + "sha256": "54636561a3ce25bd3e19253c6b0d8538147b0ae398331ac4a2d86c6d987368cd" + }, + { + "bytes": 31035320, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/data_batch_2", + "sha256": "766b2cef9fbc745cf056b3152224f7cf77163b330ea9a15f9392beb8b89bc5a8" + }, + { + "bytes": 31035999, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/data_batch_3", + "sha256": "0f00d98ebfb30b3ec0ad19f9756dc2630b89003e10525f5e148445e82aa6a1f9" + }, + { + "bytes": 31035696, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/data_batch_4", + "sha256": "3f7bb240661948b8f4d53e36ec720d8306f5668bd0071dcb4e6c947f78e9682b" + }, + { + "bytes": 31035623, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/data_batch_5", + "sha256": "d91802434d8376bbaeeadf58a737e3a1b12ac839077e931237e0dcd43adcb154" + }, + { + "bytes": 31035526, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/test_batch", + "sha256": "f53d8d457504f7cff4ea9e021afcf0e0ad8e24a91f3fc42091b8adef61157831" + } + ], + "dataset": "cifar10", + "evaluation_split": "validation", + "input_layout": "NCHW", + "input_shape": [ + 3, + 32, + 32 + ], + "loader_seed": 0, + "normalization_mean": [ + 0.49140000343322754, + 0.4821999967098236, + 0.4465000033378601 + ], + "normalization_std": [ + 0.24699999392032623, + 0.2434999942779541, + 0.26159998774528503 + ], + "split_from_training_only": true, + "split_seed": 2027, + "test_examples": 10000, + "train_examples": 10000, + "training_augmentation": "random_crop_32_padding_4_zero_then_horizontal_flip_p0.5", + "validation_class_counts": { + "0": 500, + "1": 500, + "2": 500, + "3": 500, + "4": 500, + "5": 500, + "6": 500, + "7": 500, + "8": 500, + "9": 500 + }, + "validation_examples": 5000, + "validation_index_sha256": "8328b206a97c420e49e54e3eca4abe3274c4756b084355784ea3fb8059e4515b" + }, + "timing": { + "apical_warmup_wall_s": 5.937381267547607, + "evaluation_wall_s": 0.39031529426574707, + "predictor_warmup_wall_s": 0.0, + "timing_excludes_data_loading_hashing_and_model_construction": true, + "total_timed_wall_s": 95.62685227394104, + "train_wall_s": 89.29021406173706 + }, + "work": { + "apical_macs_per_example": 1884160, + "causal_scalar_observations": 990, + "components": { + "apical_projection_macs": 400738222080, + "apical_regression_macs": 118114222080, + "bp_reverse_macs_estimate": 0, + "local_weight_correlation_macs": 8110208000000, + "ordinary_forward_macs": 8110208000000, + "perturbation_forward_macs": 5084127191040, + "warmup_clean_forward_macs": 514511595520 + }, + "definition": "multiply-accumulates in conv/linear maps; one local weight correlation equals one forward-weight MAC count; BP reverse is estimated as one weight-gradient plus one activation-gradient convolution per forward convolution; elementwise nonlinearities and optimizer arithmetic excluded", + "forward_macs_per_example": 40551040, + "logical_batch_loss_queries": 990, + "per_example_cross_entropy_terms": 125376, + "total_clean_forward_examples": 212688, + "total_forward_equivalent_examples": 338064, + "total_macs_estimate": 22337907230720 + } +} diff --git a/results/oral_a_short/sdil_spatial_template_lr0.1.json b/results/oral_a_short/sdil_spatial_template_lr0.1.json new file mode 100644 index 0000000..a5a83ec --- /dev/null +++ b/results/oral_a_short/sdil_spatial_template_lr0.1.json @@ -0,0 +1,639 @@ +{ + "apical_warmup": { + "last": { + "calibration_mse": 518.2958984375, + "prediction_target_cosine": 0.0001586056141805217, + "target_power": 518.2958984375 + }, + "steps": 100 + }, + "architecture": { + "adaptive_apical_parameters": 2260992, + "base_width": 16, + "blocks_per_stage": 3, + "bn_eps": 1e-05, + "bn_momentum": 0.1, + "depth": 20, + "family": "CIFAR 6n+2 ResNet, option-A shortcuts", + "fixed_traffic_coefficients": 188416, + "forward_parameters": 269722, + "hidden_shapes": [ + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 16, + 32, + 32 + ], + [ + 32, + 16, + 16 + ], + [ + 32, + 16, + 16 + ], + [ + 32, + 16, + 16 + ], + [ + 32, + 16, + 16 + ], + [ + 32, + 16, + 16 + ], + [ + 32, + 16, + 16 + ], + [ + 64, + 8, + 8 + ], + [ + 64, + 8, + 8 + ], + [ + 64, + 8, + 8 + ], + [ + 64, + 8, + 8 + ], + [ + 64, + 8, + 8 + ], + [ + 64, + 8, + 8 + ] + ], + "normalization": "batchnorm", + "predictor_parameters": 376832, + "residual_scale": 1.0, + "vectorizer_mode": "spatial_template", + "vectorizer_parameters": 1884160 + }, + "args": { + "a_scale": 0.0, + "a_warmup_steps": 100, + "alignment_probe": 32, + "apical_seed": null, + "augment_train": 1, + "batch_size": 128, + "bn_eps": 1e-05, + "bn_momentum": 0.1, + "data_dir": "/home/yurenh2/sdrn/data", + "depth": 20, + "device": "cuda:0", + "epochs": 20, + "eta_A": 0.001, + "eta_P": 0.01, + "eval_every": 0, + "eval_split": "validation", + "learn_P": 0, + "loader_seed": 0, + "lr": 0.1, + "lr_gamma": 0.1, + "lr_milestones": "100,150", + "lr_schedule": "cosine", + "max_steps": 0, + "mode": "sdil", + "momentum": 0.9, + "normalization": "batchnorm", + "nuisance_scale": 0.0, + "out": "results/oral_a_short/sdil_spatial_template_lr0.1.json", + "output_lr": 0.1, + "pert_directions": 1, + "pert_every": 4, + "pert_sigma": 0.01, + "perturb_seed": 1000, + "predictor_warmup_steps": 0, + "residual_scale": null, + "seed": 0, + "split_seed": 2027, + "train_limit": 10000, + "use_residual": 1, + "val_examples": 5000, + "vectorizer_mode": "spatial_template", + "warmup_epochs": 0, + "weight_decay": 0.0001, + "weight_scale": 1.0, + "width": 16 + }, + "counters": { + "apical_warmup_examples": 12688, + "calibration_event_examples": 62688, + "causal_scalar_observations": 990, + "logical_batch_loss_queries": 990, + "ordinary_examples": 200000, + "per_example_loss_terms": 125376, + "perturbation_events": 495, + "perturbation_forward_examples": 125376, + "predictor_warmup_examples": 0 + }, + "diagnostics": { + "early_third_mean": 0.0011366111769651373, + "innovation_negative_gradient_cosine": [ + -0.0005086585879325867, + 0.002177828922867775, + 0.0010308092460036278, + 0.0010501776123419404, + 0.0036033838987350464, + -0.0005338740302249789, + -0.0005385851254686713, + 0.0055239940993487835, + 0.006746778730303049, + 0.002382537815719843, + -0.0019065908854827285, + 0.0028815269470214844, + 0.003817233955487609, + 0.004685048013925552, + 0.007657486014068127, + 0.016477705910801888, + 0.012200873345136642, + 0.011948207393288612, + 0.004797697998583317 + ], + "normalization_state": "training_batch_stats_without_running_update", + "raw_negative_gradient_cosine": [ + -0.0005086585879325867, + 0.002177828922867775, + 0.0010308092460036278, + 0.0010501776123419404, + 0.0036033838987350464, + -0.0005338740302249789, + -0.0005385851254686713, + 0.0055239940993487835, + 0.006746778730303049, + 0.002382537815719843, + -0.0019065908854827285, + 0.0028815269470214844, + 0.003817233955487609, + 0.004685048013925552, + 0.007657486014068127, + 0.016477705910801888, + 0.012200873345136642, + 0.011948207393288612, + 0.004797697998583317 + ], + "teaching_negative_gradient_cosine": [ + -0.0005086585879325867, + 0.002177828922867775, + 0.0010308092460036278, + 0.0010501776123419404, + 0.0036033838987350464, + -0.0005338740302249789, + -0.0005385851254686713, + 0.0055239940993487835, + 0.006746778730303049, + 0.002382537815719843, + -0.0019065908854827285, + 0.0028815269470214844, + 0.003817233955487609, + 0.004685048013925552, + 0.007657486014068127, + 0.016477705910801888, + 0.012200873345136642, + 0.011948207393288612, + 0.004797697998583317 + ], + "wall_s": 0.10133647918701172 + }, + "epochs": [ + { + "calibration": { + "calibration_mse": 218.00981008301125, + "prediction_target_cosine": 7.372765407421974e-07, + "target_power": 218.00976970393376 + }, + "epoch": 1, + "lr": 0.1, + "output_lr": 0.1, + "step": 79, + "train_examples": 10000, + "train_loss": 2.7016247840881347 + }, + { + "calibration": { + "calibration_mse": 99.16834810680669, + "prediction_target_cosine": -2.8498735313447782e-05, + "target_power": 99.16831356921922 + }, + "epoch": 2, + "lr": 0.0993844170297569, + "output_lr": 0.0993844170297569, + "step": 158, + "train_examples": 10000, + "train_loss": 3.1584605926513674 + }, + { + "calibration": { + "calibration_mse": 298.79512195511035, + "prediction_target_cosine": 2.909768881279403e-05, + "target_power": 298.79509454188144 + }, + "epoch": 3, + "lr": 0.09755282581475769, + "output_lr": 0.09755282581475769, + "step": 237, + "train_examples": 10000, + "train_loss": 8.829980853271485 + }, + { + "calibration": { + "calibration_mse": 495.4038626001955, + "prediction_target_cosine": 6.753209127928497e-05, + "target_power": 495.40380857997405 + }, + "epoch": 4, + "lr": 0.09455032620941839, + "output_lr": 0.09455032620941839, + "step": 316, + "train_examples": 10000, + "train_loss": 14.866311837768555 + }, + { + "calibration": { + "calibration_mse": 644.076006343183, + "prediction_target_cosine": 5.199910531874288e-05, + "target_power": 644.0759610264197 + }, + "epoch": 5, + "lr": 0.09045084971874738, + "output_lr": 0.09045084971874738, + "step": 395, + "train_examples": 10000, + "train_loss": 19.877845971679687 + }, + { + "calibration": { + "calibration_mse": 769.4698152616618, + "prediction_target_cosine": 1.593025226261596e-05, + "target_power": 769.4697039468132 + }, + "epoch": 6, + "lr": 0.08535533905932738, + "output_lr": 0.08535533905932738, + "step": 474, + "train_examples": 10000, + "train_loss": 22.049921173095704 + }, + { + "calibration": { + "calibration_mse": 1704.3785989527266, + "prediction_target_cosine": -2.4309997884679826e-05, + "target_power": 1704.3783198974947 + }, + "epoch": 7, + "lr": 0.07938926261462366, + "output_lr": 0.07938926261462366, + "step": 553, + "train_examples": 10000, + "train_loss": 34.75900443725586 + }, + { + "calibration": { + "calibration_mse": 2635.6820743733197, + "prediction_target_cosine": 5.142736440269856e-05, + "target_power": 2635.6819745528614 + }, + "epoch": 8, + "lr": 0.07269952498697733, + "output_lr": 0.07269952498697733, + "step": 632, + "train_examples": 10000, + "train_loss": 28.494638110351563 + }, + { + "calibration": { + "calibration_mse": 2921.008621702878, + "prediction_target_cosine": -2.392465289065802e-06, + "target_power": 2921.0082750818005 + }, + "epoch": 9, + "lr": 0.06545084971874737, + "output_lr": 0.06545084971874737, + "step": 711, + "train_examples": 10000, + "train_loss": 41.31702725830078 + }, + { + "calibration": { + "calibration_mse": 2678.3753509645876, + "prediction_target_cosine": -3.18863488196659e-05, + "target_power": 2678.3748598513394 + }, + "epoch": 10, + "lr": 0.05782172325201155, + "output_lr": 0.05782172325201155, + "step": 790, + "train_examples": 10000, + "train_loss": 33.205736236572264 + }, + { + "calibration": { + "calibration_mse": 1499.2314302945267, + "prediction_target_cosine": -1.2890709474344284e-06, + "target_power": 1499.2308740014616 + }, + "epoch": 11, + "lr": 0.05, + "output_lr": 0.05, + "step": 869, + "train_examples": 10000, + "train_loss": 31.877177111816405 + }, + { + "calibration": { + "calibration_mse": 1585.0379899110073, + "prediction_target_cosine": 1.1932150178731406e-05, + "target_power": 1585.0374189503405 + }, + "epoch": 12, + "lr": 0.04217827674798845, + "output_lr": 0.04217827674798845, + "step": 948, + "train_examples": 10000, + "train_loss": 33.527968142700196 + }, + { + "calibration": { + "calibration_mse": 1974.6094332775508, + "prediction_target_cosine": -2.8002023685383723e-05, + "target_power": 1974.6085822364555 + }, + "epoch": 13, + "lr": 0.03454915028125263, + "output_lr": 0.03454915028125263, + "step": 1027, + "train_examples": 10000, + "train_loss": 27.32040124511719 + }, + { + "calibration": { + "calibration_mse": 2883.157492811784, + "prediction_target_cosine": -3.153829896486009e-05, + "target_power": 2883.1566430796747 + }, + "epoch": 14, + "lr": 0.027300475013022664, + "output_lr": 0.027300475013022664, + "step": 1106, + "train_examples": 10000, + "train_loss": 26.911080462646485 + }, + { + "calibration": { + "calibration_mse": 1497.195044086809, + "prediction_target_cosine": -5.236116223205009e-05, + "target_power": 1497.194315920187 + }, + "epoch": 15, + "lr": 0.02061073738537635, + "output_lr": 0.02061073738537635, + "step": 1185, + "train_examples": 10000, + "train_loss": 16.22249378967285 + }, + { + "calibration": { + "calibration_mse": 1796.1156036770597, + "prediction_target_cosine": -4.133475642912655e-05, + "target_power": 1796.114599726702 + }, + "epoch": 16, + "lr": 0.014644660940672627, + "output_lr": 0.014644660940672627, + "step": 1264, + "train_examples": 10000, + "train_loss": 13.36528768310547 + }, + { + "calibration": { + "calibration_mse": 1416.6319843007818, + "prediction_target_cosine": 6.540755980912454e-05, + "target_power": 1416.6312157147604 + }, + "epoch": 17, + "lr": 0.009549150281252633, + "output_lr": 0.009549150281252633, + "step": 1343, + "train_examples": 10000, + "train_loss": 7.271144700622559 + }, + { + "calibration": { + "calibration_mse": 1296.9401957724403, + "prediction_target_cosine": 3.232485078663147e-05, + "target_power": 1296.939295607028 + }, + "epoch": 18, + "lr": 0.005449673790581611, + "output_lr": 0.005449673790581611, + "step": 1422, + "train_examples": 10000, + "train_loss": 6.070062582397461 + }, + { + "calibration": { + "calibration_mse": 1811.3753914065335, + "prediction_target_cosine": -6.324038492581172e-05, + "target_power": 1811.3743516728937 + }, + "epoch": 19, + "lr": 0.0024471741852423235, + "output_lr": 0.0024471741852423235, + "step": 1501, + "train_examples": 10000, + "train_loss": 5.354951040649414 + }, + { + "calibration": { + "calibration_mse": 1090.3963583066752, + "prediction_target_cosine": 8.504621255287847e-05, + "target_power": 1090.3955357767625 + }, + "epoch": 20, + "lr": 0.0006155829702431171, + "output_lr": 0.0006155829702431171, + "step": 1580, + "train_examples": 10000, + "train_loss": 4.983896137237549 + } + ], + "evaluation_protocol": { + "test_evaluations": 0, + "test_used_for_selection": false, + "validation_evaluations": 1 + }, + "final": { + "accuracy": 0.2474, + "epoch": 20, + "evaluation_split": "validation", + "finite": true, + "loss": 5.14968251953125, + "step": 1580 + }, + "hardware": { + "cuda_device_name": "NVIDIA GeForce GTX 1080", + "cuda_visible_devices": "5", + "device": "cuda:0", + "device_total_memory_bytes": 8507949056, + "peak_memory_allocated_bytes": 1740965888, + "peak_memory_reserved_bytes": 2866806784, + "torch_version": "2.3.1+cu118" + }, + "protocol_family": "oral_a_cifar_local_resnet_development", + "provenance": { + "git_commit": "a1d060e6d96d4ab973d16cd75149dd61899c7c95", + "git_tracked_dirty": false + }, + "schema_version": 1, + "split": { + "cifar_source_files": [ + { + "bytes": 31035704, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/data_batch_1", + "sha256": "54636561a3ce25bd3e19253c6b0d8538147b0ae398331ac4a2d86c6d987368cd" + }, + { + "bytes": 31035320, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/data_batch_2", + "sha256": "766b2cef9fbc745cf056b3152224f7cf77163b330ea9a15f9392beb8b89bc5a8" + }, + { + "bytes": 31035999, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/data_batch_3", + "sha256": "0f00d98ebfb30b3ec0ad19f9756dc2630b89003e10525f5e148445e82aa6a1f9" + }, + { + "bytes": 31035696, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/data_batch_4", + "sha256": "3f7bb240661948b8f4d53e36ec720d8306f5668bd0071dcb4e6c947f78e9682b" + }, + { + "bytes": 31035623, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/data_batch_5", + "sha256": "d91802434d8376bbaeeadf58a737e3a1b12ac839077e931237e0dcd43adcb154" + }, + { + "bytes": 31035526, + "path": "/home/yurenh2/sdrn/data/cifar-10-batches-py/test_batch", + "sha256": "f53d8d457504f7cff4ea9e021afcf0e0ad8e24a91f3fc42091b8adef61157831" + } + ], + "dataset": "cifar10", + "evaluation_split": "validation", + "input_layout": "NCHW", + "input_shape": [ + 3, + 32, + 32 + ], + "loader_seed": 0, + "normalization_mean": [ + 0.49140000343322754, + 0.4821999967098236, + 0.4465000033378601 + ], + "normalization_std": [ + 0.24699999392032623, + 0.2434999942779541, + 0.26159998774528503 + ], + "split_from_training_only": true, + "split_seed": 2027, + "test_examples": 10000, + "train_examples": 10000, + "training_augmentation": "random_crop_32_padding_4_zero_then_horizontal_flip_p0.5", + "validation_class_counts": { + "0": 500, + "1": 500, + "2": 500, + "3": 500, + "4": 500, + "5": 500, + "6": 500, + "7": 500, + "8": 500, + "9": 500 + }, + "validation_examples": 5000, + "validation_index_sha256": "8328b206a97c420e49e54e3eca4abe3274c4756b084355784ea3fb8059e4515b" + }, + "timing": { + "apical_warmup_wall_s": 6.064634799957275, + "evaluation_wall_s": 0.3710658550262451, + "predictor_warmup_wall_s": 0.0, + "timing_excludes_data_loading_hashing_and_model_construction": true, + "total_timed_wall_s": 96.05532002449036, + "train_wall_s": 89.61682558059692 + }, + "work": { + "apical_macs_per_example": 1884160, + "causal_scalar_observations": 990, + "components": { + "apical_projection_macs": 400738222080, + "apical_regression_macs": 118114222080, + "bp_reverse_macs_estimate": 0, + "local_weight_correlation_macs": 8110208000000, + "ordinary_forward_macs": 8110208000000, + "perturbation_forward_macs": 5084127191040, + "warmup_clean_forward_macs": 514511595520 + }, + "definition": "multiply-accumulates in conv/linear maps; one local weight correlation equals one forward-weight MAC count; BP reverse is estimated as one weight-gradient plus one activation-gradient convolution per forward convolution; elementwise nonlinearities and optimizer arithmetic excluded", + "forward_macs_per_example": 40551040, + "logical_batch_loss_queries": 990, + "per_example_cross_entropy_terms": 125376, + "total_clean_forward_examples": 212688, + "total_forward_equivalent_examples": 338064, + "total_macs_estimate": 22337907230720 + } +} diff --git a/results/oral_a_short_selection.json b/results/oral_a_short_selection.json new file mode 100644 index 0000000..11e4568 --- /dev/null +++ b/results/oral_a_short_selection.json @@ -0,0 +1,176 @@ +{ + "confirmation_test_seeds_touched": false, + "protocol": "oral_a_A2b_v1", + "rows": [ + { + "a_scale": 1.0, + "accuracy": 0.7494, + "eta_A": 0.01, + "finite": true, + "loss": 0.7565972534179688, + "lr": 0.1, + "mode": "bp", + "path": "results/oral_a_short/bp_lr0.1.json", + "source_commit": "a1d060e6d96d4ab973d16cd75149dd61899c7c95", + "total_macs": 24330624000000, + "vectorizer_mode": "spatial_template" + }, + { + "a_scale": 1.0, + "accuracy": 0.314, + "eta_A": 0.01, + "finite": true, + "loss": 1.8915036865234376, + "lr": 0.01, + "mode": "dfa", + "path": "results/oral_a_short/dfa_lr0.01.json", + "source_commit": "a1d060e6d96d4ab973d16cd75149dd61899c7c95", + "total_macs": 16597248000000, + "vectorizer_mode": "spatial_template" + }, + { + "a_scale": 1.0, + "accuracy": 0.3716, + "eta_A": 0.01, + "finite": true, + "loss": 1.728732666015625, + "lr": 0.03, + "mode": "dfa", + "path": "results/oral_a_short/dfa_lr0.03.json", + "source_commit": "a1d060e6d96d4ab973d16cd75149dd61899c7c95", + "total_macs": 16597248000000, + "vectorizer_mode": "spatial_template" + }, + { + "a_scale": 1.0, + "accuracy": 0.3642, + "eta_A": 0.01, + "finite": true, + "loss": 1.7964451171875, + "lr": 0.1, + "mode": "dfa", + "path": "results/oral_a_short/dfa_lr0.1.json", + "source_commit": "a1d060e6d96d4ab973d16cd75149dd61899c7c95", + "total_macs": 16597248000000, + "vectorizer_mode": "spatial_template" + }, + { + "a_scale": 1.0, + "accuracy": 0.4054, + "eta_A": 0.001, + "finite": true, + "loss": 1.6269997802734375, + "lr": 0.01, + "mode": "sdil", + "path": "results/oral_a_short/sdil_channel_gated_lr0.01.json", + "source_commit": "a1d060e6d96d4ab973d16cd75149dd61899c7c95", + "total_macs": 21874729204736, + "vectorizer_mode": "channel_gated" + }, + { + "a_scale": 1.0, + "accuracy": 0.4198, + "eta_A": 0.001, + "finite": true, + "loss": 1.60240986328125, + "lr": 0.03, + "mode": "sdil", + "path": "results/oral_a_short/sdil_channel_gated_lr0.03.json", + "source_commit": "a1d060e6d96d4ab973d16cd75149dd61899c7c95", + "total_macs": 21874729204736, + "vectorizer_mode": "channel_gated" + }, + { + "a_scale": 1.0, + "accuracy": 0.414, + "eta_A": 0.001, + "finite": true, + "loss": 1.597408447265625, + "lr": 0.1, + "mode": "sdil", + "path": "results/oral_a_short/sdil_channel_gated_lr0.1.json", + "source_commit": "a1d060e6d96d4ab973d16cd75149dd61899c7c95", + "total_macs": 21874729204736, + "vectorizer_mode": "channel_gated" + }, + { + "a_scale": 0.0, + "accuracy": 0.3572, + "eta_A": 0.001, + "finite": true, + "loss": 1.7704051513671875, + "lr": 0.01, + "mode": "sdil", + "path": "results/oral_a_short/sdil_spatial_template_lr0.01.json", + "source_commit": "a1d060e6d96d4ab973d16cd75149dd61899c7c95", + "total_macs": 22337907230720, + "vectorizer_mode": "spatial_template" + }, + { + "a_scale": 0.0, + "accuracy": 0.3596, + "eta_A": 0.001, + "finite": true, + "loss": 1.777641943359375, + "lr": 0.03, + "mode": "sdil", + "path": "results/oral_a_short/sdil_spatial_template_lr0.03.json", + "source_commit": "a1d060e6d96d4ab973d16cd75149dd61899c7c95", + "total_macs": 22337907230720, + "vectorizer_mode": "spatial_template" + }, + { + "a_scale": 0.0, + "accuracy": 0.2474, + "eta_A": 0.001, + "finite": true, + "loss": 5.14968251953125, + "lr": 0.1, + "mode": "sdil", + "path": "results/oral_a_short/sdil_spatial_template_lr0.1.json", + "source_commit": "a1d060e6d96d4ab973d16cd75149dd61899c7c95", + "total_macs": 22337907230720, + "vectorizer_mode": "spatial_template" + } + ], + "selected_bp": { + "a_scale": 1.0, + "accuracy": 0.7494, + "eta_A": 0.01, + "finite": true, + "loss": 0.7565972534179688, + "lr": 0.1, + "mode": "bp", + "path": "results/oral_a_short/bp_lr0.1.json", + "source_commit": "a1d060e6d96d4ab973d16cd75149dd61899c7c95", + "total_macs": 24330624000000, + "vectorizer_mode": "spatial_template" + }, + "selected_dfa": { + "a_scale": 1.0, + "accuracy": 0.3716, + "eta_A": 0.01, + "finite": true, + "loss": 1.728732666015625, + "lr": 0.03, + "mode": "dfa", + "path": "results/oral_a_short/dfa_lr0.03.json", + "source_commit": "a1d060e6d96d4ab973d16cd75149dd61899c7c95", + "total_macs": 16597248000000, + "vectorizer_mode": "spatial_template" + }, + "selected_sdil": { + "a_scale": 1.0, + "accuracy": 0.4198, + "eta_A": 0.001, + "finite": true, + "loss": 1.60240986328125, + "lr": 0.03, + "mode": "sdil", + "path": "results/oral_a_short/sdil_channel_gated_lr0.03.json", + "source_commit": "a1d060e6d96d4ab973d16cd75149dd61899c7c95", + "total_macs": 21874729204736, + "vectorizer_mode": "channel_gated" + }, + "status": "selected" +} |