diff options
Diffstat (limited to 'results')
| -rw-r--r-- | results/resnet_crossover/p1_selector.json | 441 | ||||
| -rw-r--r-- | results/transformer_crossover/p1_selector.json | 1158 |
2 files changed, 1599 insertions, 0 deletions
diff --git a/results/resnet_crossover/p1_selector.json b/results/resnet_crossover/p1_selector.json new file mode 100644 index 0000000..5a54299 --- /dev/null +++ b/results/resnet_crossover/p1_selector.json @@ -0,0 +1,441 @@ +{ + "gate": "pass", + "launch_lock": { + "allowed_physical_gpus": [ + 5, + 7 + ], + "num_jobs": 19, + "registry_sha256": "9ef005d067ea4c2048c48dc46ba44b4ebbbab82e9c7ed2791208930e5279ca81", + "source": { + "git_commit": "19f3966fc2d15b1ed18862a779d91ee16ed56bfc", + "protocol_path": "/scratch/yurenh2/sdil-runs/resnet-p1/RESNET_CROSSOVER.md", + "protocol_sha256": "048dbaf06866bdd3959a637ba827c06063205f6d1cc693d89aa8fc9c1d24975d", + "python": "/home/yurenh2/miniconda3/envs/ep_pascal/bin/python3", + "tracked_files": { + "RESNET_CROSSOVER.md": "048dbaf06866bdd3959a637ba827c06063205f6d1cc693d89aa8fc9c1d24975d", + "experiments/analyze_resnet_crossover_p1.py": "6ff485a3fecc7b814dedf55d04abdb770a6ce904cdeb9e06a7dddcf26f996d5b", + "experiments/conv_run.py": "9fa2daea597d63981a78248fafb154da39775ccfae57aac9889e78d83dc4eb44", + "experiments/resnet_crossover_grid.py": "669c84846bc3115816ae3c9036be8722f21577612dc9d56a689d3321e837ca95", + "experiments/resnet_crossover_native.py": "c5e54324b000c37c53d495f7d3eb0d4a85dcf15e83ea3aab125a097ce61eff87", + "sdil/conv.py": "09f87eef636aa5397b39a2a7a66909b8861640624507eabbdfaffe98577a5d2c", + "sdil/conv_crossover.py": "3c1373aa3555e77cce37f093dec7b7aaedca475f608b092d13706bfff1dbe1a8", + "sdil/data.py": "262031353b5952fb4d1b0e3c3d1e9b3186ae42b88601ccb89e1748ffd36ed36d" + } + }, + "stage": "p1" + }, + "missing_experiments": [], + "num_audited_records": 19, + "num_expected_records": 19, + "records": [ + { + "all_training_losses_finite": true, + "best_epoch": 5, + "best_validation_accuracy": 0.1038, + "driver_wall_seconds": 4649.1066772937775, + "epochs_completed": 10, + "experiment_name": "resnet-p1-ep-d20-lr0p0003", + "final_validation_accuracy": 0.1, + "final_validation_loss": 0.09551684875488281, + "finite": true, + "manifest": "results/resnet_crossover/p1/resnet-p1-ep-d20-lr0p0003.json.manifest.json", + "method": "ep", + "output_sha256": "eadd704ff9620e03f4682e664cfe317036e54fec3f5a59489430975c8705fce2", + "physical_gpu_index": 5, + "physical_gpu_uuid": "GPU-f4aed756-228e-7522-94e7-fb543c02cc74", + "rate": 0.0003, + "status": "completed" + }, + { + "all_training_losses_finite": true, + "best_epoch": 9, + "best_validation_accuracy": 0.7444, + "driver_wall_seconds": 7024.675570964813, + "epochs_completed": 10, + "experiment_name": "resnet-p1-dualprop-d20-lr0p01", + "final_validation_accuracy": 0.7306, + "final_validation_loss": 0.8069891845703125, + "finite": true, + "manifest": "results/resnet_crossover/p1/resnet-p1-dualprop-d20-lr0p01.json.manifest.json", + "method": "dualprop", + "output_sha256": "1d6b91ed4503922725e548821e73c61b27488df1fb184576bc03491853819633", + "physical_gpu_index": 7, + "physical_gpu_uuid": "GPU-df3d5933-c3e9-9ea1-d595-9d2a3b68c14b", + "rate": 0.01, + "status": "completed" + }, + { + "all_training_losses_finite": true, + "best_epoch": 1, + "best_validation_accuracy": 0.1, + "driver_wall_seconds": 4655.980449676514, + "epochs_completed": 10, + "experiment_name": "resnet-p1-ep-d20-lr0p001", + "final_validation_accuracy": 0.1, + "final_validation_loss": 0.09599873504638672, + "finite": true, + "manifest": "results/resnet_crossover/p1/resnet-p1-ep-d20-lr0p001.json.manifest.json", + "method": "ep", + "output_sha256": "ee76e1c71f87b443a23568508f1a3b60b65d2a030c92c48e3b73fc28bf01997d", + "physical_gpu_index": 5, + "physical_gpu_uuid": "GPU-f4aed756-228e-7522-94e7-fb543c02cc74", + "rate": 0.001, + "status": "completed" + }, + { + "all_training_losses_finite": true, + "best_epoch": 7, + "best_validation_accuracy": 0.7566, + "driver_wall_seconds": 7031.210079908371, + "epochs_completed": 10, + "experiment_name": "resnet-p1-dualprop-d20-lr0p025", + "final_validation_accuracy": 0.7524, + "final_validation_loss": 0.7849000122070312, + "finite": true, + "manifest": "results/resnet_crossover/p1/resnet-p1-dualprop-d20-lr0p025.json.manifest.json", + "method": "dualprop", + "output_sha256": "0b3c04830521ee443d605bbd9b6fda2c2ddc61982174d10cbadf6f1a1838a09d", + "physical_gpu_index": 7, + "physical_gpu_uuid": "GPU-df3d5933-c3e9-9ea1-d595-9d2a3b68c14b", + "rate": 0.025, + "status": "completed" + }, + { + "all_training_losses_finite": true, + "best_epoch": 1, + "best_validation_accuracy": 0.1, + "driver_wall_seconds": 4653.510211229324, + "epochs_completed": 10, + "experiment_name": "resnet-p1-ep-d20-lr0p003", + "final_validation_accuracy": 0.1, + "final_validation_loss": 0.09405416259765625, + "finite": true, + "manifest": "results/resnet_crossover/p1/resnet-p1-ep-d20-lr0p003.json.manifest.json", + "method": "ep", + "output_sha256": "3e9ae8cf2f75641845a5b0c45b3fa68e97b384f1637ddbee3a73ccaa3515d388", + "physical_gpu_index": 5, + "physical_gpu_uuid": "GPU-f4aed756-228e-7522-94e7-fb543c02cc74", + "rate": 0.003, + "status": "completed" + }, + { + "all_training_losses_finite": true, + "best_epoch": 9, + "best_validation_accuracy": 0.8018, + "driver_wall_seconds": 7030.075044870377, + "epochs_completed": 10, + "experiment_name": "resnet-p1-dualprop-d20-lr0p05", + "final_validation_accuracy": 0.768, + "final_validation_loss": 0.74344345703125, + "finite": true, + "manifest": "results/resnet_crossover/p1/resnet-p1-dualprop-d20-lr0p05.json.manifest.json", + "method": "dualprop", + "output_sha256": "9821030b0eb8908b642be7a5514c760887a38cdfacd4181976e2714a96336882", + "physical_gpu_index": 7, + "physical_gpu_uuid": "GPU-df3d5933-c3e9-9ea1-d595-9d2a3b68c14b", + "rate": 0.05, + "status": "completed" + }, + { + "all_training_losses_finite": true, + "best_epoch": 9, + "best_validation_accuracy": 0.2044, + "driver_wall_seconds": 170.44619464874268, + "epochs_completed": 10, + "experiment_name": "resnet-p1-pepita-d20-lr0p0003", + "final_validation_accuracy": 0.203, + "final_validation_loss": 2.17982470703125, + "finite": true, + "manifest": "results/resnet_crossover/p1/resnet-p1-pepita-d20-lr0p0003.json.manifest.json", + "method": "pepita", + "output_sha256": "94015d774b810af1c0c8dfea070e8285ad915cd9198df9166b3758c2f73c764c", + "physical_gpu_index": 5, + "physical_gpu_uuid": "GPU-f4aed756-228e-7522-94e7-fb543c02cc74", + "rate": 0.0003, + "status": "completed" + }, + { + "all_training_losses_finite": true, + "best_epoch": 10, + "best_validation_accuracy": 0.3796, + "driver_wall_seconds": 207.9275677204132, + "epochs_completed": 10, + "experiment_name": "resnet-p1-fa-d20-lr0p01", + "final_validation_accuracy": 0.3796, + "final_validation_loss": 2.10581982421875, + "finite": true, + "manifest": "results/resnet_crossover/p1/resnet-p1-fa-d20-lr0p01.json.manifest.json", + "method": "fa", + "output_sha256": "36a086b8f6a1c7daed7775a91cda5e96c3142674c201521af328cf34ee6d4d8a", + "physical_gpu_index": 5, + "physical_gpu_uuid": "GPU-f4aed756-228e-7522-94e7-fb543c02cc74", + "rate": 0.01, + "status": "completed" + }, + { + "all_training_losses_finite": true, + "best_epoch": 9, + "best_validation_accuracy": 0.2352, + "driver_wall_seconds": 170.54007387161255, + "epochs_completed": 10, + "experiment_name": "resnet-p1-pepita-d20-lr0p001", + "final_validation_accuracy": 0.2334, + "final_validation_loss": 2.09603291015625, + "finite": true, + "manifest": "results/resnet_crossover/p1/resnet-p1-pepita-d20-lr0p001.json.manifest.json", + "method": "pepita", + "output_sha256": "997c07cefd9d7cfdebf41cda91be704008c8c1185231d997bad2344e230f4786", + "physical_gpu_index": 5, + "physical_gpu_uuid": "GPU-f4aed756-228e-7522-94e7-fb543c02cc74", + "rate": 0.001, + "status": "completed" + }, + { + "all_training_losses_finite": true, + "best_epoch": 10, + "best_validation_accuracy": 0.2996, + "driver_wall_seconds": 172.78827118873596, + "epochs_completed": 10, + "experiment_name": "resnet-p1-dfa-d20-lr0p01", + "final_validation_accuracy": 0.2996, + "final_validation_loss": 4.5880673828125, + "finite": true, + "manifest": "results/resnet_crossover/p1/resnet-p1-dfa-d20-lr0p01.json.manifest.json", + "method": "dfa", + "output_sha256": "cb4bbbc1d5a9d011ebefe201810129647688d4bdc30f31cab517745c05fffc69", + "physical_gpu_index": 5, + "physical_gpu_uuid": "GPU-f4aed756-228e-7522-94e7-fb543c02cc74", + "rate": 0.01, + "status": "completed" + }, + { + "all_training_losses_finite": true, + "best_epoch": 10, + "best_validation_accuracy": 0.2634, + "driver_wall_seconds": 170.33968472480774, + "epochs_completed": 10, + "experiment_name": "resnet-p1-pepita-d20-lr0p003", + "final_validation_accuracy": 0.2634, + "final_validation_loss": 2.027465869140625, + "finite": true, + "manifest": "results/resnet_crossover/p1/resnet-p1-pepita-d20-lr0p003.json.manifest.json", + "method": "pepita", + "output_sha256": "889670a9f62b242477a8f15dd5a227cb2081b4c16ff9d27ef78087fa8d956e89", + "physical_gpu_index": 5, + "physical_gpu_uuid": "GPU-f4aed756-228e-7522-94e7-fb543c02cc74", + "rate": 0.003, + "status": "completed" + }, + { + "all_training_losses_finite": true, + "best_epoch": 7, + "best_validation_accuracy": 0.4022, + "driver_wall_seconds": 209.6812744140625, + "epochs_completed": 10, + "experiment_name": "resnet-p1-fa-d20-lr0p03", + "final_validation_accuracy": 0.3436, + "final_validation_loss": 3.315408740234375, + "finite": true, + "manifest": "results/resnet_crossover/p1/resnet-p1-fa-d20-lr0p03.json.manifest.json", + "method": "fa", + "output_sha256": "f2e5b1a35f25c0c4c6937e19d41099bf7a8e42e10d3561b135bbd6b6487bdf74", + "physical_gpu_index": 5, + "physical_gpu_uuid": "GPU-f4aed756-228e-7522-94e7-fb543c02cc74", + "rate": 0.03, + "status": "completed" + }, + { + "all_training_losses_finite": true, + "best_epoch": 3, + "best_validation_accuracy": 0.2848, + "driver_wall_seconds": 176.0880675315857, + "epochs_completed": 10, + "experiment_name": "resnet-p1-dfa-d20-lr0p03", + "final_validation_accuracy": 0.277, + "final_validation_loss": 32.40516796875, + "finite": true, + "manifest": "results/resnet_crossover/p1/resnet-p1-dfa-d20-lr0p03.json.manifest.json", + "method": "dfa", + "output_sha256": "24e7a0dfd6755102438224e1457878cc0c6afdc3cc8e41edb8036748d71e93f4", + "physical_gpu_index": 5, + "physical_gpu_uuid": "GPU-f4aed756-228e-7522-94e7-fb543c02cc74", + "rate": 0.03, + "status": "completed" + }, + { + "all_training_losses_finite": true, + "best_epoch": 8, + "best_validation_accuracy": 0.4356, + "driver_wall_seconds": 209.7303650379181, + "epochs_completed": 10, + "experiment_name": "resnet-p1-fa-d20-lr0p1", + "final_validation_accuracy": 0.409, + "final_validation_loss": 2.6504962890625, + "finite": true, + "manifest": "results/resnet_crossover/p1/resnet-p1-fa-d20-lr0p1.json.manifest.json", + "method": "fa", + "output_sha256": "080674dfa112aaee5800b9f0978f86ca09bac80e367ee28c89acecb0576aa111", + "physical_gpu_index": 5, + "physical_gpu_uuid": "GPU-f4aed756-228e-7522-94e7-fb543c02cc74", + "rate": 0.1, + "status": "completed" + }, + { + "all_training_losses_finite": true, + "best_epoch": 2, + "best_validation_accuracy": 0.289, + "driver_wall_seconds": 174.0400595664978, + "epochs_completed": 10, + "experiment_name": "resnet-p1-dfa-d20-lr0p1", + "final_validation_accuracy": 0.2544, + "final_validation_loss": 280.33305, + "finite": true, + "manifest": "results/resnet_crossover/p1/resnet-p1-dfa-d20-lr0p1.json.manifest.json", + "method": "dfa", + "output_sha256": "e51cb4bbb0d60f5f5e572548b9dfddbd9e78302523eea1f12bd5cdf3286f5a6d", + "physical_gpu_index": 5, + "physical_gpu_uuid": "GPU-f4aed756-228e-7522-94e7-fb543c02cc74", + "rate": 0.1, + "status": "completed" + }, + { + "all_training_losses_finite": true, + "best_epoch": 9, + "best_validation_accuracy": 0.8066, + "driver_wall_seconds": 176.89754939079285, + "epochs_completed": 10, + "experiment_name": "resnet-p1-bp-d20-lr0p1", + "final_validation_accuracy": 0.775, + "final_validation_loss": 0.6895121826171875, + "finite": true, + "manifest": "results/resnet_crossover/p1/resnet-p1-bp-d20-lr0p1.json.manifest.json", + "method": "bp", + "output_sha256": "dc6d7c24c944c92b6b8fdd25aaa83e71246ac876ccc84c811dd004c1b7005187", + "physical_gpu_index": 5, + "physical_gpu_uuid": "GPU-f4aed756-228e-7522-94e7-fb543c02cc74", + "rate": 0.1, + "status": "completed" + }, + { + "all_training_losses_finite": true, + "best_epoch": null, + "best_validation_accuracy": 0.1146, + "driver_wall_seconds": 2130.4099085330963, + "epochs_completed": 200, + "experiment_name": "resnet-p1-ff-d20-lr0p03", + "final_validation_accuracy": 0.1146, + "final_validation_loss": 0.8081799595726861, + "finite": true, + "manifest": "results/resnet_crossover/p1/resnet-p1-ff-d20-lr0p03.json.manifest.json", + "method": "ff", + "output_sha256": "2de758d013a34bc7faf35d9f802bcc213c2eb334be96237daab1372f85ed8dc0", + "physical_gpu_index": 5, + "physical_gpu_uuid": "GPU-f4aed756-228e-7522-94e7-fb543c02cc74", + "rate": 0.03, + "status": "completed" + }, + { + "all_training_losses_finite": true, + "best_epoch": 10, + "best_validation_accuracy": 0.7952, + "driver_wall_seconds": 288.57743978500366, + "epochs_completed": 10, + "experiment_name": "resnet-p1-clean_kp-d20-lr0p1", + "final_validation_accuracy": 0.7952, + "final_validation_loss": 0.6321130859375, + "finite": true, + "manifest": "results/resnet_crossover/p1/resnet-p1-clean_kp-d20-lr0p1.json.manifest.json", + "method": "clean_kp", + "output_sha256": "256c3a7587a2803970d54a8925020741274b5620601a79a3770851488f43e55a", + "physical_gpu_index": 5, + "physical_gpu_uuid": "GPU-f4aed756-228e-7522-94e7-fb543c02cc74", + "rate": 0.1, + "status": "completed" + }, + { + "all_training_losses_finite": true, + "best_epoch": 10, + "best_validation_accuracy": 0.7646, + "driver_wall_seconds": 416.28714871406555, + "epochs_completed": 10, + "experiment_name": "resnet-p1-sdil-d20-lr0p1", + "final_validation_accuracy": 0.7646, + "final_validation_loss": 0.7005346435546875, + "finite": true, + "manifest": "results/resnet_crossover/p1/resnet-p1-sdil-d20-lr0p1.json.manifest.json", + "method": "sdil", + "output_sha256": "f465e0eb6c0c0ce0da1f3bfced5a04961c291d6c7ca5f8f25f53fd3acb0c0d72", + "physical_gpu_index": 5, + "physical_gpu_uuid": "GPU-f4aed756-228e-7522-94e7-fb543c02cc74", + "rate": 0.1, + "status": "completed" + } + ], + "selected": { + "bp": { + "best_validation_accuracy": 0.8066, + "experiment_name": "resnet-p1-bp-d20-lr0p1", + "rate": 0.1 + }, + "clean_kp": { + "best_validation_accuracy": 0.7952, + "experiment_name": "resnet-p1-clean_kp-d20-lr0p1", + "rate": 0.1 + }, + "dfa": { + "best_validation_accuracy": 0.2996, + "experiment_name": "resnet-p1-dfa-d20-lr0p01", + "rate": 0.01 + }, + "dualprop": { + "best_validation_accuracy": 0.8018, + "experiment_name": "resnet-p1-dualprop-d20-lr0p05", + "rate": 0.05 + }, + "ep": { + "best_validation_accuracy": 0.1038, + "experiment_name": "resnet-p1-ep-d20-lr0p0003", + "rate": 0.0003 + }, + "fa": { + "best_validation_accuracy": 0.4356, + "experiment_name": "resnet-p1-fa-d20-lr0p1", + "rate": 0.1 + }, + "ff": { + "best_validation_accuracy": 0.1146, + "experiment_name": "resnet-p1-ff-d20-lr0p03", + "rate": 0.03 + }, + "pepita": { + "best_validation_accuracy": 0.2634, + "experiment_name": "resnet-p1-pepita-d20-lr0p003", + "rate": 0.003 + }, + "sdil": { + "best_validation_accuracy": 0.7646, + "experiment_name": "resnet-p1-sdil-d20-lr0p1", + "rate": 0.1 + } + }, + "selection_rule": "maximum best validation accuracy, then finite final loss, then lower learning rate", + "source": { + "git_commit": "19f3966fc2d15b1ed18862a779d91ee16ed56bfc", + "protocol_path": "/scratch/yurenh2/sdil-runs/resnet-p1/RESNET_CROSSOVER.md", + "protocol_sha256": "048dbaf06866bdd3959a637ba827c06063205f6d1cc693d89aa8fc9c1d24975d", + "python": "/home/yurenh2/miniconda3/envs/ep_pascal/bin/python3", + "tracked_files": { + "RESNET_CROSSOVER.md": "048dbaf06866bdd3959a637ba827c06063205f6d1cc693d89aa8fc9c1d24975d", + "experiments/analyze_resnet_crossover_p1.py": "6ff485a3fecc7b814dedf55d04abdb770a6ce904cdeb9e06a7dddcf26f996d5b", + "experiments/conv_run.py": "9fa2daea597d63981a78248fafb154da39775ccfae57aac9889e78d83dc4eb44", + "experiments/resnet_crossover_grid.py": "669c84846bc3115816ae3c9036be8722f21577612dc9d56a689d3321e837ca95", + "experiments/resnet_crossover_native.py": "c5e54324b000c37c53d495f7d3eb0d4a85dcf15e83ea3aab125a097ce61eff87", + "sdil/conv.py": "09f87eef636aa5397b39a2a7a66909b8861640624507eabbdfaffe98577a5d2c", + "sdil/conv_crossover.py": "3c1373aa3555e77cce37f093dec7b7aaedca475f608b092d13706bfff1dbe1a8", + "sdil/data.py": "262031353b5952fb4d1b0e3c3d1e9b3186ae42b88601ccb89e1748ffd36ed36d" + } + }, + "stage": "resnet_crossover_p1", + "test_policy": "none" +} diff --git a/results/transformer_crossover/p1_selector.json b/results/transformer_crossover/p1_selector.json new file mode 100644 index 0000000..0e2e076 --- /dev/null +++ b/results/transformer_crossover/p1_selector.json @@ -0,0 +1,1158 @@ +{ + "gate": "pass", + "launch_lock": { + "allowed_physical_gpus": [ + 5, + 7 + ], + "num_jobs": 27, + "registry_sha256": "ed2d757c69e6ef5614e436e9394a77951b75cdc91a97569f2f7451b0fd7bfb5b", + "source": { + "environment_freeze": [ + "filelock==3.29.0", + "fsspec==2026.4.0", + "Jinja2==3.1.6", + "MarkupSafe==3.0.3", + "mpmath==1.3.0", + "networkx==3.6.1", + "numpy==2.4.4", + "nvidia-cublas-cu11==11.11.3.6", + "nvidia-cuda-cupti-cu11==11.8.87", + "nvidia-cuda-nvrtc-cu11==11.8.89", + "nvidia-cuda-runtime-cu11==11.8.89", + "nvidia-cudnn-cu11==8.7.0.84", + "nvidia-cufft-cu11==10.9.0.58", + "nvidia-curand-cu11==10.3.0.86", + "nvidia-cusolver-cu11==11.4.1.48", + "nvidia-cusparse-cu11==11.7.5.86", + "nvidia-nccl-cu11==2.20.5", + "nvidia-nvtx-cu11==11.8.86", + "packaging==26.0", + "scipy==1.17.1", + "sympy==1.14.0", + "torch==2.3.1+cu118", + "triton==2.3.1", + "typing_extensions==4.15.0" + ], + "environment_freeze_sha256": "bca82e6091ef12808e890cc36a0095dd885916b1d5e0cd4a8aa2119b6ca861a5", + "git_commit": "19f3966fc2d15b1ed18862a779d91ee16ed56bfc", + "protocol_path": "/scratch/yurenh2/sdil-runs/transformer-p1/TRANSFORMER_CROSSOVER.md", + "protocol_sha256": "841a107328f4961c9fa14cbf357b0c5699ac9b3e34b8d124ccb389ab145c8a0e", + "python": "/home/yurenh2/miniconda3/envs/ep_pascal/bin/python3", + "tracked_files": { + "TRANSFORMER_CROSSOVER.md": "841a107328f4961c9fa14cbf357b0c5699ac9b3e34b8d124ccb389ab145c8a0e", + "experiments/analyze_transformer_crossover_p1.py": "76a1df0db00d36d2732b76fd71b54f3aa0057dd611186162d8b0fc442a60d294", + "experiments/transformer_crossover_grid.py": "947152fb257dd005e22aa546be8ff0f63afe80436aa5f591db4dc29f99fb5eac", + "experiments/transformer_crossover_native.py": "c45299ecfdd80bab8ededa6f8c0e1ad2275ea7a3d68f711ee7cddad29a933c5e", + "experiments/transformer_feedback_smoke.py": "a8c0ba44091765ee2e05381f36515923990fd80e0acfef5aeea58ca29ec52092", + "sdil/transformer.py": "09180e5625054c4dc16567af8939d05c4cb7cbc08f959445f25daae7c830cb34" + } + }, + "stage": "p1" + }, + "missing_experiments": [], + "num_audited_records": 27, + "num_expected_records": 27, + "records": [ + { + "completed_optimizer_steps": 1000, + "driver_wall_seconds": 622.442409992218, + "experiment_name": "transformer-p1-ep-d4-lr0p0001", + "final_validation_accuracy": 0.08571326061997704, + "final_validation_nll": 4.152099359733502, + "final_validation_perplexity": 63.56731096864925, + "finite": true, + "manifest": "results/transformer_crossover/p1/transformer-p1-ep-d4-lr0p0001.json.manifest.json", + "method": "ep", + "output_sha256": "82bb6da5cc9abe6707c62559be34b78d9c90bc40beb8f5039e8ac39dad9c6dde", + "peak_memory_allocated_bytes": 88484352, + "physical_gpu_index": 5, + "physical_gpu_uuid": "GPU-f4aed756-228e-7522-94e7-fb543c02cc74", + "rate": 0.0001, + "status": "completed", + "total_wall_seconds": 616.3357925415039, + "work": { + "block_affine_macs_per_token": 196608, + "block_attention_macs_per_token": 16384, + "candidate_token_presentations": 0, + "completed_optimizer_steps": 1000, + "enumerated_full_forward_macs": 3619651436544, + "feedback_parameter_count": 0, + "forward_affine_macs_per_token": 794752, + "forward_attention_macs_per_token": 65536, + "forward_parameter_count": 813568, + "full_forward_token_passes": 4207488, + "local_block_token_evaluations": 427438080, + "local_head_token_evaluations": 106859520, + "local_vjp_token_evaluations": 245760000, + "ordinary_training_tokens": 2048000, + "ordinary_validation_tokens": 111488, + "relaxation_token_passes": 51381760, + "training_token_presentations": 2048000 + } + }, + { + "completed_optimizer_steps": 1000, + "driver_wall_seconds": 382.97613072395325, + "experiment_name": "transformer-p1-dualprop-d4-lr0p0001", + "final_validation_accuracy": 0.3468714121699196, + "final_validation_nll": 2.233302180994255, + "final_validation_perplexity": 9.33062668108521, + "finite": true, + "manifest": "results/transformer_crossover/p1/transformer-p1-dualprop-d4-lr0p0001.json.manifest.json", + "method": "dualprop", + "output_sha256": "26e104e226d741df34d4c6f78e13ff24880825bb346c37572133a50b1c52625d", + "peak_memory_allocated_bytes": 97268224, + "physical_gpu_index": 7, + "physical_gpu_uuid": "GPU-df3d5933-c3e9-9ea1-d595-9d2a3b68c14b", + "rate": 0.0001, + "status": "completed", + "total_wall_seconds": 378.6061809062958, + "work": { + "block_affine_macs_per_token": 196608, + "block_attention_macs_per_token": 16384, + "candidate_token_presentations": 0, + "completed_optimizer_steps": 1000, + "enumerated_full_forward_macs": 1857781612544, + "feedback_parameter_count": 0, + "forward_affine_macs_per_token": 794752, + "forward_attention_macs_per_token": 65536, + "forward_parameter_count": 813568, + "full_forward_token_passes": 2159488, + "local_block_token_evaluations": 270336000, + "local_head_token_evaluations": 67584000, + "local_vjp_token_evaluations": 163840000, + "ordinary_training_tokens": 2048000, + "ordinary_validation_tokens": 111488, + "relaxation_token_passes": 32768000, + "training_token_presentations": 2048000 + } + }, + { + "completed_optimizer_steps": 1000, + "driver_wall_seconds": 34.00058722496033, + "experiment_name": "transformer-p1-ff-d4-lr0p0001", + "final_validation_accuracy": 0.14900258323765786, + "final_validation_nll": 8.052283456760762, + "final_validation_perplexity": 3140.959032901827, + "finite": true, + "manifest": "results/transformer_crossover/p1/transformer-p1-ff-d4-lr0p0001.json.manifest.json", + "method": "ff", + "output_sha256": "cb314c837f4fe047781bc38cbf45222cb93b81717eded487460abbbc32af38a0", + "peak_memory_allocated_bytes": 87777792, + "physical_gpu_index": 5, + "physical_gpu_uuid": "GPU-f4aed756-228e-7522-94e7-fb543c02cc74", + "rate": 0.0001, + "status": "completed", + "total_wall_seconds": 28.628129959106445, + "work": { + "block_affine_macs_per_token": 196608, + "block_attention_macs_per_token": 16384, + "candidate_token_presentations": 7246720, + "completed_optimizer_steps": 1000, + "enumerated_full_forward_macs": 9758005903360, + "feedback_parameter_count": 0, + "forward_affine_macs_per_token": 794752, + "forward_attention_macs_per_token": 65536, + "forward_parameter_count": 813568, + "full_forward_token_passes": 11342720, + "local_block_token_evaluations": 1368064, + "local_head_token_evaluations": 337920, + "local_vjp_token_evaluations": 0, + "ordinary_training_tokens": 2048000, + "ordinary_validation_tokens": 111488, + "relaxation_token_passes": 0, + "training_token_presentations": 4096000 + } + }, + { + "completed_optimizer_steps": 1000, + "driver_wall_seconds": 39.868736267089844, + "experiment_name": "transformer-p1-pepita-d4-lr0p0001", + "final_validation_accuracy": 0.16012485648679678, + "final_validation_nll": 4.210206946056554, + "final_validation_perplexity": 67.3704804228411, + "finite": true, + "manifest": "results/transformer_crossover/p1/transformer-p1-pepita-d4-lr0p0001.json.manifest.json", + "method": "pepita", + "output_sha256": "9047420c0d7af0d50bf7309fb326a955dd259bd2820f7b9b2f83fac7ad671f9e", + "peak_memory_allocated_bytes": 74502144, + "physical_gpu_index": 7, + "physical_gpu_uuid": "GPU-df3d5933-c3e9-9ea1-d595-9d2a3b68c14b", + "rate": 0.0001, + "status": "completed", + "total_wall_seconds": 34.56653308868408, + "work": { + "block_affine_macs_per_token": 196608, + "block_attention_macs_per_token": 16384, + "candidate_token_presentations": 0, + "completed_optimizer_steps": 1000, + "enumerated_full_forward_macs": 3619651436544, + "feedback_parameter_count": 8320, + "forward_affine_macs_per_token": 794752, + "forward_attention_macs_per_token": 65536, + "forward_parameter_count": 813568, + "full_forward_token_passes": 4207488, + "local_block_token_evaluations": 8192000, + "local_head_token_evaluations": 2048000, + "local_vjp_token_evaluations": 0, + "ordinary_training_tokens": 2048000, + "ordinary_validation_tokens": 111488, + "relaxation_token_passes": 0, + "training_token_presentations": 4096000 + } + }, + { + "completed_optimizer_steps": 1000, + "driver_wall_seconds": 34.50551080703735, + "experiment_name": "transformer-p1-dfa-d4-lr0p0003", + "final_validation_accuracy": 0.2341238518943743, + "final_validation_nll": 2.8146539484455446, + "final_validation_perplexity": 16.687400080306357, + "finite": true, + "manifest": "results/transformer_crossover/p1/transformer-p1-dfa-d4-lr0p0003.json.manifest.json", + "method": "dfa", + "output_sha256": "50d89a4d090da1127556dea87b9e6208c29231692286d0f4274f1b0277a32c6a", + "peak_memory_allocated_bytes": 66264064, + "physical_gpu_index": 5, + "physical_gpu_uuid": "GPU-f4aed756-228e-7522-94e7-fb543c02cc74", + "rate": 0.0003, + "status": "completed", + "total_wall_seconds": 30.448689222335815, + "work": { + "block_affine_macs_per_token": 196608, + "block_attention_macs_per_token": 16384, + "candidate_token_presentations": 0, + "completed_optimizer_steps": 1000, + "enumerated_full_forward_macs": 1857781612544, + "feedback_parameter_count": 49920, + "forward_affine_macs_per_token": 794752, + "forward_attention_macs_per_token": 65536, + "forward_parameter_count": 813568, + "full_forward_token_passes": 2159488, + "local_block_token_evaluations": 8192000, + "local_head_token_evaluations": 2048000, + "local_vjp_token_evaluations": 0, + "ordinary_training_tokens": 2048000, + "ordinary_validation_tokens": 111488, + "relaxation_token_passes": 0, + "training_token_presentations": 2048000 + } + }, + { + "completed_optimizer_steps": 1000, + "driver_wall_seconds": 43.72804021835327, + "experiment_name": "transformer-p1-fa-d4-lr0p0003", + "final_validation_accuracy": 0.31162098163031, + "final_validation_nll": 2.3651696017908037, + "final_validation_perplexity": 10.645844218650149, + "finite": true, + "manifest": "results/transformer_crossover/p1/transformer-p1-fa-d4-lr0p0003.json.manifest.json", + "method": "fa", + "output_sha256": "9d1fb33d21e0a4a78f7a3f4804e83134f5813bc02f72948ce168b7c8b09a8710", + "peak_memory_allocated_bytes": 146653184, + "physical_gpu_index": 7, + "physical_gpu_uuid": "GPU-df3d5933-c3e9-9ea1-d595-9d2a3b68c14b", + "rate": 0.0003, + "status": "completed", + "total_wall_seconds": 39.57168173789978, + "work": { + "block_affine_macs_per_token": 196608, + "block_attention_macs_per_token": 16384, + "candidate_token_presentations": 0, + "completed_optimizer_steps": 1000, + "enumerated_full_forward_macs": 1857781612544, + "feedback_parameter_count": 794752, + "forward_affine_macs_per_token": 794752, + "forward_attention_macs_per_token": 65536, + "forward_parameter_count": 813568, + "full_forward_token_passes": 2159488, + "local_block_token_evaluations": 0, + "local_head_token_evaluations": 0, + "local_vjp_token_evaluations": 0, + "ordinary_training_tokens": 2048000, + "ordinary_validation_tokens": 111488, + "relaxation_token_passes": 0, + "training_token_presentations": 2048000 + } + }, + { + "completed_optimizer_steps": 1000, + "driver_wall_seconds": 43.616663694381714, + "experiment_name": "transformer-p1-clean_kp-d4-lr0p0003", + "final_validation_accuracy": 0.3621017508610792, + "final_validation_nll": 2.1674982721304645, + "final_validation_perplexity": 8.736400582109368, + "finite": true, + "manifest": "results/transformer_crossover/p1/transformer-p1-clean_kp-d4-lr0p0003.json.manifest.json", + "method": "clean_kp", + "output_sha256": "ce7553d786874d54a90be4637b3f1ee24ddb179ffac553bf70c46d8f09959f46", + "peak_memory_allocated_bytes": 153568768, + "physical_gpu_index": 5, + "physical_gpu_uuid": "GPU-f4aed756-228e-7522-94e7-fb543c02cc74", + "rate": 0.0003, + "status": "completed", + "total_wall_seconds": 38.4737503528595, + "work": { + "block_affine_macs_per_token": 196608, + "block_attention_macs_per_token": 16384, + "candidate_token_presentations": 0, + "completed_optimizer_steps": 1000, + "enumerated_full_forward_macs": 1857781612544, + "feedback_parameter_count": 794752, + "forward_affine_macs_per_token": 794752, + "forward_attention_macs_per_token": 65536, + "forward_parameter_count": 813568, + "full_forward_token_passes": 2159488, + "local_block_token_evaluations": 0, + "local_head_token_evaluations": 0, + "local_vjp_token_evaluations": 0, + "ordinary_training_tokens": 2048000, + "ordinary_validation_tokens": 111488, + "relaxation_token_passes": 0, + "training_token_presentations": 2048000 + } + }, + { + "completed_optimizer_steps": 1000, + "driver_wall_seconds": 49.88345217704773, + "experiment_name": "transformer-p1-sdil-d4-lr0p0003", + "final_validation_accuracy": 0.3621017508610792, + "final_validation_nll": 2.1674982283337108, + "final_validation_perplexity": 8.736400199483393, + "finite": true, + "manifest": "results/transformer_crossover/p1/transformer-p1-sdil-d4-lr0p0003.json.manifest.json", + "method": "sdil", + "output_sha256": "932ac4dced706ae75f44a852c56992c7135d6afc954acee1b8af31507b350f50", + "peak_memory_allocated_bytes": 157764096, + "physical_gpu_index": 7, + "physical_gpu_uuid": "GPU-df3d5933-c3e9-9ea1-d595-9d2a3b68c14b", + "rate": 0.0003, + "status": "completed", + "total_wall_seconds": 44.673834562301636, + "work": { + "block_affine_macs_per_token": 196608, + "block_attention_macs_per_token": 16384, + "candidate_token_presentations": 0, + "completed_optimizer_steps": 1000, + "enumerated_full_forward_macs": 1857781612544, + "feedback_parameter_count": 794752, + "forward_affine_macs_per_token": 794752, + "forward_attention_macs_per_token": 65536, + "forward_parameter_count": 813568, + "full_forward_token_passes": 2159488, + "local_block_token_evaluations": 0, + "local_head_token_evaluations": 0, + "local_vjp_token_evaluations": 0, + "ordinary_training_tokens": 2048000, + "ordinary_validation_tokens": 111488, + "relaxation_token_passes": 0, + "training_token_presentations": 2048000 + } + }, + { + "completed_optimizer_steps": 1000, + "driver_wall_seconds": 25.821476459503174, + "experiment_name": "transformer-p1-bp-d4-lr0p0003", + "final_validation_accuracy": 0.3950470005740528, + "final_validation_nll": 2.048308381256909, + "final_validation_perplexity": 7.754771887148325, + "finite": true, + "manifest": "results/transformer_crossover/p1/transformer-p1-bp-d4-lr0p0003.json.manifest.json", + "method": "bp", + "output_sha256": "e7d74c34973507feb23ab114108c8ef3f0883e5785a171fc86f4c33db0b06cc2", + "peak_memory_allocated_bytes": 109656064, + "physical_gpu_index": 5, + "physical_gpu_uuid": "GPU-f4aed756-228e-7522-94e7-fb543c02cc74", + "rate": 0.0003, + "status": "completed", + "total_wall_seconds": 21.73075294494629, + "work": { + "block_affine_macs_per_token": 196608, + "block_attention_macs_per_token": 16384, + "candidate_token_presentations": 0, + "completed_optimizer_steps": 1000, + "enumerated_full_forward_macs": 1857781612544, + "feedback_parameter_count": 0, + "forward_affine_macs_per_token": 794752, + "forward_attention_macs_per_token": 65536, + "forward_parameter_count": 813568, + "full_forward_token_passes": 2159488, + "local_block_token_evaluations": 0, + "local_head_token_evaluations": 0, + "local_vjp_token_evaluations": 0, + "ordinary_training_tokens": 2048000, + "ordinary_validation_tokens": 111488, + "relaxation_token_passes": 0, + "training_token_presentations": 2048000 + } + }, + { + "completed_optimizer_steps": 1000, + "driver_wall_seconds": 584.4020130634308, + "experiment_name": "transformer-p1-ep-d4-lr0p0003", + "final_validation_accuracy": 0.05100997416762342, + "final_validation_nll": 4.157737329003457, + "final_validation_perplexity": 63.926713713260646, + "finite": true, + "manifest": "results/transformer_crossover/p1/transformer-p1-ep-d4-lr0p0003.json.manifest.json", + "method": "ep", + "output_sha256": "a8c78c833c7db8823685bf70e3b3303acd9945da01ce9a6733421e8229dc9390", + "peak_memory_allocated_bytes": 88484352, + "physical_gpu_index": 7, + "physical_gpu_uuid": "GPU-df3d5933-c3e9-9ea1-d595-9d2a3b68c14b", + "rate": 0.0003, + "status": "completed", + "total_wall_seconds": 580.2041375637054, + "work": { + "block_affine_macs_per_token": 196608, + "block_attention_macs_per_token": 16384, + "candidate_token_presentations": 0, + "completed_optimizer_steps": 1000, + "enumerated_full_forward_macs": 3619651436544, + "feedback_parameter_count": 0, + "forward_affine_macs_per_token": 794752, + "forward_attention_macs_per_token": 65536, + "forward_parameter_count": 813568, + "full_forward_token_passes": 4207488, + "local_block_token_evaluations": 427438080, + "local_head_token_evaluations": 106859520, + "local_vjp_token_evaluations": 245760000, + "ordinary_training_tokens": 2048000, + "ordinary_validation_tokens": 111488, + "relaxation_token_passes": 51381760, + "training_token_presentations": 2048000 + } + }, + { + "completed_optimizer_steps": 1000, + "driver_wall_seconds": 403.95608258247375, + "experiment_name": "transformer-p1-dualprop-d4-lr0p0003", + "final_validation_accuracy": 0.3950470005740528, + "final_validation_nll": 2.0483097104883905, + "final_validation_perplexity": 7.7547821950420985, + "finite": true, + "manifest": "results/transformer_crossover/p1/transformer-p1-dualprop-d4-lr0p0003.json.manifest.json", + "method": "dualprop", + "output_sha256": "3d97a86f64e7c3485412ac0c737d7bd4b38f9d368d9fad4b7036ea0727f0f944", + "peak_memory_allocated_bytes": 97268224, + "physical_gpu_index": 5, + "physical_gpu_uuid": "GPU-f4aed756-228e-7522-94e7-fb543c02cc74", + "rate": 0.0003, + "status": "completed", + "total_wall_seconds": 398.5918562412262, + "work": { + "block_affine_macs_per_token": 196608, + "block_attention_macs_per_token": 16384, + "candidate_token_presentations": 0, + "completed_optimizer_steps": 1000, + "enumerated_full_forward_macs": 1857781612544, + "feedback_parameter_count": 0, + "forward_affine_macs_per_token": 794752, + "forward_attention_macs_per_token": 65536, + "forward_parameter_count": 813568, + "full_forward_token_passes": 2159488, + "local_block_token_evaluations": 270336000, + "local_head_token_evaluations": 67584000, + "local_vjp_token_evaluations": 163840000, + "ordinary_training_tokens": 2048000, + "ordinary_validation_tokens": 111488, + "relaxation_token_passes": 32768000, + "training_token_presentations": 2048000 + } + }, + { + "completed_optimizer_steps": 1000, + "driver_wall_seconds": 35.40010952949524, + "experiment_name": "transformer-p1-ff-d4-lr0p0003", + "final_validation_accuracy": 0.14900258323765786, + "final_validation_nll": 10.76929515921837, + "final_validation_perplexity": 47538.49862988898, + "finite": true, + "manifest": "results/transformer_crossover/p1/transformer-p1-ff-d4-lr0p0003.json.manifest.json", + "method": "ff", + "output_sha256": "918384adde3cb51f64b54c37a66052de6735a4df9dc3007364551ec4de5706bf", + "peak_memory_allocated_bytes": 87777792, + "physical_gpu_index": 7, + "physical_gpu_uuid": "GPU-df3d5933-c3e9-9ea1-d595-9d2a3b68c14b", + "rate": 0.0003, + "status": "completed", + "total_wall_seconds": 29.88270902633667, + "work": { + "block_affine_macs_per_token": 196608, + "block_attention_macs_per_token": 16384, + "candidate_token_presentations": 7246720, + "completed_optimizer_steps": 1000, + "enumerated_full_forward_macs": 9758005903360, + "feedback_parameter_count": 0, + "forward_affine_macs_per_token": 794752, + "forward_attention_macs_per_token": 65536, + "forward_parameter_count": 813568, + "full_forward_token_passes": 11342720, + "local_block_token_evaluations": 1368064, + "local_head_token_evaluations": 337920, + "local_vjp_token_evaluations": 0, + "ordinary_training_tokens": 2048000, + "ordinary_validation_tokens": 111488, + "relaxation_token_passes": 0, + "training_token_presentations": 4096000 + } + }, + { + "completed_optimizer_steps": 1000, + "driver_wall_seconds": 39.819350242614746, + "experiment_name": "transformer-p1-pepita-d4-lr0p0003", + "final_validation_accuracy": 0.10280927095292768, + "final_validation_nll": 4.14622573326978, + "final_validation_perplexity": 63.19503470485133, + "finite": true, + "manifest": "results/transformer_crossover/p1/transformer-p1-pepita-d4-lr0p0003.json.manifest.json", + "method": "pepita", + "output_sha256": "bfa7811c0079e15203af4238fdf563fd9ba30c30a77899f19b7b1fb2e865ab9f", + "peak_memory_allocated_bytes": 74502144, + "physical_gpu_index": 5, + "physical_gpu_uuid": "GPU-f4aed756-228e-7522-94e7-fb543c02cc74", + "rate": 0.0003, + "status": "completed", + "total_wall_seconds": 34.4016318321228, + "work": { + "block_affine_macs_per_token": 196608, + "block_attention_macs_per_token": 16384, + "candidate_token_presentations": 0, + "completed_optimizer_steps": 1000, + "enumerated_full_forward_macs": 3619651436544, + "feedback_parameter_count": 8320, + "forward_affine_macs_per_token": 794752, + "forward_attention_macs_per_token": 65536, + "forward_parameter_count": 813568, + "full_forward_token_passes": 4207488, + "local_block_token_evaluations": 8192000, + "local_head_token_evaluations": 2048000, + "local_vjp_token_evaluations": 0, + "ordinary_training_tokens": 2048000, + "ordinary_validation_tokens": 111488, + "relaxation_token_passes": 0, + "training_token_presentations": 4096000 + } + }, + { + "completed_optimizer_steps": 1000, + "driver_wall_seconds": 33.60178256034851, + "experiment_name": "transformer-p1-dfa-d4-lr0p001", + "final_validation_accuracy": 0.25691554247990817, + "final_validation_nll": 2.6988882969222305, + "final_validation_perplexity": 14.863199072720752, + "finite": true, + "manifest": "results/transformer_crossover/p1/transformer-p1-dfa-d4-lr0p001.json.manifest.json", + "method": "dfa", + "output_sha256": "160ac3a0b5c19cc36841f4cb4f373d6fbd9bf3ac325aa42757632858d512531c", + "peak_memory_allocated_bytes": 66264064, + "physical_gpu_index": 7, + "physical_gpu_uuid": "GPU-df3d5933-c3e9-9ea1-d595-9d2a3b68c14b", + "rate": 0.001, + "status": "completed", + "total_wall_seconds": 29.528637886047363, + "work": { + "block_affine_macs_per_token": 196608, + "block_attention_macs_per_token": 16384, + "candidate_token_presentations": 0, + "completed_optimizer_steps": 1000, + "enumerated_full_forward_macs": 1857781612544, + "feedback_parameter_count": 49920, + "forward_affine_macs_per_token": 794752, + "forward_attention_macs_per_token": 65536, + "forward_parameter_count": 813568, + "full_forward_token_passes": 2159488, + "local_block_token_evaluations": 8192000, + "local_head_token_evaluations": 2048000, + "local_vjp_token_evaluations": 0, + "ordinary_training_tokens": 2048000, + "ordinary_validation_tokens": 111488, + "relaxation_token_passes": 0, + "training_token_presentations": 2048000 + } + }, + { + "completed_optimizer_steps": 1000, + "driver_wall_seconds": 40.06952881813049, + "experiment_name": "transformer-p1-fa-d4-lr0p001", + "final_validation_accuracy": 0.3423866245694604, + "final_validation_nll": 2.2787026888193687, + "final_validation_perplexity": 9.764005236323833, + "finite": true, + "manifest": "results/transformer_crossover/p1/transformer-p1-fa-d4-lr0p001.json.manifest.json", + "method": "fa", + "output_sha256": "2ff3bc58c0a6c7e341a00df7bd4112f267092721ea44758d5d16404e72249883", + "peak_memory_allocated_bytes": 146653184, + "physical_gpu_index": 5, + "physical_gpu_uuid": "GPU-f4aed756-228e-7522-94e7-fb543c02cc74", + "rate": 0.001, + "status": "completed", + "total_wall_seconds": 35.74962830543518, + "work": { + "block_affine_macs_per_token": 196608, + "block_attention_macs_per_token": 16384, + "candidate_token_presentations": 0, + "completed_optimizer_steps": 1000, + "enumerated_full_forward_macs": 1857781612544, + "feedback_parameter_count": 794752, + "forward_affine_macs_per_token": 794752, + "forward_attention_macs_per_token": 65536, + "forward_parameter_count": 813568, + "full_forward_token_passes": 2159488, + "local_block_token_evaluations": 0, + "local_head_token_evaluations": 0, + "local_vjp_token_evaluations": 0, + "ordinary_training_tokens": 2048000, + "ordinary_validation_tokens": 111488, + "relaxation_token_passes": 0, + "training_token_presentations": 2048000 + } + }, + { + "completed_optimizer_steps": 1000, + "driver_wall_seconds": 43.57275652885437, + "experiment_name": "transformer-p1-clean_kp-d4-lr0p001", + "final_validation_accuracy": 0.42315764925373134, + "final_validation_nll": 1.9609453122459788, + "final_validation_perplexity": 7.1060413190027285, + "finite": true, + "manifest": "results/transformer_crossover/p1/transformer-p1-clean_kp-d4-lr0p001.json.manifest.json", + "method": "clean_kp", + "output_sha256": "27dc7a7d4cd7e260d661063e765dfcd933668f85266b7460437706b8d00ad30e", + "peak_memory_allocated_bytes": 153568768, + "physical_gpu_index": 7, + "physical_gpu_uuid": "GPU-df3d5933-c3e9-9ea1-d595-9d2a3b68c14b", + "rate": 0.001, + "status": "completed", + "total_wall_seconds": 39.329062700271606, + "work": { + "block_affine_macs_per_token": 196608, + "block_attention_macs_per_token": 16384, + "candidate_token_presentations": 0, + "completed_optimizer_steps": 1000, + "enumerated_full_forward_macs": 1857781612544, + "feedback_parameter_count": 794752, + "forward_affine_macs_per_token": 794752, + "forward_attention_macs_per_token": 65536, + "forward_parameter_count": 813568, + "full_forward_token_passes": 2159488, + "local_block_token_evaluations": 0, + "local_head_token_evaluations": 0, + "local_vjp_token_evaluations": 0, + "ordinary_training_tokens": 2048000, + "ordinary_validation_tokens": 111488, + "relaxation_token_passes": 0, + "training_token_presentations": 2048000 + } + }, + { + "completed_optimizer_steps": 1000, + "driver_wall_seconds": 45.98284411430359, + "experiment_name": "transformer-p1-sdil-d4-lr0p001", + "final_validation_accuracy": 0.42315764925373134, + "final_validation_nll": 1.960945281588251, + "final_validation_perplexity": 7.106041101147652, + "finite": true, + "manifest": "results/transformer_crossover/p1/transformer-p1-sdil-d4-lr0p001.json.manifest.json", + "method": "sdil", + "output_sha256": "83e17e562e609928c8a3ad4306acf42e8906dc57df8d171eef9ecf61346cde0d", + "peak_memory_allocated_bytes": 157764096, + "physical_gpu_index": 5, + "physical_gpu_uuid": "GPU-f4aed756-228e-7522-94e7-fb543c02cc74", + "rate": 0.001, + "status": "completed", + "total_wall_seconds": 40.63735795021057, + "work": { + "block_affine_macs_per_token": 196608, + "block_attention_macs_per_token": 16384, + "candidate_token_presentations": 0, + "completed_optimizer_steps": 1000, + "enumerated_full_forward_macs": 1857781612544, + "feedback_parameter_count": 794752, + "forward_affine_macs_per_token": 794752, + "forward_attention_macs_per_token": 65536, + "forward_parameter_count": 813568, + "full_forward_token_passes": 2159488, + "local_block_token_evaluations": 0, + "local_head_token_evaluations": 0, + "local_vjp_token_evaluations": 0, + "ordinary_training_tokens": 2048000, + "ordinary_validation_tokens": 111488, + "relaxation_token_passes": 0, + "training_token_presentations": 2048000 + } + }, + { + "completed_optimizer_steps": 1000, + "driver_wall_seconds": 29.84394907951355, + "experiment_name": "transformer-p1-bp-d4-lr0p001", + "final_validation_accuracy": 0.4394015499425947, + "final_validation_nll": 1.899204445761189, + "final_validation_perplexity": 6.680577565827694, + "finite": true, + "manifest": "results/transformer_crossover/p1/transformer-p1-bp-d4-lr0p001.json.manifest.json", + "method": "bp", + "output_sha256": "e1fed8bf22615ededcb0a352462243efbfba74dd11ba17f74ec9860dc4b3a6f5", + "peak_memory_allocated_bytes": 109656064, + "physical_gpu_index": 7, + "physical_gpu_uuid": "GPU-df3d5933-c3e9-9ea1-d595-9d2a3b68c14b", + "rate": 0.001, + "status": "completed", + "total_wall_seconds": 25.80730676651001, + "work": { + "block_affine_macs_per_token": 196608, + "block_attention_macs_per_token": 16384, + "candidate_token_presentations": 0, + "completed_optimizer_steps": 1000, + "enumerated_full_forward_macs": 1857781612544, + "feedback_parameter_count": 0, + "forward_affine_macs_per_token": 794752, + "forward_attention_macs_per_token": 65536, + "forward_parameter_count": 813568, + "full_forward_token_passes": 2159488, + "local_block_token_evaluations": 0, + "local_head_token_evaluations": 0, + "local_vjp_token_evaluations": 0, + "ordinary_training_tokens": 2048000, + "ordinary_validation_tokens": 111488, + "relaxation_token_passes": 0, + "training_token_presentations": 2048000 + } + }, + { + "completed_optimizer_steps": 1000, + "driver_wall_seconds": 597.9700076580048, + "experiment_name": "transformer-p1-ep-d4-lr0p001", + "final_validation_accuracy": 0.1410555396096441, + "final_validation_nll": 4.147124764566169, + "final_validation_perplexity": 63.25187456540571, + "finite": true, + "manifest": "results/transformer_crossover/p1/transformer-p1-ep-d4-lr0p001.json.manifest.json", + "method": "ep", + "output_sha256": "9a0649f21001760c62b8390d892aae02fc1b4c14f6424f7bcd647fe8456b17fc", + "peak_memory_allocated_bytes": 88484352, + "physical_gpu_index": 5, + "physical_gpu_uuid": "GPU-f4aed756-228e-7522-94e7-fb543c02cc74", + "rate": 0.001, + "status": "completed", + "total_wall_seconds": 593.5642428398132, + "work": { + "block_affine_macs_per_token": 196608, + "block_attention_macs_per_token": 16384, + "candidate_token_presentations": 0, + "completed_optimizer_steps": 1000, + "enumerated_full_forward_macs": 3619651436544, + "feedback_parameter_count": 0, + "forward_affine_macs_per_token": 794752, + "forward_attention_macs_per_token": 65536, + "forward_parameter_count": 813568, + "full_forward_token_passes": 4207488, + "local_block_token_evaluations": 427438080, + "local_head_token_evaluations": 106859520, + "local_vjp_token_evaluations": 245760000, + "ordinary_training_tokens": 2048000, + "ordinary_validation_tokens": 111488, + "relaxation_token_passes": 51381760, + "training_token_presentations": 2048000 + } + }, + { + "completed_optimizer_steps": 1000, + "driver_wall_seconds": 391.04854702949524, + "experiment_name": "transformer-p1-dualprop-d4-lr0p001", + "final_validation_accuracy": 0.43941051951779564, + "final_validation_nll": 1.8991675294773176, + "final_validation_perplexity": 6.680330948281979, + "finite": true, + "manifest": "results/transformer_crossover/p1/transformer-p1-dualprop-d4-lr0p001.json.manifest.json", + "method": "dualprop", + "output_sha256": "38add2a558e5453ead85aed340cfea310886b273a23266820b2fb86b6a5538af", + "peak_memory_allocated_bytes": 97268224, + "physical_gpu_index": 7, + "physical_gpu_uuid": "GPU-df3d5933-c3e9-9ea1-d595-9d2a3b68c14b", + "rate": 0.001, + "status": "completed", + "total_wall_seconds": 386.51674580574036, + "work": { + "block_affine_macs_per_token": 196608, + "block_attention_macs_per_token": 16384, + "candidate_token_presentations": 0, + "completed_optimizer_steps": 1000, + "enumerated_full_forward_macs": 1857781612544, + "feedback_parameter_count": 0, + "forward_affine_macs_per_token": 794752, + "forward_attention_macs_per_token": 65536, + "forward_parameter_count": 813568, + "full_forward_token_passes": 2159488, + "local_block_token_evaluations": 270336000, + "local_head_token_evaluations": 67584000, + "local_vjp_token_evaluations": 163840000, + "ordinary_training_tokens": 2048000, + "ordinary_validation_tokens": 111488, + "relaxation_token_passes": 32768000, + "training_token_presentations": 2048000 + } + }, + { + "completed_optimizer_steps": 1000, + "driver_wall_seconds": 33.252097606658936, + "experiment_name": "transformer-p1-ff-d4-lr0p001", + "final_validation_accuracy": 0.08121950344431687, + "final_validation_nll": 10.612460314885347, + "final_validation_perplexity": 40638.05878056827, + "finite": true, + "manifest": "results/transformer_crossover/p1/transformer-p1-ff-d4-lr0p001.json.manifest.json", + "method": "ff", + "output_sha256": "b0be9d8098a3b330b4503f86520cec42d36973bfe9316702ba5dd3a20ca4bc3a", + "peak_memory_allocated_bytes": 87777792, + "physical_gpu_index": 5, + "physical_gpu_uuid": "GPU-f4aed756-228e-7522-94e7-fb543c02cc74", + "rate": 0.001, + "status": "completed", + "total_wall_seconds": 27.866358995437622, + "work": { + "block_affine_macs_per_token": 196608, + "block_attention_macs_per_token": 16384, + "candidate_token_presentations": 7246720, + "completed_optimizer_steps": 1000, + "enumerated_full_forward_macs": 9758005903360, + "feedback_parameter_count": 0, + "forward_affine_macs_per_token": 794752, + "forward_attention_macs_per_token": 65536, + "forward_parameter_count": 813568, + "full_forward_token_passes": 11342720, + "local_block_token_evaluations": 1368064, + "local_head_token_evaluations": 337920, + "local_vjp_token_evaluations": 0, + "ordinary_training_tokens": 2048000, + "ordinary_validation_tokens": 111488, + "relaxation_token_passes": 0, + "training_token_presentations": 4096000 + } + }, + { + "completed_optimizer_steps": 1000, + "driver_wall_seconds": 43.27602219581604, + "experiment_name": "transformer-p1-pepita-d4-lr0p001", + "final_validation_accuracy": 0.14852719575200918, + "final_validation_nll": 3.1760056514280124, + "final_validation_perplexity": 23.950894012013105, + "finite": true, + "manifest": "results/transformer_crossover/p1/transformer-p1-pepita-d4-lr0p001.json.manifest.json", + "method": "pepita", + "output_sha256": "1a93e4b9b359500018ee3c14131fd4b37fd79f96a047c281eba66006fe813813", + "peak_memory_allocated_bytes": 74502144, + "physical_gpu_index": 7, + "physical_gpu_uuid": "GPU-df3d5933-c3e9-9ea1-d595-9d2a3b68c14b", + "rate": 0.001, + "status": "completed", + "total_wall_seconds": 37.90927314758301, + "work": { + "block_affine_macs_per_token": 196608, + "block_attention_macs_per_token": 16384, + "candidate_token_presentations": 0, + "completed_optimizer_steps": 1000, + "enumerated_full_forward_macs": 3619651436544, + "feedback_parameter_count": 8320, + "forward_affine_macs_per_token": 794752, + "forward_attention_macs_per_token": 65536, + "forward_parameter_count": 813568, + "full_forward_token_passes": 4207488, + "local_block_token_evaluations": 8192000, + "local_head_token_evaluations": 2048000, + "local_vjp_token_evaluations": 0, + "ordinary_training_tokens": 2048000, + "ordinary_validation_tokens": 111488, + "relaxation_token_passes": 0, + "training_token_presentations": 4096000 + } + }, + { + "completed_optimizer_steps": 1000, + "driver_wall_seconds": 34.64618635177612, + "experiment_name": "transformer-p1-dfa-d4-lr0p003", + "final_validation_accuracy": 0.2621627439724455, + "final_validation_nll": 2.619194227847111, + "final_validation_perplexity": 13.724660179403989, + "finite": true, + "manifest": "results/transformer_crossover/p1/transformer-p1-dfa-d4-lr0p003.json.manifest.json", + "method": "dfa", + "output_sha256": "39dcbd351925b28853fbbd5306303d8d05faab8e0c86fa69da4ae69bc0c84d20", + "peak_memory_allocated_bytes": 66264064, + "physical_gpu_index": 5, + "physical_gpu_uuid": "GPU-f4aed756-228e-7522-94e7-fb543c02cc74", + "rate": 0.003, + "status": "completed", + "total_wall_seconds": 30.523794651031494, + "work": { + "block_affine_macs_per_token": 196608, + "block_attention_macs_per_token": 16384, + "candidate_token_presentations": 0, + "completed_optimizer_steps": 1000, + "enumerated_full_forward_macs": 1857781612544, + "feedback_parameter_count": 49920, + "forward_affine_macs_per_token": 794752, + "forward_attention_macs_per_token": 65536, + "forward_parameter_count": 813568, + "full_forward_token_passes": 2159488, + "local_block_token_evaluations": 8192000, + "local_head_token_evaluations": 2048000, + "local_vjp_token_evaluations": 0, + "ordinary_training_tokens": 2048000, + "ordinary_validation_tokens": 111488, + "relaxation_token_passes": 0, + "training_token_presentations": 2048000 + } + }, + { + "completed_optimizer_steps": 1000, + "driver_wall_seconds": 38.71434664726257, + "experiment_name": "transformer-p1-fa-d4-lr0p003", + "final_validation_accuracy": 0.3378211107921929, + "final_validation_nll": 2.2882593507471096, + "final_validation_perplexity": 9.857763829438403, + "finite": true, + "manifest": "results/transformer_crossover/p1/transformer-p1-fa-d4-lr0p003.json.manifest.json", + "method": "fa", + "output_sha256": "1837dd0289f4575ccaa1f723a500dd4d73fc60c1fbdcc2dc769b9460254a5dff", + "peak_memory_allocated_bytes": 146653184, + "physical_gpu_index": 7, + "physical_gpu_uuid": "GPU-df3d5933-c3e9-9ea1-d595-9d2a3b68c14b", + "rate": 0.003, + "status": "completed", + "total_wall_seconds": 34.66529703140259, + "work": { + "block_affine_macs_per_token": 196608, + "block_attention_macs_per_token": 16384, + "candidate_token_presentations": 0, + "completed_optimizer_steps": 1000, + "enumerated_full_forward_macs": 1857781612544, + "feedback_parameter_count": 794752, + "forward_affine_macs_per_token": 794752, + "forward_attention_macs_per_token": 65536, + "forward_parameter_count": 813568, + "full_forward_token_passes": 2159488, + "local_block_token_evaluations": 0, + "local_head_token_evaluations": 0, + "local_vjp_token_evaluations": 0, + "ordinary_training_tokens": 2048000, + "ordinary_validation_tokens": 111488, + "relaxation_token_passes": 0, + "training_token_presentations": 2048000 + } + }, + { + "completed_optimizer_steps": 1000, + "driver_wall_seconds": 49.393547773361206, + "experiment_name": "transformer-p1-clean_kp-d4-lr0p003", + "final_validation_accuracy": 0.42341776693455796, + "final_validation_nll": 1.9429430802845653, + "final_validation_perplexity": 6.97926130039316, + "finite": true, + "manifest": "results/transformer_crossover/p1/transformer-p1-clean_kp-d4-lr0p003.json.manifest.json", + "method": "clean_kp", + "output_sha256": "caf1d2c39fd2ed1d2a2f5ea6d0324969ee891472b4fab00a268a78c810c8a8e1", + "peak_memory_allocated_bytes": 153568768, + "physical_gpu_index": 5, + "physical_gpu_uuid": "GPU-f4aed756-228e-7522-94e7-fb543c02cc74", + "rate": 0.003, + "status": "completed", + "total_wall_seconds": 44.18988084793091, + "work": { + "block_affine_macs_per_token": 196608, + "block_attention_macs_per_token": 16384, + "candidate_token_presentations": 0, + "completed_optimizer_steps": 1000, + "enumerated_full_forward_macs": 1857781612544, + "feedback_parameter_count": 794752, + "forward_affine_macs_per_token": 794752, + "forward_attention_macs_per_token": 65536, + "forward_parameter_count": 813568, + "full_forward_token_passes": 2159488, + "local_block_token_evaluations": 0, + "local_head_token_evaluations": 0, + "local_vjp_token_evaluations": 0, + "ordinary_training_tokens": 2048000, + "ordinary_validation_tokens": 111488, + "relaxation_token_passes": 0, + "training_token_presentations": 2048000 + } + }, + { + "completed_optimizer_steps": 1000, + "driver_wall_seconds": 57.563411235809326, + "experiment_name": "transformer-p1-sdil-d4-lr0p003", + "final_validation_accuracy": 0.4066267221584386, + "final_validation_nll": 1.9958930294768271, + "final_validation_perplexity": 7.358771694484222, + "finite": true, + "manifest": "results/transformer_crossover/p1/transformer-p1-sdil-d4-lr0p003.json.manifest.json", + "method": "sdil", + "output_sha256": "66cb605c4129a7e29e71e7c97d031e05566ecf6050d8967e5dfbd538520ecaf9", + "peak_memory_allocated_bytes": 157764096, + "physical_gpu_index": 7, + "physical_gpu_uuid": "GPU-df3d5933-c3e9-9ea1-d595-9d2a3b68c14b", + "rate": 0.003, + "status": "completed", + "total_wall_seconds": 52.4935040473938, + "work": { + "block_affine_macs_per_token": 196608, + "block_attention_macs_per_token": 16384, + "candidate_token_presentations": 0, + "completed_optimizer_steps": 1000, + "enumerated_full_forward_macs": 1857781612544, + "feedback_parameter_count": 794752, + "forward_affine_macs_per_token": 794752, + "forward_attention_macs_per_token": 65536, + "forward_parameter_count": 813568, + "full_forward_token_passes": 2159488, + "local_block_token_evaluations": 0, + "local_head_token_evaluations": 0, + "local_vjp_token_evaluations": 0, + "ordinary_training_tokens": 2048000, + "ordinary_validation_tokens": 111488, + "relaxation_token_passes": 0, + "training_token_presentations": 2048000 + } + }, + { + "completed_optimizer_steps": 1000, + "driver_wall_seconds": 27.13742709159851, + "experiment_name": "transformer-p1-bp-d4-lr0p003", + "final_validation_accuracy": 0.40393584959816303, + "final_validation_nll": 2.0219551145551398, + "final_validation_perplexity": 7.553077639376615, + "finite": true, + "manifest": "results/transformer_crossover/p1/transformer-p1-bp-d4-lr0p003.json.manifest.json", + "method": "bp", + "output_sha256": "7d04f9496efa13daba776257680bd3676ec842fe0294779b156a3f709cb00020", + "peak_memory_allocated_bytes": 109656064, + "physical_gpu_index": 5, + "physical_gpu_uuid": "GPU-f4aed756-228e-7522-94e7-fb543c02cc74", + "rate": 0.003, + "status": "completed", + "total_wall_seconds": 22.951324939727783, + "work": { + "block_affine_macs_per_token": 196608, + "block_attention_macs_per_token": 16384, + "candidate_token_presentations": 0, + "completed_optimizer_steps": 1000, + "enumerated_full_forward_macs": 1857781612544, + "feedback_parameter_count": 0, + "forward_affine_macs_per_token": 794752, + "forward_attention_macs_per_token": 65536, + "forward_parameter_count": 813568, + "full_forward_token_passes": 2159488, + "local_block_token_evaluations": 0, + "local_head_token_evaluations": 0, + "local_vjp_token_evaluations": 0, + "ordinary_training_tokens": 2048000, + "ordinary_validation_tokens": 111488, + "relaxation_token_passes": 0, + "training_token_presentations": 2048000 + } + } + ], + "registry_sha256": "25cbf03bbf1a7eadea3894cb8a4ef577b737f38c47a3857723abe6e7e4c53201", + "selected": { + "bp": { + "experiment_name": "transformer-p1-bp-d4-lr0p001", + "final_validation_nll": 1.899204445761189, + "finite": true, + "rate": 0.001 + }, + "clean_kp": { + "experiment_name": "transformer-p1-clean_kp-d4-lr0p003", + "final_validation_nll": 1.9429430802845653, + "finite": true, + "rate": 0.003 + }, + "dfa": { + "experiment_name": "transformer-p1-dfa-d4-lr0p003", + "final_validation_nll": 2.619194227847111, + "finite": true, + "rate": 0.003 + }, + "dualprop": { + "experiment_name": "transformer-p1-dualprop-d4-lr0p001", + "final_validation_nll": 1.8991675294773176, + "finite": true, + "rate": 0.001 + }, + "ep": { + "experiment_name": "transformer-p1-ep-d4-lr0p001", + "final_validation_nll": 4.147124764566169, + "finite": true, + "rate": 0.001 + }, + "fa": { + "experiment_name": "transformer-p1-fa-d4-lr0p001", + "final_validation_nll": 2.2787026888193687, + "finite": true, + "rate": 0.001 + }, + "ff": { + "experiment_name": "transformer-p1-ff-d4-lr0p0001", + "final_validation_nll": 8.052283456760762, + "finite": true, + "rate": 0.0001 + }, + "pepita": { + "experiment_name": "transformer-p1-pepita-d4-lr0p001", + "final_validation_nll": 3.1760056514280124, + "finite": true, + "rate": 0.001 + }, + "sdil": { + "experiment_name": "transformer-p1-sdil-d4-lr0p001", + "final_validation_nll": 1.960945281588251, + "finite": true, + "rate": 0.001 + } + }, + "selection_rule": "minimum finite final validation NLL, then lower learning rate; failed/timeout records rank after finite records", + "source": { + "environment_freeze": [ + "filelock==3.29.0", + "fsspec==2026.4.0", + "Jinja2==3.1.6", + "MarkupSafe==3.0.3", + "mpmath==1.3.0", + "networkx==3.6.1", + "numpy==2.4.4", + "nvidia-cublas-cu11==11.11.3.6", + "nvidia-cuda-cupti-cu11==11.8.87", + "nvidia-cuda-nvrtc-cu11==11.8.89", + "nvidia-cuda-runtime-cu11==11.8.89", + "nvidia-cudnn-cu11==8.7.0.84", + "nvidia-cufft-cu11==10.9.0.58", + "nvidia-curand-cu11==10.3.0.86", + "nvidia-cusolver-cu11==11.4.1.48", + "nvidia-cusparse-cu11==11.7.5.86", + "nvidia-nccl-cu11==2.20.5", + "nvidia-nvtx-cu11==11.8.86", + "packaging==26.0", + "scipy==1.17.1", + "sympy==1.14.0", + "torch==2.3.1+cu118", + "triton==2.3.1", + "typing_extensions==4.15.0" + ], + "environment_freeze_sha256": "bca82e6091ef12808e890cc36a0095dd885916b1d5e0cd4a8aa2119b6ca861a5", + "git_commit": "19f3966fc2d15b1ed18862a779d91ee16ed56bfc", + "protocol_path": "/scratch/yurenh2/sdil-runs/transformer-p1/TRANSFORMER_CROSSOVER.md", + "protocol_sha256": "841a107328f4961c9fa14cbf357b0c5699ac9b3e34b8d124ccb389ab145c8a0e", + "python": "/home/yurenh2/miniconda3/envs/ep_pascal/bin/python3", + "tracked_files": { + "TRANSFORMER_CROSSOVER.md": "841a107328f4961c9fa14cbf357b0c5699ac9b3e34b8d124ccb389ab145c8a0e", + "experiments/analyze_transformer_crossover_p1.py": "76a1df0db00d36d2732b76fd71b54f3aa0057dd611186162d8b0fc442a60d294", + "experiments/transformer_crossover_grid.py": "947152fb257dd005e22aa546be8ff0f63afe80436aa5f591db4dc29f99fb5eac", + "experiments/transformer_crossover_native.py": "c45299ecfdd80bab8ededa6f8c0e1ad2275ea7a3d68f711ee7cddad29a933c5e", + "experiments/transformer_feedback_smoke.py": "a8c0ba44091765ee2e05381f36515923990fd80e0acfef5aeea58ca29ec52092", + "sdil/transformer.py": "09180e5625054c4dc16567af8939d05c4cb7cbc08f959445f25daae7c830cb34" + } + }, + "stage": "transformer_crossover_p1", + "test_policy": "none" +} |