experiment: freeze ResNet and Transformer selectors

1 files changed, 1158 insertions, 0 deletions

diff --git a/results/transformer_crossover/p1_selector.json b/results/transformer_crossover/p1_selector.json
new file mode 100644
index 0000000..0e2e076
--- /dev/null
+++ b/results/transformer_crossover/p1_selector.json
@@ -0,0 +1,1158 @@
+{
+  "gate": "pass",
+  "launch_lock": {
+    "allowed_physical_gpus": [
+      5,
+      7
+    ],
+    "num_jobs": 27,
+    "registry_sha256": "ed2d757c69e6ef5614e436e9394a77951b75cdc91a97569f2f7451b0fd7bfb5b",
+    "source": {
+      "environment_freeze": [
+        "filelock==3.29.0",
+        "fsspec==2026.4.0",
+        "Jinja2==3.1.6",
+        "MarkupSafe==3.0.3",
+        "mpmath==1.3.0",
+        "networkx==3.6.1",
+        "numpy==2.4.4",
+        "nvidia-cublas-cu11==11.11.3.6",
+        "nvidia-cuda-cupti-cu11==11.8.87",
+        "nvidia-cuda-nvrtc-cu11==11.8.89",
+        "nvidia-cuda-runtime-cu11==11.8.89",
+        "nvidia-cudnn-cu11==8.7.0.84",
+        "nvidia-cufft-cu11==10.9.0.58",
+        "nvidia-curand-cu11==10.3.0.86",
+        "nvidia-cusolver-cu11==11.4.1.48",
+        "nvidia-cusparse-cu11==11.7.5.86",
+        "nvidia-nccl-cu11==2.20.5",
+        "nvidia-nvtx-cu11==11.8.86",
+        "packaging==26.0",
+        "scipy==1.17.1",
+        "sympy==1.14.0",
+        "torch==2.3.1+cu118",
+        "triton==2.3.1",
+        "typing_extensions==4.15.0"
+      ],
+      "environment_freeze_sha256": "bca82e6091ef12808e890cc36a0095dd885916b1d5e0cd4a8aa2119b6ca861a5",
+      "git_commit": "19f3966fc2d15b1ed18862a779d91ee16ed56bfc",
+      "protocol_path": "/scratch/yurenh2/sdil-runs/transformer-p1/TRANSFORMER_CROSSOVER.md",
+      "protocol_sha256": "841a107328f4961c9fa14cbf357b0c5699ac9b3e34b8d124ccb389ab145c8a0e",
+      "python": "/home/yurenh2/miniconda3/envs/ep_pascal/bin/python3",
+      "tracked_files": {
+        "TRANSFORMER_CROSSOVER.md": "841a107328f4961c9fa14cbf357b0c5699ac9b3e34b8d124ccb389ab145c8a0e",
+        "experiments/analyze_transformer_crossover_p1.py": "76a1df0db00d36d2732b76fd71b54f3aa0057dd611186162d8b0fc442a60d294",
+        "experiments/transformer_crossover_grid.py": "947152fb257dd005e22aa546be8ff0f63afe80436aa5f591db4dc29f99fb5eac",
+        "experiments/transformer_crossover_native.py": "c45299ecfdd80bab8ededa6f8c0e1ad2275ea7a3d68f711ee7cddad29a933c5e",
+        "experiments/transformer_feedback_smoke.py": "a8c0ba44091765ee2e05381f36515923990fd80e0acfef5aeea58ca29ec52092",
+        "sdil/transformer.py": "09180e5625054c4dc16567af8939d05c4cb7cbc08f959445f25daae7c830cb34"
+      }
+    },
+    "stage": "p1"
+  },
+  "missing_experiments": [],
+  "num_audited_records": 27,
+  "num_expected_records": 27,
+  "records": [
+    {
+      "completed_optimizer_steps": 1000,
+      "driver_wall_seconds": 622.442409992218,
+      "experiment_name": "transformer-p1-ep-d4-lr0p0001",
+      "final_validation_accuracy": 0.08571326061997704,
+      "final_validation_nll": 4.152099359733502,
+      "final_validation_perplexity": 63.56731096864925,
+      "finite": true,
+      "manifest": "results/transformer_crossover/p1/transformer-p1-ep-d4-lr0p0001.json.manifest.json",
+      "method": "ep",
+      "output_sha256": "82bb6da5cc9abe6707c62559be34b78d9c90bc40beb8f5039e8ac39dad9c6dde",
+      "peak_memory_allocated_bytes": 88484352,
+      "physical_gpu_index": 5,
+      "physical_gpu_uuid": "GPU-f4aed756-228e-7522-94e7-fb543c02cc74",
+      "rate": 0.0001,
+      "status": "completed",
+      "total_wall_seconds": 616.3357925415039,
+      "work": {
+        "block_affine_macs_per_token": 196608,
+        "block_attention_macs_per_token": 16384,
+        "candidate_token_presentations": 0,
+        "completed_optimizer_steps": 1000,
+        "enumerated_full_forward_macs": 3619651436544,
+        "feedback_parameter_count": 0,
+        "forward_affine_macs_per_token": 794752,
+        "forward_attention_macs_per_token": 65536,
+        "forward_parameter_count": 813568,
+        "full_forward_token_passes": 4207488,
+        "local_block_token_evaluations": 427438080,
+        "local_head_token_evaluations": 106859520,
+        "local_vjp_token_evaluations": 245760000,
+        "ordinary_training_tokens": 2048000,
+        "ordinary_validation_tokens": 111488,
+        "relaxation_token_passes": 51381760,
+        "training_token_presentations": 2048000
+      }
+    },
+    {
+      "completed_optimizer_steps": 1000,
+      "driver_wall_seconds": 382.97613072395325,
+      "experiment_name": "transformer-p1-dualprop-d4-lr0p0001",
+      "final_validation_accuracy": 0.3468714121699196,
+      "final_validation_nll": 2.233302180994255,
+      "final_validation_perplexity": 9.33062668108521,
+      "finite": true,
+      "manifest": "results/transformer_crossover/p1/transformer-p1-dualprop-d4-lr0p0001.json.manifest.json",
+      "method": "dualprop",
+      "output_sha256": "26e104e226d741df34d4c6f78e13ff24880825bb346c37572133a50b1c52625d",
+      "peak_memory_allocated_bytes": 97268224,
+      "physical_gpu_index": 7,
+      "physical_gpu_uuid": "GPU-df3d5933-c3e9-9ea1-d595-9d2a3b68c14b",
+      "rate": 0.0001,
+      "status": "completed",
+      "total_wall_seconds": 378.6061809062958,
+      "work": {
+        "block_affine_macs_per_token": 196608,
+        "block_attention_macs_per_token": 16384,
+        "candidate_token_presentations": 0,
+        "completed_optimizer_steps": 1000,
+        "enumerated_full_forward_macs": 1857781612544,
+        "feedback_parameter_count": 0,
+        "forward_affine_macs_per_token": 794752,
+        "forward_attention_macs_per_token": 65536,
+        "forward_parameter_count": 813568,
+        "full_forward_token_passes": 2159488,
+        "local_block_token_evaluations": 270336000,
+        "local_head_token_evaluations": 67584000,
+        "local_vjp_token_evaluations": 163840000,
+        "ordinary_training_tokens": 2048000,
+        "ordinary_validation_tokens": 111488,
+        "relaxation_token_passes": 32768000,
+        "training_token_presentations": 2048000
+      }
+    },
+    {
+      "completed_optimizer_steps": 1000,
+      "driver_wall_seconds": 34.00058722496033,
+      "experiment_name": "transformer-p1-ff-d4-lr0p0001",
+      "final_validation_accuracy": 0.14900258323765786,
+      "final_validation_nll": 8.052283456760762,
+      "final_validation_perplexity": 3140.959032901827,
+      "finite": true,
+      "manifest": "results/transformer_crossover/p1/transformer-p1-ff-d4-lr0p0001.json.manifest.json",
+      "method": "ff",
+      "output_sha256": "cb314c837f4fe047781bc38cbf45222cb93b81717eded487460abbbc32af38a0",
+      "peak_memory_allocated_bytes": 87777792,
+      "physical_gpu_index": 5,
+      "physical_gpu_uuid": "GPU-f4aed756-228e-7522-94e7-fb543c02cc74",
+      "rate": 0.0001,
+      "status": "completed",
+      "total_wall_seconds": 28.628129959106445,
+      "work": {
+        "block_affine_macs_per_token": 196608,
+        "block_attention_macs_per_token": 16384,
+        "candidate_token_presentations": 7246720,
+        "completed_optimizer_steps": 1000,
+        "enumerated_full_forward_macs": 9758005903360,
+        "feedback_parameter_count": 0,
+        "forward_affine_macs_per_token": 794752,
+        "forward_attention_macs_per_token": 65536,
+        "forward_parameter_count": 813568,
+        "full_forward_token_passes": 11342720,
+        "local_block_token_evaluations": 1368064,
+        "local_head_token_evaluations": 337920,
+        "local_vjp_token_evaluations": 0,
+        "ordinary_training_tokens": 2048000,
+        "ordinary_validation_tokens": 111488,
+        "relaxation_token_passes": 0,
+        "training_token_presentations": 4096000
+      }
+    },
+    {
+      "completed_optimizer_steps": 1000,
+      "driver_wall_seconds": 39.868736267089844,
+      "experiment_name": "transformer-p1-pepita-d4-lr0p0001",
+      "final_validation_accuracy": 0.16012485648679678,
+      "final_validation_nll": 4.210206946056554,
+      "final_validation_perplexity": 67.3704804228411,
+      "finite": true,
+      "manifest": "results/transformer_crossover/p1/transformer-p1-pepita-d4-lr0p0001.json.manifest.json",
+      "method": "pepita",
+      "output_sha256": "9047420c0d7af0d50bf7309fb326a955dd259bd2820f7b9b2f83fac7ad671f9e",
+      "peak_memory_allocated_bytes": 74502144,
+      "physical_gpu_index": 7,
+      "physical_gpu_uuid": "GPU-df3d5933-c3e9-9ea1-d595-9d2a3b68c14b",
+      "rate": 0.0001,
+      "status": "completed",
+      "total_wall_seconds": 34.56653308868408,
+      "work": {
+        "block_affine_macs_per_token": 196608,
+        "block_attention_macs_per_token": 16384,
+        "candidate_token_presentations": 0,
+        "completed_optimizer_steps": 1000,
+        "enumerated_full_forward_macs": 3619651436544,
+        "feedback_parameter_count": 8320,
+        "forward_affine_macs_per_token": 794752,
+        "forward_attention_macs_per_token": 65536,
+        "forward_parameter_count": 813568,
+        "full_forward_token_passes": 4207488,
+        "local_block_token_evaluations": 8192000,
+        "local_head_token_evaluations": 2048000,
+        "local_vjp_token_evaluations": 0,
+        "ordinary_training_tokens": 2048000,
+        "ordinary_validation_tokens": 111488,
+        "relaxation_token_passes": 0,
+        "training_token_presentations": 4096000
+      }
+    },
+    {
+      "completed_optimizer_steps": 1000,
+      "driver_wall_seconds": 34.50551080703735,
+      "experiment_name": "transformer-p1-dfa-d4-lr0p0003",
+      "final_validation_accuracy": 0.2341238518943743,
+      "final_validation_nll": 2.8146539484455446,
+      "final_validation_perplexity": 16.687400080306357,
+      "finite": true,
+      "manifest": "results/transformer_crossover/p1/transformer-p1-dfa-d4-lr0p0003.json.manifest.json",
+      "method": "dfa",
+      "output_sha256": "50d89a4d090da1127556dea87b9e6208c29231692286d0f4274f1b0277a32c6a",
+      "peak_memory_allocated_bytes": 66264064,
+      "physical_gpu_index": 5,
+      "physical_gpu_uuid": "GPU-f4aed756-228e-7522-94e7-fb543c02cc74",
+      "rate": 0.0003,
+      "status": "completed",
+      "total_wall_seconds": 30.448689222335815,
+      "work": {
+        "block_affine_macs_per_token": 196608,
+        "block_attention_macs_per_token": 16384,
+        "candidate_token_presentations": 0,
+        "completed_optimizer_steps": 1000,
+        "enumerated_full_forward_macs": 1857781612544,
+        "feedback_parameter_count": 49920,
+        "forward_affine_macs_per_token": 794752,
+        "forward_attention_macs_per_token": 65536,
+        "forward_parameter_count": 813568,
+        "full_forward_token_passes": 2159488,
+        "local_block_token_evaluations": 8192000,
+        "local_head_token_evaluations": 2048000,
+        "local_vjp_token_evaluations": 0,
+        "ordinary_training_tokens": 2048000,
+        "ordinary_validation_tokens": 111488,
+        "relaxation_token_passes": 0,
+        "training_token_presentations": 2048000
+      }
+    },
+    {
+      "completed_optimizer_steps": 1000,
+      "driver_wall_seconds": 43.72804021835327,
+      "experiment_name": "transformer-p1-fa-d4-lr0p0003",
+      "final_validation_accuracy": 0.31162098163031,
+      "final_validation_nll": 2.3651696017908037,
+      "final_validation_perplexity": 10.645844218650149,
+      "finite": true,
+      "manifest": "results/transformer_crossover/p1/transformer-p1-fa-d4-lr0p0003.json.manifest.json",
+      "method": "fa",
+      "output_sha256": "9d1fb33d21e0a4a78f7a3f4804e83134f5813bc02f72948ce168b7c8b09a8710",
+      "peak_memory_allocated_bytes": 146653184,
+      "physical_gpu_index": 7,
+      "physical_gpu_uuid": "GPU-df3d5933-c3e9-9ea1-d595-9d2a3b68c14b",
+      "rate": 0.0003,
+      "status": "completed",
+      "total_wall_seconds": 39.57168173789978,
+      "work": {
+        "block_affine_macs_per_token": 196608,
+        "block_attention_macs_per_token": 16384,
+        "candidate_token_presentations": 0,
+        "completed_optimizer_steps": 1000,
+        "enumerated_full_forward_macs": 1857781612544,
+        "feedback_parameter_count": 794752,
+        "forward_affine_macs_per_token": 794752,
+        "forward_attention_macs_per_token": 65536,
+        "forward_parameter_count": 813568,
+        "full_forward_token_passes": 2159488,
+        "local_block_token_evaluations": 0,
+        "local_head_token_evaluations": 0,
+        "local_vjp_token_evaluations": 0,
+        "ordinary_training_tokens": 2048000,
+        "ordinary_validation_tokens": 111488,
+        "relaxation_token_passes": 0,
+        "training_token_presentations": 2048000
+      }
+    },
+    {
+      "completed_optimizer_steps": 1000,
+      "driver_wall_seconds": 43.616663694381714,
+      "experiment_name": "transformer-p1-clean_kp-d4-lr0p0003",
+      "final_validation_accuracy": 0.3621017508610792,
+      "final_validation_nll": 2.1674982721304645,
+      "final_validation_perplexity": 8.736400582109368,
+      "finite": true,
+      "manifest": "results/transformer_crossover/p1/transformer-p1-clean_kp-d4-lr0p0003.json.manifest.json",
+      "method": "clean_kp",
+      "output_sha256": "ce7553d786874d54a90be4637b3f1ee24ddb179ffac553bf70c46d8f09959f46",
+      "peak_memory_allocated_bytes": 153568768,
+      "physical_gpu_index": 5,
+      "physical_gpu_uuid": "GPU-f4aed756-228e-7522-94e7-fb543c02cc74",
+      "rate": 0.0003,
+      "status": "completed",
+      "total_wall_seconds": 38.4737503528595,
+      "work": {
+        "block_affine_macs_per_token": 196608,
+        "block_attention_macs_per_token": 16384,
+        "candidate_token_presentations": 0,
+        "completed_optimizer_steps": 1000,
+        "enumerated_full_forward_macs": 1857781612544,
+        "feedback_parameter_count": 794752,
+        "forward_affine_macs_per_token": 794752,
+        "forward_attention_macs_per_token": 65536,
+        "forward_parameter_count": 813568,
+        "full_forward_token_passes": 2159488,
+        "local_block_token_evaluations": 0,
+        "local_head_token_evaluations": 0,
+        "local_vjp_token_evaluations": 0,
+        "ordinary_training_tokens": 2048000,
+        "ordinary_validation_tokens": 111488,
+        "relaxation_token_passes": 0,
+        "training_token_presentations": 2048000
+      }
+    },
+    {
+      "completed_optimizer_steps": 1000,
+      "driver_wall_seconds": 49.88345217704773,
+      "experiment_name": "transformer-p1-sdil-d4-lr0p0003",
+      "final_validation_accuracy": 0.3621017508610792,
+      "final_validation_nll": 2.1674982283337108,
+      "final_validation_perplexity": 8.736400199483393,
+      "finite": true,
+      "manifest": "results/transformer_crossover/p1/transformer-p1-sdil-d4-lr0p0003.json.manifest.json",
+      "method": "sdil",
+      "output_sha256": "932ac4dced706ae75f44a852c56992c7135d6afc954acee1b8af31507b350f50",
+      "peak_memory_allocated_bytes": 157764096,
+      "physical_gpu_index": 7,
+      "physical_gpu_uuid": "GPU-df3d5933-c3e9-9ea1-d595-9d2a3b68c14b",
+      "rate": 0.0003,
+      "status": "completed",
+      "total_wall_seconds": 44.673834562301636,
+      "work": {
+        "block_affine_macs_per_token": 196608,
+        "block_attention_macs_per_token": 16384,
+        "candidate_token_presentations": 0,
+        "completed_optimizer_steps": 1000,
+        "enumerated_full_forward_macs": 1857781612544,
+        "feedback_parameter_count": 794752,
+        "forward_affine_macs_per_token": 794752,
+        "forward_attention_macs_per_token": 65536,
+        "forward_parameter_count": 813568,
+        "full_forward_token_passes": 2159488,
+        "local_block_token_evaluations": 0,
+        "local_head_token_evaluations": 0,
+        "local_vjp_token_evaluations": 0,
+        "ordinary_training_tokens": 2048000,
+        "ordinary_validation_tokens": 111488,
+        "relaxation_token_passes": 0,
+        "training_token_presentations": 2048000
+      }
+    },
+    {
+      "completed_optimizer_steps": 1000,
+      "driver_wall_seconds": 25.821476459503174,
+      "experiment_name": "transformer-p1-bp-d4-lr0p0003",
+      "final_validation_accuracy": 0.3950470005740528,
+      "final_validation_nll": 2.048308381256909,
+      "final_validation_perplexity": 7.754771887148325,
+      "finite": true,
+      "manifest": "results/transformer_crossover/p1/transformer-p1-bp-d4-lr0p0003.json.manifest.json",
+      "method": "bp",
+      "output_sha256": "e7d74c34973507feb23ab114108c8ef3f0883e5785a171fc86f4c33db0b06cc2",
+      "peak_memory_allocated_bytes": 109656064,
+      "physical_gpu_index": 5,
+      "physical_gpu_uuid": "GPU-f4aed756-228e-7522-94e7-fb543c02cc74",
+      "rate": 0.0003,
+      "status": "completed",
+      "total_wall_seconds": 21.73075294494629,
+      "work": {
+        "block_affine_macs_per_token": 196608,
+        "block_attention_macs_per_token": 16384,
+        "candidate_token_presentations": 0,
+        "completed_optimizer_steps": 1000,
+        "enumerated_full_forward_macs": 1857781612544,
+        "feedback_parameter_count": 0,
+        "forward_affine_macs_per_token": 794752,
+        "forward_attention_macs_per_token": 65536,
+        "forward_parameter_count": 813568,
+        "full_forward_token_passes": 2159488,
+        "local_block_token_evaluations": 0,
+        "local_head_token_evaluations": 0,
+        "local_vjp_token_evaluations": 0,
+        "ordinary_training_tokens": 2048000,
+        "ordinary_validation_tokens": 111488,
+        "relaxation_token_passes": 0,
+        "training_token_presentations": 2048000
+      }
+    },
+    {
+      "completed_optimizer_steps": 1000,
+      "driver_wall_seconds": 584.4020130634308,
+      "experiment_name": "transformer-p1-ep-d4-lr0p0003",
+      "final_validation_accuracy": 0.05100997416762342,
+      "final_validation_nll": 4.157737329003457,
+      "final_validation_perplexity": 63.926713713260646,
+      "finite": true,
+      "manifest": "results/transformer_crossover/p1/transformer-p1-ep-d4-lr0p0003.json.manifest.json",
+      "method": "ep",
+      "output_sha256": "a8c78c833c7db8823685bf70e3b3303acd9945da01ce9a6733421e8229dc9390",
+      "peak_memory_allocated_bytes": 88484352,
+      "physical_gpu_index": 7,
+      "physical_gpu_uuid": "GPU-df3d5933-c3e9-9ea1-d595-9d2a3b68c14b",
+      "rate": 0.0003,
+      "status": "completed",
+      "total_wall_seconds": 580.2041375637054,
+      "work": {
+        "block_affine_macs_per_token": 196608,
+        "block_attention_macs_per_token": 16384,
+        "candidate_token_presentations": 0,
+        "completed_optimizer_steps": 1000,
+        "enumerated_full_forward_macs": 3619651436544,
+        "feedback_parameter_count": 0,
+        "forward_affine_macs_per_token": 794752,
+        "forward_attention_macs_per_token": 65536,
+        "forward_parameter_count": 813568,
+        "full_forward_token_passes": 4207488,
+        "local_block_token_evaluations": 427438080,
+        "local_head_token_evaluations": 106859520,
+        "local_vjp_token_evaluations": 245760000,
+        "ordinary_training_tokens": 2048000,
+        "ordinary_validation_tokens": 111488,
+        "relaxation_token_passes": 51381760,
+        "training_token_presentations": 2048000
+      }
+    },
+    {
+      "completed_optimizer_steps": 1000,
+      "driver_wall_seconds": 403.95608258247375,
+      "experiment_name": "transformer-p1-dualprop-d4-lr0p0003",
+      "final_validation_accuracy": 0.3950470005740528,
+      "final_validation_nll": 2.0483097104883905,
+      "final_validation_perplexity": 7.7547821950420985,
+      "finite": true,
+      "manifest": "results/transformer_crossover/p1/transformer-p1-dualprop-d4-lr0p0003.json.manifest.json",
+      "method": "dualprop",
+      "output_sha256": "3d97a86f64e7c3485412ac0c737d7bd4b38f9d368d9fad4b7036ea0727f0f944",
+      "peak_memory_allocated_bytes": 97268224,
+      "physical_gpu_index": 5,
+      "physical_gpu_uuid": "GPU-f4aed756-228e-7522-94e7-fb543c02cc74",
+      "rate": 0.0003,
+      "status": "completed",
+      "total_wall_seconds": 398.5918562412262,
+      "work": {
+        "block_affine_macs_per_token": 196608,
+        "block_attention_macs_per_token": 16384,
+        "candidate_token_presentations": 0,
+        "completed_optimizer_steps": 1000,
+        "enumerated_full_forward_macs": 1857781612544,
+        "feedback_parameter_count": 0,
+        "forward_affine_macs_per_token": 794752,
+        "forward_attention_macs_per_token": 65536,
+        "forward_parameter_count": 813568,
+        "full_forward_token_passes": 2159488,
+        "local_block_token_evaluations": 270336000,
+        "local_head_token_evaluations": 67584000,
+        "local_vjp_token_evaluations": 163840000,
+        "ordinary_training_tokens": 2048000,
+        "ordinary_validation_tokens": 111488,
+        "relaxation_token_passes": 32768000,
+        "training_token_presentations": 2048000
+      }
+    },
+    {
+      "completed_optimizer_steps": 1000,
+      "driver_wall_seconds": 35.40010952949524,
+      "experiment_name": "transformer-p1-ff-d4-lr0p0003",
+      "final_validation_accuracy": 0.14900258323765786,
+      "final_validation_nll": 10.76929515921837,
+      "final_validation_perplexity": 47538.49862988898,
+      "finite": true,
+      "manifest": "results/transformer_crossover/p1/transformer-p1-ff-d4-lr0p0003.json.manifest.json",
+      "method": "ff",
+      "output_sha256": "918384adde3cb51f64b54c37a66052de6735a4df9dc3007364551ec4de5706bf",
+      "peak_memory_allocated_bytes": 87777792,
+      "physical_gpu_index": 7,
+      "physical_gpu_uuid": "GPU-df3d5933-c3e9-9ea1-d595-9d2a3b68c14b",
+      "rate": 0.0003,
+      "status": "completed",
+      "total_wall_seconds": 29.88270902633667,
+      "work": {
+        "block_affine_macs_per_token": 196608,
+        "block_attention_macs_per_token": 16384,
+        "candidate_token_presentations": 7246720,
+        "completed_optimizer_steps": 1000,
+        "enumerated_full_forward_macs": 9758005903360,
+        "feedback_parameter_count": 0,
+        "forward_affine_macs_per_token": 794752,
+        "forward_attention_macs_per_token": 65536,
+        "forward_parameter_count": 813568,
+        "full_forward_token_passes": 11342720,
+        "local_block_token_evaluations": 1368064,
+        "local_head_token_evaluations": 337920,
+        "local_vjp_token_evaluations": 0,
+        "ordinary_training_tokens": 2048000,
+        "ordinary_validation_tokens": 111488,
+        "relaxation_token_passes": 0,
+        "training_token_presentations": 4096000
+      }
+    },
+    {
+      "completed_optimizer_steps": 1000,
+      "driver_wall_seconds": 39.819350242614746,
+      "experiment_name": "transformer-p1-pepita-d4-lr0p0003",
+      "final_validation_accuracy": 0.10280927095292768,
+      "final_validation_nll": 4.14622573326978,
+      "final_validation_perplexity": 63.19503470485133,
+      "finite": true,
+      "manifest": "results/transformer_crossover/p1/transformer-p1-pepita-d4-lr0p0003.json.manifest.json",
+      "method": "pepita",
+      "output_sha256": "bfa7811c0079e15203af4238fdf563fd9ba30c30a77899f19b7b1fb2e865ab9f",
+      "peak_memory_allocated_bytes": 74502144,
+      "physical_gpu_index": 5,
+      "physical_gpu_uuid": "GPU-f4aed756-228e-7522-94e7-fb543c02cc74",
+      "rate": 0.0003,
+      "status": "completed",
+      "total_wall_seconds": 34.4016318321228,
+      "work": {
+        "block_affine_macs_per_token": 196608,
+        "block_attention_macs_per_token": 16384,
+        "candidate_token_presentations": 0,
+        "completed_optimizer_steps": 1000,
+        "enumerated_full_forward_macs": 3619651436544,
+        "feedback_parameter_count": 8320,
+        "forward_affine_macs_per_token": 794752,
+        "forward_attention_macs_per_token": 65536,
+        "forward_parameter_count": 813568,
+        "full_forward_token_passes": 4207488,
+        "local_block_token_evaluations": 8192000,
+        "local_head_token_evaluations": 2048000,
+        "local_vjp_token_evaluations": 0,
+        "ordinary_training_tokens": 2048000,
+        "ordinary_validation_tokens": 111488,
+        "relaxation_token_passes": 0,
+        "training_token_presentations": 4096000
+      }
+    },
+    {
+      "completed_optimizer_steps": 1000,
+      "driver_wall_seconds": 33.60178256034851,
+      "experiment_name": "transformer-p1-dfa-d4-lr0p001",
+      "final_validation_accuracy": 0.25691554247990817,
+      "final_validation_nll": 2.6988882969222305,
+      "final_validation_perplexity": 14.863199072720752,
+      "finite": true,
+      "manifest": "results/transformer_crossover/p1/transformer-p1-dfa-d4-lr0p001.json.manifest.json",
+      "method": "dfa",
+      "output_sha256": "160ac3a0b5c19cc36841f4cb4f373d6fbd9bf3ac325aa42757632858d512531c",
+      "peak_memory_allocated_bytes": 66264064,
+      "physical_gpu_index": 7,
+      "physical_gpu_uuid": "GPU-df3d5933-c3e9-9ea1-d595-9d2a3b68c14b",
+      "rate": 0.001,
+      "status": "completed",
+      "total_wall_seconds": 29.528637886047363,
+      "work": {
+        "block_affine_macs_per_token": 196608,
+        "block_attention_macs_per_token": 16384,
+        "candidate_token_presentations": 0,
+        "completed_optimizer_steps": 1000,
+        "enumerated_full_forward_macs": 1857781612544,
+        "feedback_parameter_count": 49920,
+        "forward_affine_macs_per_token": 794752,
+        "forward_attention_macs_per_token": 65536,
+        "forward_parameter_count": 813568,
+        "full_forward_token_passes": 2159488,
+        "local_block_token_evaluations": 8192000,
+        "local_head_token_evaluations": 2048000,
+        "local_vjp_token_evaluations": 0,
+        "ordinary_training_tokens": 2048000,
+        "ordinary_validation_tokens": 111488,
+        "relaxation_token_passes": 0,
+        "training_token_presentations": 2048000
+      }
+    },
+    {
+      "completed_optimizer_steps": 1000,
+      "driver_wall_seconds": 40.06952881813049,
+      "experiment_name": "transformer-p1-fa-d4-lr0p001",
+      "final_validation_accuracy": 0.3423866245694604,
+      "final_validation_nll": 2.2787026888193687,
+      "final_validation_perplexity": 9.764005236323833,
+      "finite": true,
+      "manifest": "results/transformer_crossover/p1/transformer-p1-fa-d4-lr0p001.json.manifest.json",
+      "method": "fa",
+      "output_sha256": "2ff3bc58c0a6c7e341a00df7bd4112f267092721ea44758d5d16404e72249883",
+      "peak_memory_allocated_bytes": 146653184,
+      "physical_gpu_index": 5,
+      "physical_gpu_uuid": "GPU-f4aed756-228e-7522-94e7-fb543c02cc74",
+      "rate": 0.001,
+      "status": "completed",
+      "total_wall_seconds": 35.74962830543518,
+      "work": {
+        "block_affine_macs_per_token": 196608,
+        "block_attention_macs_per_token": 16384,
+        "candidate_token_presentations": 0,
+        "completed_optimizer_steps": 1000,
+        "enumerated_full_forward_macs": 1857781612544,
+        "feedback_parameter_count": 794752,
+        "forward_affine_macs_per_token": 794752,
+        "forward_attention_macs_per_token": 65536,
+        "forward_parameter_count": 813568,
+        "full_forward_token_passes": 2159488,
+        "local_block_token_evaluations": 0,
+        "local_head_token_evaluations": 0,
+        "local_vjp_token_evaluations": 0,
+        "ordinary_training_tokens": 2048000,
+        "ordinary_validation_tokens": 111488,
+        "relaxation_token_passes": 0,
+        "training_token_presentations": 2048000
+      }
+    },
+    {
+      "completed_optimizer_steps": 1000,
+      "driver_wall_seconds": 43.57275652885437,
+      "experiment_name": "transformer-p1-clean_kp-d4-lr0p001",
+      "final_validation_accuracy": 0.42315764925373134,
+      "final_validation_nll": 1.9609453122459788,
+      "final_validation_perplexity": 7.1060413190027285,
+      "finite": true,
+      "manifest": "results/transformer_crossover/p1/transformer-p1-clean_kp-d4-lr0p001.json.manifest.json",
+      "method": "clean_kp",
+      "output_sha256": "27dc7a7d4cd7e260d661063e765dfcd933668f85266b7460437706b8d00ad30e",
+      "peak_memory_allocated_bytes": 153568768,
+      "physical_gpu_index": 7,
+      "physical_gpu_uuid": "GPU-df3d5933-c3e9-9ea1-d595-9d2a3b68c14b",
+      "rate": 0.001,
+      "status": "completed",
+      "total_wall_seconds": 39.329062700271606,
+      "work": {
+        "block_affine_macs_per_token": 196608,
+        "block_attention_macs_per_token": 16384,
+        "candidate_token_presentations": 0,
+        "completed_optimizer_steps": 1000,
+        "enumerated_full_forward_macs": 1857781612544,
+        "feedback_parameter_count": 794752,
+        "forward_affine_macs_per_token": 794752,
+        "forward_attention_macs_per_token": 65536,
+        "forward_parameter_count": 813568,
+        "full_forward_token_passes": 2159488,
+        "local_block_token_evaluations": 0,
+        "local_head_token_evaluations": 0,
+        "local_vjp_token_evaluations": 0,
+        "ordinary_training_tokens": 2048000,
+        "ordinary_validation_tokens": 111488,
+        "relaxation_token_passes": 0,
+        "training_token_presentations": 2048000
+      }
+    },
+    {
+      "completed_optimizer_steps": 1000,
+      "driver_wall_seconds": 45.98284411430359,
+      "experiment_name": "transformer-p1-sdil-d4-lr0p001",
+      "final_validation_accuracy": 0.42315764925373134,
+      "final_validation_nll": 1.960945281588251,
+      "final_validation_perplexity": 7.106041101147652,
+      "finite": true,
+      "manifest": "results/transformer_crossover/p1/transformer-p1-sdil-d4-lr0p001.json.manifest.json",
+      "method": "sdil",
+      "output_sha256": "83e17e562e609928c8a3ad4306acf42e8906dc57df8d171eef9ecf61346cde0d",
+      "peak_memory_allocated_bytes": 157764096,
+      "physical_gpu_index": 5,
+      "physical_gpu_uuid": "GPU-f4aed756-228e-7522-94e7-fb543c02cc74",
+      "rate": 0.001,
+      "status": "completed",
+      "total_wall_seconds": 40.63735795021057,
+      "work": {
+        "block_affine_macs_per_token": 196608,
+        "block_attention_macs_per_token": 16384,
+        "candidate_token_presentations": 0,
+        "completed_optimizer_steps": 1000,
+        "enumerated_full_forward_macs": 1857781612544,
+        "feedback_parameter_count": 794752,
+        "forward_affine_macs_per_token": 794752,
+        "forward_attention_macs_per_token": 65536,
+        "forward_parameter_count": 813568,
+        "full_forward_token_passes": 2159488,
+        "local_block_token_evaluations": 0,
+        "local_head_token_evaluations": 0,
+        "local_vjp_token_evaluations": 0,
+        "ordinary_training_tokens": 2048000,
+        "ordinary_validation_tokens": 111488,
+        "relaxation_token_passes": 0,
+        "training_token_presentations": 2048000
+      }
+    },
+    {
+      "completed_optimizer_steps": 1000,
+      "driver_wall_seconds": 29.84394907951355,
+      "experiment_name": "transformer-p1-bp-d4-lr0p001",
+      "final_validation_accuracy": 0.4394015499425947,
+      "final_validation_nll": 1.899204445761189,
+      "final_validation_perplexity": 6.680577565827694,
+      "finite": true,
+      "manifest": "results/transformer_crossover/p1/transformer-p1-bp-d4-lr0p001.json.manifest.json",
+      "method": "bp",
+      "output_sha256": "e1fed8bf22615ededcb0a352462243efbfba74dd11ba17f74ec9860dc4b3a6f5",
+      "peak_memory_allocated_bytes": 109656064,
+      "physical_gpu_index": 7,
+      "physical_gpu_uuid": "GPU-df3d5933-c3e9-9ea1-d595-9d2a3b68c14b",
+      "rate": 0.001,
+      "status": "completed",
+      "total_wall_seconds": 25.80730676651001,
+      "work": {
+        "block_affine_macs_per_token": 196608,
+        "block_attention_macs_per_token": 16384,
+        "candidate_token_presentations": 0,
+        "completed_optimizer_steps": 1000,
+        "enumerated_full_forward_macs": 1857781612544,
+        "feedback_parameter_count": 0,
+        "forward_affine_macs_per_token": 794752,
+        "forward_attention_macs_per_token": 65536,
+        "forward_parameter_count": 813568,
+        "full_forward_token_passes": 2159488,
+        "local_block_token_evaluations": 0,
+        "local_head_token_evaluations": 0,
+        "local_vjp_token_evaluations": 0,
+        "ordinary_training_tokens": 2048000,
+        "ordinary_validation_tokens": 111488,
+        "relaxation_token_passes": 0,
+        "training_token_presentations": 2048000
+      }
+    },
+    {
+      "completed_optimizer_steps": 1000,
+      "driver_wall_seconds": 597.9700076580048,
+      "experiment_name": "transformer-p1-ep-d4-lr0p001",
+      "final_validation_accuracy": 0.1410555396096441,
+      "final_validation_nll": 4.147124764566169,
+      "final_validation_perplexity": 63.25187456540571,
+      "finite": true,
+      "manifest": "results/transformer_crossover/p1/transformer-p1-ep-d4-lr0p001.json.manifest.json",
+      "method": "ep",
+      "output_sha256": "9a0649f21001760c62b8390d892aae02fc1b4c14f6424f7bcd647fe8456b17fc",
+      "peak_memory_allocated_bytes": 88484352,
+      "physical_gpu_index": 5,
+      "physical_gpu_uuid": "GPU-f4aed756-228e-7522-94e7-fb543c02cc74",
+      "rate": 0.001,
+      "status": "completed",
+      "total_wall_seconds": 593.5642428398132,
+      "work": {
+        "block_affine_macs_per_token": 196608,
+        "block_attention_macs_per_token": 16384,
+        "candidate_token_presentations": 0,
+        "completed_optimizer_steps": 1000,
+        "enumerated_full_forward_macs": 3619651436544,
+        "feedback_parameter_count": 0,
+        "forward_affine_macs_per_token": 794752,
+        "forward_attention_macs_per_token": 65536,
+        "forward_parameter_count": 813568,
+        "full_forward_token_passes": 4207488,
+        "local_block_token_evaluations": 427438080,
+        "local_head_token_evaluations": 106859520,
+        "local_vjp_token_evaluations": 245760000,
+        "ordinary_training_tokens": 2048000,
+        "ordinary_validation_tokens": 111488,
+        "relaxation_token_passes": 51381760,
+        "training_token_presentations": 2048000
+      }
+    },
+    {
+      "completed_optimizer_steps": 1000,
+      "driver_wall_seconds": 391.04854702949524,
+      "experiment_name": "transformer-p1-dualprop-d4-lr0p001",
+      "final_validation_accuracy": 0.43941051951779564,
+      "final_validation_nll": 1.8991675294773176,
+      "final_validation_perplexity": 6.680330948281979,
+      "finite": true,
+      "manifest": "results/transformer_crossover/p1/transformer-p1-dualprop-d4-lr0p001.json.manifest.json",
+      "method": "dualprop",
+      "output_sha256": "38add2a558e5453ead85aed340cfea310886b273a23266820b2fb86b6a5538af",
+      "peak_memory_allocated_bytes": 97268224,
+      "physical_gpu_index": 7,
+      "physical_gpu_uuid": "GPU-df3d5933-c3e9-9ea1-d595-9d2a3b68c14b",
+      "rate": 0.001,
+      "status": "completed",
+      "total_wall_seconds": 386.51674580574036,
+      "work": {
+        "block_affine_macs_per_token": 196608,
+        "block_attention_macs_per_token": 16384,
+        "candidate_token_presentations": 0,
+        "completed_optimizer_steps": 1000,
+        "enumerated_full_forward_macs": 1857781612544,
+        "feedback_parameter_count": 0,
+        "forward_affine_macs_per_token": 794752,
+        "forward_attention_macs_per_token": 65536,
+        "forward_parameter_count": 813568,
+        "full_forward_token_passes": 2159488,
+        "local_block_token_evaluations": 270336000,
+        "local_head_token_evaluations": 67584000,
+        "local_vjp_token_evaluations": 163840000,
+        "ordinary_training_tokens": 2048000,
+        "ordinary_validation_tokens": 111488,
+        "relaxation_token_passes": 32768000,
+        "training_token_presentations": 2048000
+      }
+    },
+    {
+      "completed_optimizer_steps": 1000,
+      "driver_wall_seconds": 33.252097606658936,
+      "experiment_name": "transformer-p1-ff-d4-lr0p001",
+      "final_validation_accuracy": 0.08121950344431687,
+      "final_validation_nll": 10.612460314885347,
+      "final_validation_perplexity": 40638.05878056827,
+      "finite": true,
+      "manifest": "results/transformer_crossover/p1/transformer-p1-ff-d4-lr0p001.json.manifest.json",
+      "method": "ff",
+      "output_sha256": "b0be9d8098a3b330b4503f86520cec42d36973bfe9316702ba5dd3a20ca4bc3a",
+      "peak_memory_allocated_bytes": 87777792,
+      "physical_gpu_index": 5,
+      "physical_gpu_uuid": "GPU-f4aed756-228e-7522-94e7-fb543c02cc74",
+      "rate": 0.001,
+      "status": "completed",
+      "total_wall_seconds": 27.866358995437622,
+      "work": {
+        "block_affine_macs_per_token": 196608,
+        "block_attention_macs_per_token": 16384,
+        "candidate_token_presentations": 7246720,
+        "completed_optimizer_steps": 1000,
+        "enumerated_full_forward_macs": 9758005903360,
+        "feedback_parameter_count": 0,
+        "forward_affine_macs_per_token": 794752,
+        "forward_attention_macs_per_token": 65536,
+        "forward_parameter_count": 813568,
+        "full_forward_token_passes": 11342720,
+        "local_block_token_evaluations": 1368064,
+        "local_head_token_evaluations": 337920,
+        "local_vjp_token_evaluations": 0,
+        "ordinary_training_tokens": 2048000,
+        "ordinary_validation_tokens": 111488,
+        "relaxation_token_passes": 0,
+        "training_token_presentations": 4096000
+      }
+    },
+    {
+      "completed_optimizer_steps": 1000,
+      "driver_wall_seconds": 43.27602219581604,
+      "experiment_name": "transformer-p1-pepita-d4-lr0p001",
+      "final_validation_accuracy": 0.14852719575200918,
+      "final_validation_nll": 3.1760056514280124,
+      "final_validation_perplexity": 23.950894012013105,
+      "finite": true,
+      "manifest": "results/transformer_crossover/p1/transformer-p1-pepita-d4-lr0p001.json.manifest.json",
+      "method": "pepita",
+      "output_sha256": "1a93e4b9b359500018ee3c14131fd4b37fd79f96a047c281eba66006fe813813",
+      "peak_memory_allocated_bytes": 74502144,
+      "physical_gpu_index": 7,
+      "physical_gpu_uuid": "GPU-df3d5933-c3e9-9ea1-d595-9d2a3b68c14b",
+      "rate": 0.001,
+      "status": "completed",
+      "total_wall_seconds": 37.90927314758301,
+      "work": {
+        "block_affine_macs_per_token": 196608,
+        "block_attention_macs_per_token": 16384,
+        "candidate_token_presentations": 0,
+        "completed_optimizer_steps": 1000,
+        "enumerated_full_forward_macs": 3619651436544,
+        "feedback_parameter_count": 8320,
+        "forward_affine_macs_per_token": 794752,
+        "forward_attention_macs_per_token": 65536,
+        "forward_parameter_count": 813568,
+        "full_forward_token_passes": 4207488,
+        "local_block_token_evaluations": 8192000,
+        "local_head_token_evaluations": 2048000,
+        "local_vjp_token_evaluations": 0,
+        "ordinary_training_tokens": 2048000,
+        "ordinary_validation_tokens": 111488,
+        "relaxation_token_passes": 0,
+        "training_token_presentations": 4096000
+      }
+    },
+    {
+      "completed_optimizer_steps": 1000,
+      "driver_wall_seconds": 34.64618635177612,
+      "experiment_name": "transformer-p1-dfa-d4-lr0p003",
+      "final_validation_accuracy": 0.2621627439724455,
+      "final_validation_nll": 2.619194227847111,
+      "final_validation_perplexity": 13.724660179403989,
+      "finite": true,
+      "manifest": "results/transformer_crossover/p1/transformer-p1-dfa-d4-lr0p003.json.manifest.json",
+      "method": "dfa",
+      "output_sha256": "39dcbd351925b28853fbbd5306303d8d05faab8e0c86fa69da4ae69bc0c84d20",
+      "peak_memory_allocated_bytes": 66264064,
+      "physical_gpu_index": 5,
+      "physical_gpu_uuid": "GPU-f4aed756-228e-7522-94e7-fb543c02cc74",
+      "rate": 0.003,
+      "status": "completed",
+      "total_wall_seconds": 30.523794651031494,
+      "work": {
+        "block_affine_macs_per_token": 196608,
+        "block_attention_macs_per_token": 16384,
+        "candidate_token_presentations": 0,
+        "completed_optimizer_steps": 1000,
+        "enumerated_full_forward_macs": 1857781612544,
+        "feedback_parameter_count": 49920,
+        "forward_affine_macs_per_token": 794752,
+        "forward_attention_macs_per_token": 65536,
+        "forward_parameter_count": 813568,
+        "full_forward_token_passes": 2159488,
+        "local_block_token_evaluations": 8192000,
+        "local_head_token_evaluations": 2048000,
+        "local_vjp_token_evaluations": 0,
+        "ordinary_training_tokens": 2048000,
+        "ordinary_validation_tokens": 111488,
+        "relaxation_token_passes": 0,
+        "training_token_presentations": 2048000
+      }
+    },
+    {
+      "completed_optimizer_steps": 1000,
+      "driver_wall_seconds": 38.71434664726257,
+      "experiment_name": "transformer-p1-fa-d4-lr0p003",
+      "final_validation_accuracy": 0.3378211107921929,
+      "final_validation_nll": 2.2882593507471096,
+      "final_validation_perplexity": 9.857763829438403,
+      "finite": true,
+      "manifest": "results/transformer_crossover/p1/transformer-p1-fa-d4-lr0p003.json.manifest.json",
+      "method": "fa",
+      "output_sha256": "1837dd0289f4575ccaa1f723a500dd4d73fc60c1fbdcc2dc769b9460254a5dff",
+      "peak_memory_allocated_bytes": 146653184,
+      "physical_gpu_index": 7,
+      "physical_gpu_uuid": "GPU-df3d5933-c3e9-9ea1-d595-9d2a3b68c14b",
+      "rate": 0.003,
+      "status": "completed",
+      "total_wall_seconds": 34.66529703140259,
+      "work": {
+        "block_affine_macs_per_token": 196608,
+        "block_attention_macs_per_token": 16384,
+        "candidate_token_presentations": 0,
+        "completed_optimizer_steps": 1000,
+        "enumerated_full_forward_macs": 1857781612544,
+        "feedback_parameter_count": 794752,
+        "forward_affine_macs_per_token": 794752,
+        "forward_attention_macs_per_token": 65536,
+        "forward_parameter_count": 813568,
+        "full_forward_token_passes": 2159488,
+        "local_block_token_evaluations": 0,
+        "local_head_token_evaluations": 0,
+        "local_vjp_token_evaluations": 0,
+        "ordinary_training_tokens": 2048000,
+        "ordinary_validation_tokens": 111488,
+        "relaxation_token_passes": 0,
+        "training_token_presentations": 2048000
+      }
+    },
+    {
+      "completed_optimizer_steps": 1000,
+      "driver_wall_seconds": 49.393547773361206,
+      "experiment_name": "transformer-p1-clean_kp-d4-lr0p003",
+      "final_validation_accuracy": 0.42341776693455796,
+      "final_validation_nll": 1.9429430802845653,
+      "final_validation_perplexity": 6.97926130039316,
+      "finite": true,
+      "manifest": "results/transformer_crossover/p1/transformer-p1-clean_kp-d4-lr0p003.json.manifest.json",
+      "method": "clean_kp",
+      "output_sha256": "caf1d2c39fd2ed1d2a2f5ea6d0324969ee891472b4fab00a268a78c810c8a8e1",
+      "peak_memory_allocated_bytes": 153568768,
+      "physical_gpu_index": 5,
+      "physical_gpu_uuid": "GPU-f4aed756-228e-7522-94e7-fb543c02cc74",
+      "rate": 0.003,
+      "status": "completed",
+      "total_wall_seconds": 44.18988084793091,
+      "work": {
+        "block_affine_macs_per_token": 196608,
+        "block_attention_macs_per_token": 16384,
+        "candidate_token_presentations": 0,
+        "completed_optimizer_steps": 1000,
+        "enumerated_full_forward_macs": 1857781612544,
+        "feedback_parameter_count": 794752,
+        "forward_affine_macs_per_token": 794752,
+        "forward_attention_macs_per_token": 65536,
+        "forward_parameter_count": 813568,
+        "full_forward_token_passes": 2159488,
+        "local_block_token_evaluations": 0,
+        "local_head_token_evaluations": 0,
+        "local_vjp_token_evaluations": 0,
+        "ordinary_training_tokens": 2048000,
+        "ordinary_validation_tokens": 111488,
+        "relaxation_token_passes": 0,
+        "training_token_presentations": 2048000
+      }
+    },
+    {
+      "completed_optimizer_steps": 1000,
+      "driver_wall_seconds": 57.563411235809326,
+      "experiment_name": "transformer-p1-sdil-d4-lr0p003",
+      "final_validation_accuracy": 0.4066267221584386,
+      "final_validation_nll": 1.9958930294768271,
+      "final_validation_perplexity": 7.358771694484222,
+      "finite": true,
+      "manifest": "results/transformer_crossover/p1/transformer-p1-sdil-d4-lr0p003.json.manifest.json",
+      "method": "sdil",
+      "output_sha256": "66cb605c4129a7e29e71e7c97d031e05566ecf6050d8967e5dfbd538520ecaf9",
+      "peak_memory_allocated_bytes": 157764096,
+      "physical_gpu_index": 7,
+      "physical_gpu_uuid": "GPU-df3d5933-c3e9-9ea1-d595-9d2a3b68c14b",
+      "rate": 0.003,
+      "status": "completed",
+      "total_wall_seconds": 52.4935040473938,
+      "work": {
+        "block_affine_macs_per_token": 196608,
+        "block_attention_macs_per_token": 16384,
+        "candidate_token_presentations": 0,
+        "completed_optimizer_steps": 1000,
+        "enumerated_full_forward_macs": 1857781612544,
+        "feedback_parameter_count": 794752,
+        "forward_affine_macs_per_token": 794752,
+        "forward_attention_macs_per_token": 65536,
+        "forward_parameter_count": 813568,
+        "full_forward_token_passes": 2159488,
+        "local_block_token_evaluations": 0,
+        "local_head_token_evaluations": 0,
+        "local_vjp_token_evaluations": 0,
+        "ordinary_training_tokens": 2048000,
+        "ordinary_validation_tokens": 111488,
+        "relaxation_token_passes": 0,
+        "training_token_presentations": 2048000
+      }
+    },
+    {
+      "completed_optimizer_steps": 1000,
+      "driver_wall_seconds": 27.13742709159851,
+      "experiment_name": "transformer-p1-bp-d4-lr0p003",
+      "final_validation_accuracy": 0.40393584959816303,
+      "final_validation_nll": 2.0219551145551398,
+      "final_validation_perplexity": 7.553077639376615,
+      "finite": true,
+      "manifest": "results/transformer_crossover/p1/transformer-p1-bp-d4-lr0p003.json.manifest.json",
+      "method": "bp",
+      "output_sha256": "7d04f9496efa13daba776257680bd3676ec842fe0294779b156a3f709cb00020",
+      "peak_memory_allocated_bytes": 109656064,
+      "physical_gpu_index": 5,
+      "physical_gpu_uuid": "GPU-f4aed756-228e-7522-94e7-fb543c02cc74",
+      "rate": 0.003,
+      "status": "completed",
+      "total_wall_seconds": 22.951324939727783,
+      "work": {
+        "block_affine_macs_per_token": 196608,
+        "block_attention_macs_per_token": 16384,
+        "candidate_token_presentations": 0,
+        "completed_optimizer_steps": 1000,
+        "enumerated_full_forward_macs": 1857781612544,
+        "feedback_parameter_count": 0,
+        "forward_affine_macs_per_token": 794752,
+        "forward_attention_macs_per_token": 65536,
+        "forward_parameter_count": 813568,
+        "full_forward_token_passes": 2159488,
+        "local_block_token_evaluations": 0,
+        "local_head_token_evaluations": 0,
+        "local_vjp_token_evaluations": 0,
+        "ordinary_training_tokens": 2048000,
+        "ordinary_validation_tokens": 111488,
+        "relaxation_token_passes": 0,
+        "training_token_presentations": 2048000
+      }
+    }
+  ],
+  "registry_sha256": "25cbf03bbf1a7eadea3894cb8a4ef577b737f38c47a3857723abe6e7e4c53201",
+  "selected": {
+    "bp": {
+      "experiment_name": "transformer-p1-bp-d4-lr0p001",
+      "final_validation_nll": 1.899204445761189,
+      "finite": true,
+      "rate": 0.001
+    },
+    "clean_kp": {
+      "experiment_name": "transformer-p1-clean_kp-d4-lr0p003",
+      "final_validation_nll": 1.9429430802845653,
+      "finite": true,
+      "rate": 0.003
+    },
+    "dfa": {
+      "experiment_name": "transformer-p1-dfa-d4-lr0p003",
+      "final_validation_nll": 2.619194227847111,
+      "finite": true,
+      "rate": 0.003
+    },
+    "dualprop": {
+      "experiment_name": "transformer-p1-dualprop-d4-lr0p001",
+      "final_validation_nll": 1.8991675294773176,
+      "finite": true,
+      "rate": 0.001
+    },
+    "ep": {
+      "experiment_name": "transformer-p1-ep-d4-lr0p001",
+      "final_validation_nll": 4.147124764566169,
+      "finite": true,
+      "rate": 0.001
+    },
+    "fa": {
+      "experiment_name": "transformer-p1-fa-d4-lr0p001",
+      "final_validation_nll": 2.2787026888193687,
+      "finite": true,
+      "rate": 0.001
+    },
+    "ff": {
+      "experiment_name": "transformer-p1-ff-d4-lr0p0001",
+      "final_validation_nll": 8.052283456760762,
+      "finite": true,
+      "rate": 0.0001
+    },
+    "pepita": {
+      "experiment_name": "transformer-p1-pepita-d4-lr0p001",
+      "final_validation_nll": 3.1760056514280124,
+      "finite": true,
+      "rate": 0.001
+    },
+    "sdil": {
+      "experiment_name": "transformer-p1-sdil-d4-lr0p001",
+      "final_validation_nll": 1.960945281588251,
+      "finite": true,
+      "rate": 0.001
+    }
+  },
+  "selection_rule": "minimum finite final validation NLL, then lower learning rate; failed/timeout records rank after finite records",
+  "source": {
+    "environment_freeze": [
+      "filelock==3.29.0",
+      "fsspec==2026.4.0",
+      "Jinja2==3.1.6",
+      "MarkupSafe==3.0.3",
+      "mpmath==1.3.0",
+      "networkx==3.6.1",
+      "numpy==2.4.4",
+      "nvidia-cublas-cu11==11.11.3.6",
+      "nvidia-cuda-cupti-cu11==11.8.87",
+      "nvidia-cuda-nvrtc-cu11==11.8.89",
+      "nvidia-cuda-runtime-cu11==11.8.89",
+      "nvidia-cudnn-cu11==8.7.0.84",
+      "nvidia-cufft-cu11==10.9.0.58",
+      "nvidia-curand-cu11==10.3.0.86",
+      "nvidia-cusolver-cu11==11.4.1.48",
+      "nvidia-cusparse-cu11==11.7.5.86",
+      "nvidia-nccl-cu11==2.20.5",
+      "nvidia-nvtx-cu11==11.8.86",
+      "packaging==26.0",
+      "scipy==1.17.1",
+      "sympy==1.14.0",
+      "torch==2.3.1+cu118",
+      "triton==2.3.1",
+      "typing_extensions==4.15.0"
+    ],
+    "environment_freeze_sha256": "bca82e6091ef12808e890cc36a0095dd885916b1d5e0cd4a8aa2119b6ca861a5",
+    "git_commit": "19f3966fc2d15b1ed18862a779d91ee16ed56bfc",
+    "protocol_path": "/scratch/yurenh2/sdil-runs/transformer-p1/TRANSFORMER_CROSSOVER.md",
+    "protocol_sha256": "841a107328f4961c9fa14cbf357b0c5699ac9b3e34b8d124ccb389ab145c8a0e",
+    "python": "/home/yurenh2/miniconda3/envs/ep_pascal/bin/python3",
+    "tracked_files": {
+      "TRANSFORMER_CROSSOVER.md": "841a107328f4961c9fa14cbf357b0c5699ac9b3e34b8d124ccb389ab145c8a0e",
+      "experiments/analyze_transformer_crossover_p1.py": "76a1df0db00d36d2732b76fd71b54f3aa0057dd611186162d8b0fc442a60d294",
+      "experiments/transformer_crossover_grid.py": "947152fb257dd005e22aa546be8ff0f63afe80436aa5f591db4dc29f99fb5eac",
+      "experiments/transformer_crossover_native.py": "c45299ecfdd80bab8ededa6f8c0e1ad2275ea7a3d68f711ee7cddad29a933c5e",
+      "experiments/transformer_feedback_smoke.py": "a8c0ba44091765ee2e05381f36515923990fd80e0acfef5aeea58ca29ec52092",
+      "sdil/transformer.py": "09180e5625054c4dc16567af8939d05c4cb7cbc08f959445f25daae7c830cb34"
+    }
+  },
+  "stage": "transformer_crossover_p1",
+  "test_policy": "none"
+}