results: pass calibrated oral-B-v2 development gate

1 files changed, 461 insertions, 0 deletions

diff --git a/results/bci_v2_calibrated_dev/bci_v2_calibrated_t28_m0.json b/results/bci_v2_calibrated_dev/bci_v2_calibrated_t28_m0.json
new file mode 100644
index 0000000..3880956
--- /dev/null
+++ b/results/bci_v2_calibrated_dev/bci_v2_calibrated_t28_m0.json
@@ -0,0 +1,461 @@
+{
+  "args": {
+    "model_seed": 0,
+    "outdir": "results/bci_v2_calibrated_dev",
+    "task_seed": 28
+  },
+  "assays": {
+    "challenge": {
+      "active_state_episode_steps_by_mode": {
+        "acute_critic_lesion": 12957,
+        "acute_outcome_lesion": 12957,
+        "intact": 12957
+      },
+      "calibration": {
+        "active_state_episode_steps": 14336,
+        "episodes": 512,
+        "maximum_cursor_summary": {
+          "maximum": 1.9806733131408691,
+          "median": 1.9220820665359497,
+          "minimum": 1.6861028671264648
+        },
+        "quantiles": [
+          0.2,
+          0.35,
+          0.5,
+          0.65,
+          0.8
+        ],
+        "seed": 550028,
+        "targets": [
+          1.8872586727142333,
+          1.9072148621082305,
+          1.9221965074539185,
+          1.9328129887580872,
+          1.9453917741775513
+        ],
+        "uses_outcome_labels": false
+      },
+      "episodes_per_target": 128,
+      "maximum_steps_per_episode": 28,
+      "selection_over_evaluation_outcomes": false,
+      "targets": [
+        1.8872586727142333,
+        1.9072148621082305,
+        1.9221965074539185,
+        1.9328129887580872,
+        1.9453917741775513
+      ],
+      "trajectory_seeds": {
+        "1": 588000,
+        "2": 588001,
+        "3": 588002,
+        "4": 588003,
+        "5": 588004
+      }
+    },
+    "performance_evaluation_episodes": 256,
+    "performance_evaluation_seed": 540028
+  },
+  "conditions": {
+    "critic_training_lesion": {
+      "cost": {
+        "active_state_episode_steps": 17384,
+        "cursor_scalar_observations": 9062,
+        "maximum_state_episode_steps": 25088,
+        "reverse_mode_calls": 0,
+        "role_probe_examples": 4531,
+        "task_loss_queries": 0,
+        "terminal_outcome_observations": 896
+      },
+      "critic_l2_after_training": 0.0,
+      "daily_success": [
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.015625,
+        0.015625,
+        0.0625,
+        0.296875,
+        0.859375,
+        1.0,
+        1.0,
+        1.0,
+        1.0
+      ],
+      "early_success": 0.0,
+      "evaluation_active_state_episode_steps": 261,
+      "final_success": 1.0,
+      "late_success": 1.0,
+      "learning_gain": 1.0,
+      "role_cosine_after_training": 0.9773973226547241,
+      "training_wall_s": 0.1755989007651806
+    },
+    "fixed_role": {
+      "cost": {
+        "active_state_episode_steps": 25088,
+        "cursor_scalar_observations": 12544,
+        "maximum_state_episode_steps": 25088,
+        "reverse_mode_calls": 0,
+        "role_probe_examples": 6272,
+        "task_loss_queries": 0,
+        "terminal_outcome_observations": 896
+      },
+      "critic_l2_after_training": 0.06155245006084442,
+      "daily_success": [
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0
+      ],
+      "early_success": 0.0,
+      "evaluation_active_state_episode_steps": 7168,
+      "final_success": 0.0,
+      "late_success": 0.0,
+      "learning_gain": 0.0,
+      "role_cosine_after_training": -0.1393236517906189,
+      "training_wall_s": 0.21381374821066856
+    },
+    "intact": {
+      "cost": {
+        "active_state_episode_steps": 14224,
+        "cursor_scalar_observations": 7682,
+        "maximum_state_episode_steps": 25088,
+        "reverse_mode_calls": 0,
+        "role_probe_examples": 3841,
+        "task_loss_queries": 0,
+        "terminal_outcome_observations": 896
+      },
+      "critic_l2_after_training": 0.533806562423706,
+      "daily_success": [
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.03125,
+        0.03125,
+        0.203125,
+        0.890625,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0
+      ],
+      "early_success": 0.0,
+      "evaluation_active_state_episode_steps": 260,
+      "final_success": 1.0,
+      "late_success": 1.0,
+      "learning_gain": 1.0,
+      "role_cosine_after_training": 0.9761794805526733,
+      "training_wall_s": 0.258306160569191
+    },
+    "oracle_role": {
+      "cost": {
+        "active_state_episode_steps": 14083,
+        "cursor_scalar_observations": 7614,
+        "maximum_state_episode_steps": 25088,
+        "reverse_mode_calls": 0,
+        "role_probe_examples": 3807,
+        "task_loss_queries": 0,
+        "terminal_outcome_observations": 896
+      },
+      "critic_l2_after_training": 0.4723714292049408,
+      "daily_success": [
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.046875,
+        0.0625,
+        0.234375,
+        0.890625,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0
+      ],
+      "early_success": 0.0,
+      "evaluation_active_state_episode_steps": 256,
+      "final_success": 1.0,
+      "late_success": 1.0,
+      "learning_gain": 1.0,
+      "role_cosine_after_training": 1.0000001192092896,
+      "training_wall_s": 0.18617762252688408
+    },
+    "outcome_training_lesion": {
+      "cost": {
+        "active_state_episode_steps": 18917,
+        "cursor_scalar_observations": 9704,
+        "maximum_state_episode_steps": 25088,
+        "reverse_mode_calls": 0,
+        "role_probe_examples": 4852,
+        "task_loss_queries": 0,
+        "terminal_outcome_observations": 896
+      },
+      "critic_l2_after_training": 0.17078045010566711,
+      "daily_success": [
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.03125,
+        0.03125,
+        0.078125,
+        0.265625,
+        0.5,
+        0.78125,
+        0.84375,
+        0.953125,
+        0.9375,
+        0.984375
+      ],
+      "early_success": 0.0,
+      "evaluation_active_state_episode_steps": 1684,
+      "final_success": 0.96875,
+      "late_success": 0.9583333333333334,
+      "learning_gain": 0.9583333333333334,
+      "role_cosine_after_training": 0.9445701241493225,
+      "training_wall_s": 0.21972168236970901
+    },
+    "plasticity_lesion": {
+      "cost": {
+        "active_state_episode_steps": 25088,
+        "cursor_scalar_observations": 12544,
+        "maximum_state_episode_steps": 25088,
+        "reverse_mode_calls": 0,
+        "role_probe_examples": 6272,
+        "task_loss_queries": 0,
+        "terminal_outcome_observations": 896
+      },
+      "critic_l2_after_training": 0.06252607703208923,
+      "daily_success": [
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0
+      ],
+      "early_success": 0.0,
+      "evaluation_active_state_episode_steps": 7168,
+      "final_success": 0.0,
+      "late_success": 0.0,
+      "learning_gain": 0.0,
+      "role_cosine_after_training": 0.9957386255264282,
+      "training_wall_s": 0.1857893168926239
+    }
+  },
+  "config": {
+    "context_ar": 0.8,
+    "context_dim": 16,
+    "coupling_scale": 1.0,
+    "critic_eta": 0.03,
+    "days": 14,
+    "eligibility_decay": 0.8,
+    "episodes_per_day": 64,
+    "feedback": "performance_velocity",
+    "forward_eta": 0.1,
+    "gamma": 0.8,
+    "inertia": 0.65,
+    "kappa": 0.0,
+    "n_background": 30,
+    "n_minus": 5,
+    "n_plus": 5,
+    "perturb_every": 4,
+    "perturb_sigma": 0.03,
+    "predictor_eta": 0.2,
+    "process_noise": 0.12,
+    "steps_per_episode": 28,
+    "target": 0.8,
+    "terminal_reward": 1.0,
+    "vectorizer_eta": 0.03,
+    "velocity_reward_scale": 1.0
+  },
+  "finite": true,
+  "hardware": {
+    "device": "cpu",
+    "platform": "Linux-5.15.0-161-generic-x86_64-with-glibc2.35",
+    "threads": 1,
+    "torch_version": "2.10.0+cu128"
+  },
+  "peak_rss_mib": 954.734375,
+  "protocol": {
+    "calibration_uses_outcome_labels": false,
+    "confirmation_seeds_touched": false,
+    "failed_target_gate_sha256": "1df37c0b639542bbee3017da54306b0d572cbf96230428f142fafed139ac92f1",
+    "fixed_config": {
+      "critic_eta": 0.03,
+      "forward_eta": 0.1,
+      "gamma": 0.8,
+      "velocity_reward_scale": 1.0
+    },
+    "hyperparameter_selection": false,
+    "name": "oral_b_v2_calibrated_recovery_development_v1",
+    "protocol_sha256": "a175dc80fdb7218c010953d5d7e3c77fee0690d5f10fabaf9d700bb8505acde8",
+    "selection_split": "development_validation"
+  },
+  "provenance": {
+    "git_commit": "d289a46293e0b422c195746ad8b310d4b50ded23",
+    "git_tracked_dirty": false,
+    "input_sha256": {
+      "base_dynamics": "d5a373314562af0daedf55baa04b975fe643236c7821be1560d2cfb4359688fc",
+      "common_runner": "4157a57806dedfc2eb70ec84f7a7ca1f73ab684e5d8f0b227cd80465e8e97aa5",
+      "confirmation_analyzer": "2536fa01f62c3b2d87e782b08075cb5948aa26bfdce71a1d8c2446d5b68bb34c",
+      "confirmation_runner": "5e9beba106b4b29c4ee1568052d6e8c15690dbde92917a2497a52e43e7f557d1",
+      "d4_gate": "636c587e47287338cdca7d9558dc3bfb99a2cec615badb23337025d85b8128ef",
+      "development_analyzer": "dde4a2e4d62f5f9da94357b388139c50251b6863f9c991f1a006e71eeb7953e4",
+      "failed_target_gate": "1df37c0b639542bbee3017da54306b0d572cbf96230428f142fafed139ac92f1",
+      "failed_v2_gate": "e53f46cf456d60ce4d33586ace4f4619fb1a31a58082b2b72943d4ed10cd25fd",
+      "old_r2_gate": "4f6f969ceae88afa2523e3472a3373522991f5ecaab69440830c07479d2d3597",
+      "protocol": "a175dc80fdb7218c010953d5d7e3c77fee0690d5f10fabaf9d700bb8505acde8",
+      "recovery_confirmation_analyzer": "29a168143266bc7078e5229183bb7ea45283973592fecd1fcd715488f5d6242b",
+      "recovery_development_analyzer": "2768b877132ece6d612d2a5f1e483c495b277e26fdf54fb8a716aa8de6fd67e2",
+      "recovery_metrics": "371cb6a7e2192d25ddbfad69c0ce7a1150aa1c21ca98e141bedb2afc2acbdbaa",
+      "recovery_runner": "cea0ef658c698c376b1fda0cf2f7f910ee44f7bcde153f2fbaf18e66a2d3661a",
+      "runner": "446ad26fc015eb79d28e0bb442c8aef71bdd1d0320910494cac20f92e3c7dc18",
+      "v2_dynamics": "786c5aed3091644272ae0a740913ac3cc38cdb02a8cacb6dfc8496e36836cdae",
+      "v2_metrics": "421da9287b92e62c742f9eeefbfdccba582c6a4ad5cf92ade4709689120fa1ee"
+    },
+    "tracked_inputs": {
+      "base_dynamics": true,
+      "common_runner": true,
+      "confirmation_analyzer": true,
+      "confirmation_runner": true,
+      "d4_gate": true,
+      "development_analyzer": true,
+      "failed_target_gate": true,
+      "failed_v2_gate": true,
+      "old_r2_gate": true,
+      "protocol": true,
+      "recovery_confirmation_analyzer": true,
+      "recovery_development_analyzer": true,
+      "recovery_metrics": true,
+      "recovery_runner": true,
+      "runner": true,
+      "v2_dynamics": true,
+      "v2_metrics": true
+    }
+  },
+  "schema_version": 4,
+  "signatures": {
+    "acute_outcome_lesion_outcome_balanced_acc": 0.540903540903541,
+    "acute_outcome_lesion_role_aligned_separation": 0.010854148777368705,
+    "causal_role_sign_inversion_index": 0.03921735111033796,
+    "challenge_episodes": 640,
+    "challenge_failure_count": 315,
+    "challenge_success_count": 325,
+    "challenge_success_fraction": 0.5078125,
+    "critic_contribution_value_prediction_corr": 0.9999999999994963,
+    "decoder_distance_residual_corr": 0.13642571216406077,
+    "mean_abs_raw_soma_corr": 0.9990992136226702,
+    "mean_abs_residual_soma_corr": 0.05195367815050812,
+    "mean_critic_expectedness_contribution": 0.24424682945902534,
+    "nonterminal_training_events": 13328,
+    "raw_minus_residual_abs_soma_corr": 0.9471455354721621,
+    "role_aligned_error_cv_corr": 0.34897921307347385,
+    "role_aligned_velocity_cv_corr": 0.9982339724621896,
+    "surrounding_event_decoder_balanced_acc": 0.5528030824638526,
+    "target_success_fraction": {
+      "1.8872586727142333": 0.78125,
+      "1.9072148621082305": 0.640625,
+      "1.9221965074539185": 0.4765625,
+      "1.9328129887580872": 0.40625,
+      "1.9453917741775513": 0.234375
+    },
+    "terminal_outcome_separation_drop_under_acute_lesion": 0.38343148953250883,
+    "terminal_previous_soma_outcome_balanced_acc": 0.7603663003663004,
+    "terminal_residual_minus_previous_soma_acc": 0.23804639804639804,
+    "terminal_residual_outcome_balanced_acc": 0.9984126984126984,
+    "terminal_role_aligned_outcome_separation": 0.39428563830987756,
+    "terminal_training_events": 896,
+    "velocity_minus_error_abs_cv_corr": 0.6492547593887157
+  },
+  "split": "development_validation",
+  "wall_s": 2.422760460525751,
+  "warmup": {
+    "critic_training_lesion": {
+      "batches": 100,
+      "examples": 6400,
+      "instruction_present": false,
+      "predictor_max_abs_error": 2.384185791015625e-07,
+      "rng_seed": 602800,
+      "role_cosine_after_warmup": 0.9957716464996338,
+      "role_cursor_scalar_observations": 12800,
+      "role_update_enabled": true
+    },
+    "fixed_role": {
+      "batches": 100,
+      "examples": 6400,
+      "instruction_present": false,
+      "predictor_max_abs_error": 2.384185791015625e-07,
+      "rng_seed": 602800,
+      "role_cosine_after_warmup": -0.1393236517906189,
+      "role_cursor_scalar_observations": 12800,
+      "role_update_enabled": false
+    },
+    "intact": {
+      "batches": 100,
+      "examples": 6400,
+      "instruction_present": false,
+      "predictor_max_abs_error": 2.384185791015625e-07,
+      "rng_seed": 602800,
+      "role_cosine_after_warmup": 0.9957716464996338,
+      "role_cursor_scalar_observations": 12800,
+      "role_update_enabled": true
+    },
+    "oracle_role": {
+      "batches": 100,
+      "examples": 6400,
+      "instruction_present": false,
+      "predictor_max_abs_error": 2.384185791015625e-07,
+      "rng_seed": 602800,
+      "role_cosine_after_warmup": 1.0000001192092896,
+      "role_cursor_scalar_observations": 12800,
+      "role_update_enabled": false
+    },
+    "outcome_training_lesion": {
+      "batches": 100,
+      "examples": 6400,
+      "instruction_present": false,
+      "predictor_max_abs_error": 2.384185791015625e-07,
+      "rng_seed": 602800,
+      "role_cosine_after_warmup": 0.9957716464996338,
+      "role_cursor_scalar_observations": 12800,
+      "role_update_enabled": true
+    },
+    "plasticity_lesion": {
+      "batches": 100,
+      "examples": 6400,
+      "instruction_present": false,
+      "predictor_max_abs_error": 2.384185791015625e-07,
+      "rng_seed": 602800,
+      "role_cosine_after_warmup": 0.9957716464996338,
+      "role_cursor_scalar_observations": 12800,
+      "role_update_enabled": true
+    }
+  }
+}