diff options
| author | YurenHao0426 <Blackhao0426@gmail.com> | 2026-07-23 08:08:32 -0500 |
|---|---|---|
| committer | YurenHao0426 <Blackhao0426@gmail.com> | 2026-07-23 08:08:32 -0500 |
| commit | 378e68dc424acb6b5a2082bc071314ce810d8ab5 (patch) | |
| tree | 9fb5698bee534df1a1045ae4f0e1fa7c2f42ff95 /results | |
| parent | dae172b1425c646f0441609c28fc5156b50f820d (diff) | |
results: retain target-ladder recovery failure
Diffstat (limited to 'results')
| -rw-r--r-- | results/bci_v2_recovery_dev/bci_v2_recovery_t23_m0.json | 430 | ||||
| -rw-r--r-- | results/bci_v2_recovery_dev/bci_v2_recovery_t24_m0.json | 430 | ||||
| -rw-r--r-- | results/bci_v2_recovery_dev/bci_v2_recovery_t25_m0.json | 430 | ||||
| -rw-r--r-- | results/bci_v2_recovery_dev_gate.json | 113 |
4 files changed, 1403 insertions, 0 deletions
diff --git a/results/bci_v2_recovery_dev/bci_v2_recovery_t23_m0.json b/results/bci_v2_recovery_dev/bci_v2_recovery_t23_m0.json new file mode 100644 index 0000000..41aca9f --- /dev/null +++ b/results/bci_v2_recovery_dev/bci_v2_recovery_t23_m0.json @@ -0,0 +1,430 @@ +{ + "args": { + "model_seed": 0, + "outdir": "results/bci_v2_recovery_dev", + "task_seed": 23 + }, + "assays": { + "challenge": { + "active_state_episode_steps_by_mode": { + "acute_critic_lesion": 10799, + "acute_outcome_lesion": 10799, + "intact": 10799 + }, + "episodes_per_target": 128, + "maximum_steps_per_episode": 28, + "selection_over_targets": false, + "targets": [ + 1.55, + 1.6, + 1.65, + 1.7, + 1.75, + 1.8 + ], + "trajectory_seeds": { + "1.55": 493000, + "1.6": 493001, + "1.65": 493002, + "1.7": 493003, + "1.75": 493004, + "1.8": 493005 + } + }, + "performance_evaluation_episodes": 256, + "performance_evaluation_seed": 460023 + }, + "conditions": { + "critic_training_lesion": { + "cost": { + "active_state_episode_steps": 25069, + "cursor_scalar_observations": 12536, + "maximum_state_episode_steps": 25088, + "reverse_mode_calls": 0, + "role_probe_examples": 6268, + "task_loss_queries": 0, + "terminal_outcome_observations": 896 + }, + "critic_l2_after_training": 0.0, + "daily_success": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.015625, + 0.0, + 0.0 + ], + "early_success": 0.0, + "evaluation_active_state_episode_steps": 7168, + "final_success": 0.0, + "late_success": 0.005208333333333333, + "learning_gain": 0.005208333333333333, + "role_cosine_after_training": 0.9950354099273682, + "training_wall_s": 0.21576084941625595 + }, + "fixed_role": { + "cost": { + "active_state_episode_steps": 25088, + "cursor_scalar_observations": 12544, + "maximum_state_episode_steps": 25088, + "reverse_mode_calls": 0, + "role_probe_examples": 6272, + "task_loss_queries": 0, + "terminal_outcome_observations": 896 + }, + "critic_l2_after_training": 0.061243072152137756, + "daily_success": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "early_success": 0.0, + "evaluation_active_state_episode_steps": 7168, + "final_success": 0.0, + "late_success": 0.0, + "learning_gain": 0.0, + "role_cosine_after_training": -0.1393236517906189, + "training_wall_s": 0.2373625412583351 + }, + "intact": { + "cost": { + "active_state_episode_steps": 14590, + "cursor_scalar_observations": 7848, + "maximum_state_episode_steps": 25088, + "reverse_mode_calls": 0, + "role_probe_examples": 3924, + "task_loss_queries": 0, + "terminal_outcome_observations": 896 + }, + "critic_l2_after_training": 0.4425785541534424, + "daily_success": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.046875, + 0.140625, + 0.703125, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "early_success": 0.0, + "evaluation_active_state_episode_steps": 263, + "final_success": 1.0, + "late_success": 1.0, + "learning_gain": 1.0, + "role_cosine_after_training": 0.9832371473312378, + "training_wall_s": 0.2712929956614971 + }, + "oracle_role": { + "cost": { + "active_state_episode_steps": 14626, + "cursor_scalar_observations": 7862, + "maximum_state_episode_steps": 25088, + "reverse_mode_calls": 0, + "role_probe_examples": 3931, + "task_loss_queries": 0, + "terminal_outcome_observations": 896 + }, + "critic_l2_after_training": 0.45056235790252686, + "daily_success": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.046875, + 0.140625, + 0.703125, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "early_success": 0.0, + "evaluation_active_state_episode_steps": 262, + "final_success": 1.0, + "late_success": 1.0, + "learning_gain": 1.0, + "role_cosine_after_training": 1.0000001192092896, + "training_wall_s": 0.1980770006775856 + }, + "outcome_training_lesion": { + "cost": { + "active_state_episode_steps": 18170, + "cursor_scalar_observations": 9358, + "maximum_state_episode_steps": 25088, + "reverse_mode_calls": 0, + "role_probe_examples": 4679, + "task_loss_queries": 0, + "terminal_outcome_observations": 896 + }, + "critic_l2_after_training": 0.1713862419128418, + "daily_success": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.046875, + 0.0625, + 0.21875, + 0.59375, + 0.828125, + 0.90625, + 0.96875, + 1.0, + 0.984375 + ], + "early_success": 0.0, + "evaluation_active_state_episode_steps": 1534, + "final_success": 0.98828125, + "late_success": 0.984375, + "learning_gain": 0.984375, + "role_cosine_after_training": 0.9750589728355408, + "training_wall_s": 0.22877522557973862 + }, + "plasticity_lesion": { + "cost": { + "active_state_episode_steps": 25088, + "cursor_scalar_observations": 12544, + "maximum_state_episode_steps": 25088, + "reverse_mode_calls": 0, + "role_probe_examples": 6272, + "task_loss_queries": 0, + "terminal_outcome_observations": 896 + }, + "critic_l2_after_training": 0.06210753321647644, + "daily_success": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "early_success": 0.0, + "evaluation_active_state_episode_steps": 7168, + "final_success": 0.0, + "late_success": 0.0, + "learning_gain": 0.0, + "role_cosine_after_training": 0.9950442314147949, + "training_wall_s": 0.20400389283895493 + } + }, + "config": { + "context_ar": 0.8, + "context_dim": 16, + "coupling_scale": 1.0, + "critic_eta": 0.03, + "days": 14, + "eligibility_decay": 0.8, + "episodes_per_day": 64, + "feedback": "performance_velocity", + "forward_eta": 0.1, + "gamma": 0.8, + "inertia": 0.65, + "kappa": 0.0, + "n_background": 30, + "n_minus": 5, + "n_plus": 5, + "perturb_every": 4, + "perturb_sigma": 0.03, + "predictor_eta": 0.2, + "process_noise": 0.12, + "steps_per_episode": 28, + "target": 0.8, + "terminal_reward": 1.0, + "vectorizer_eta": 0.03, + "velocity_reward_scale": 1.0 + }, + "finite": true, + "hardware": { + "device": "cpu", + "platform": "Linux-5.15.0-161-generic-x86_64-with-glibc2.35", + "threads": 1, + "torch_version": "2.10.0+cu128" + }, + "peak_rss_mib": 998.17578125, + "protocol": { + "confirmation_seeds_touched": false, + "d4_gate_sha256": "636c587e47287338cdca7d9558dc3bfb99a2cec615badb23337025d85b8128ef", + "failed_v2_gate_sha256": "e53f46cf456d60ce4d33586ace4f4619fb1a31a58082b2b72943d4ed10cd25fd", + "fixed_config": { + "critic_eta": 0.03, + "forward_eta": 0.1, + "gamma": 0.8, + "velocity_reward_scale": 1.0 + }, + "hyperparameter_selection": false, + "name": "oral_b_v2_cold_start_recovery_development_v1", + "old_r2_gate_sha256": "4f6f969ceae88afa2523e3472a3373522991f5ecaab69440830c07479d2d3597", + "protocol_sha256": "0621b47367074cb99ee74b332192878e76acbca11678b5238e1312adc2a2bcd9", + "selection_split": "development_validation" + }, + "provenance": { + "git_commit": "dae172b1425c646f0441609c28fc5156b50f820d", + "git_tracked_dirty": false, + "input_sha256": { + "base_dynamics": "d5a373314562af0daedf55baa04b975fe643236c7821be1560d2cfb4359688fc", + "confirmation_analyzer": "29a168143266bc7078e5229183bb7ea45283973592fecd1fcd715488f5d6242b", + "confirmation_runner": "0582ae7a0c173d2fef842ea57d75b77b23cb6f70391f7013ad27ec29e0fedfdf", + "d4_gate": "636c587e47287338cdca7d9558dc3bfb99a2cec615badb23337025d85b8128ef", + "development_analyzer": "2768b877132ece6d612d2a5f1e483c495b277e26fdf54fb8a716aa8de6fd67e2", + "failed_v2_gate": "e53f46cf456d60ce4d33586ace4f4619fb1a31a58082b2b72943d4ed10cd25fd", + "old_r2_gate": "4f6f969ceae88afa2523e3472a3373522991f5ecaab69440830c07479d2d3597", + "protocol": "0621b47367074cb99ee74b332192878e76acbca11678b5238e1312adc2a2bcd9", + "recovery_metrics": "371cb6a7e2192d25ddbfad69c0ce7a1150aa1c21ca98e141bedb2afc2acbdbaa", + "runner": "cea0ef658c698c376b1fda0cf2f7f910ee44f7bcde153f2fbaf18e66a2d3661a", + "v2_dynamics": "786c5aed3091644272ae0a740913ac3cc38cdb02a8cacb6dfc8496e36836cdae", + "v2_metrics": "421da9287b92e62c742f9eeefbfdccba582c6a4ad5cf92ade4709689120fa1ee" + }, + "tracked_inputs": { + "base_dynamics": true, + "confirmation_analyzer": true, + "confirmation_runner": true, + "d4_gate": true, + "development_analyzer": true, + "failed_v2_gate": true, + "old_r2_gate": true, + "protocol": true, + "recovery_metrics": true, + "runner": true, + "v2_dynamics": true, + "v2_metrics": true + } + }, + "schema_version": 3, + "signatures": { + "acute_outcome_lesion_outcome_balanced_acc": 0.5541654667379767, + "acute_outcome_lesion_role_aligned_separation": 0.015703842176951255, + "causal_role_sign_inversion_index": 0.04036224680362202, + "challenge_episodes": 768, + "challenge_failure_count": 223, + "challenge_success_count": 545, + "challenge_success_fraction": 0.7096354166666666, + "critic_contribution_value_prediction_corr": 0.9999999999993424, + "decoder_distance_residual_corr": 0.11840537575539738, + "mean_abs_raw_soma_corr": 0.9990800248304881, + "mean_abs_residual_soma_corr": 0.05415501113040725, + "mean_critic_expectedness_contribution": 0.3046008384890047, + "nonterminal_training_events": 13694, + "raw_minus_residual_abs_soma_corr": 0.9449250137000809, + "role_aligned_error_cv_corr": 0.35695106277832594, + "role_aligned_velocity_cv_corr": 0.9984821724524531, + "surrounding_event_decoder_balanced_acc": 0.551090861356458, + "target_success_fraction": { + "1.55": 1.0, + "1.6": 0.984375, + "1.65": 0.96875, + "1.7": 0.7578125, + "1.75": 0.421875, + "1.8": 0.125 + }, + "terminal_outcome_separation_drop_under_acute_lesion": 0.40689127104423845, + "terminal_previous_soma_outcome_balanced_acc": 0.5775167647179824, + "terminal_residual_minus_previous_soma_acc": 0.4224832352820176, + "terminal_residual_outcome_balanced_acc": 1.0, + "terminal_role_aligned_outcome_separation": 0.4225951132211897, + "terminal_training_events": 896, + "velocity_minus_error_abs_cv_corr": 0.6415311096741272 + }, + "split": "development", + "wall_s": 2.6419225111603737, + "warmup": { + "critic_training_lesion": { + "batches": 100, + "examples": 6400, + "instruction_present": false, + "predictor_max_abs_error": 2.384185791015625e-07, + "rng_seed": 602300, + "role_cosine_after_warmup": 0.9931626915931702, + "role_cursor_scalar_observations": 12800, + "role_update_enabled": true + }, + "fixed_role": { + "batches": 100, + "examples": 6400, + "instruction_present": false, + "predictor_max_abs_error": 2.384185791015625e-07, + "rng_seed": 602300, + "role_cosine_after_warmup": -0.1393236517906189, + "role_cursor_scalar_observations": 12800, + "role_update_enabled": false + }, + "intact": { + "batches": 100, + "examples": 6400, + "instruction_present": false, + "predictor_max_abs_error": 2.384185791015625e-07, + "rng_seed": 602300, + "role_cosine_after_warmup": 0.9931626915931702, + "role_cursor_scalar_observations": 12800, + "role_update_enabled": true + }, + "oracle_role": { + "batches": 100, + "examples": 6400, + "instruction_present": false, + "predictor_max_abs_error": 2.384185791015625e-07, + "rng_seed": 602300, + "role_cosine_after_warmup": 1.0000001192092896, + "role_cursor_scalar_observations": 12800, + "role_update_enabled": false + }, + "outcome_training_lesion": { + "batches": 100, + "examples": 6400, + "instruction_present": false, + "predictor_max_abs_error": 2.384185791015625e-07, + "rng_seed": 602300, + "role_cosine_after_warmup": 0.9931626915931702, + "role_cursor_scalar_observations": 12800, + "role_update_enabled": true + }, + "plasticity_lesion": { + "batches": 100, + "examples": 6400, + "instruction_present": false, + "predictor_max_abs_error": 2.384185791015625e-07, + "rng_seed": 602300, + "role_cosine_after_warmup": 0.9931626915931702, + "role_cursor_scalar_observations": 12800, + "role_update_enabled": true + } + } +} diff --git a/results/bci_v2_recovery_dev/bci_v2_recovery_t24_m0.json b/results/bci_v2_recovery_dev/bci_v2_recovery_t24_m0.json new file mode 100644 index 0000000..9909867 --- /dev/null +++ b/results/bci_v2_recovery_dev/bci_v2_recovery_t24_m0.json @@ -0,0 +1,430 @@ +{ + "args": { + "model_seed": 0, + "outdir": "results/bci_v2_recovery_dev", + "task_seed": 24 + }, + "assays": { + "challenge": { + "active_state_episode_steps_by_mode": { + "acute_critic_lesion": 12158, + "acute_outcome_lesion": 12158, + "intact": 12158 + }, + "episodes_per_target": 128, + "maximum_steps_per_episode": 28, + "selection_over_targets": false, + "targets": [ + 1.55, + 1.6, + 1.65, + 1.7, + 1.75, + 1.8 + ], + "trajectory_seeds": { + "1.55": 494000, + "1.6": 494001, + "1.65": 494002, + "1.7": 494003, + "1.75": 494004, + "1.8": 494005 + } + }, + "performance_evaluation_episodes": 256, + "performance_evaluation_seed": 460024 + }, + "conditions": { + "critic_training_lesion": { + "cost": { + "active_state_episode_steps": 16221, + "cursor_scalar_observations": 8576, + "maximum_state_episode_steps": 25088, + "reverse_mode_calls": 0, + "role_probe_examples": 4288, + "task_loss_queries": 0, + "terminal_outcome_observations": 896 + }, + "critic_l2_after_training": 0.0, + "daily_success": [ + 0.0, + 0.0, + 0.0, + 0.015625, + 0.0, + 0.0, + 0.046875, + 0.140625, + 0.71875, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "early_success": 0.0, + "evaluation_active_state_episode_steps": 256, + "final_success": 1.0, + "late_success": 1.0, + "learning_gain": 1.0, + "role_cosine_after_training": 0.9901587963104248, + "training_wall_s": 0.19473905116319656 + }, + "fixed_role": { + "cost": { + "active_state_episode_steps": 25088, + "cursor_scalar_observations": 12544, + "maximum_state_episode_steps": 25088, + "reverse_mode_calls": 0, + "role_probe_examples": 6272, + "task_loss_queries": 0, + "terminal_outcome_observations": 896 + }, + "critic_l2_after_training": 0.06098082661628723, + "daily_success": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "early_success": 0.0, + "evaluation_active_state_episode_steps": 7168, + "final_success": 0.0, + "late_success": 0.0, + "learning_gain": 0.0, + "role_cosine_after_training": -0.1393236517906189, + "training_wall_s": 0.23979467898607254 + }, + "intact": { + "cost": { + "active_state_episode_steps": 14085, + "cursor_scalar_observations": 7610, + "maximum_state_episode_steps": 25088, + "reverse_mode_calls": 0, + "role_probe_examples": 3805, + "task_loss_queries": 0, + "terminal_outcome_observations": 896 + }, + "critic_l2_after_training": 0.4679003357887268, + "daily_success": [ + 0.0, + 0.0, + 0.0, + 0.015625, + 0.0, + 0.03125, + 0.28125, + 0.875, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "early_success": 0.0, + "evaluation_active_state_episode_steps": 257, + "final_success": 1.0, + "late_success": 1.0, + "learning_gain": 1.0, + "role_cosine_after_training": 0.9919018745422363, + "training_wall_s": 0.27622058615088463 + }, + "oracle_role": { + "cost": { + "active_state_episode_steps": 13623, + "cursor_scalar_observations": 7408, + "maximum_state_episode_steps": 25088, + "reverse_mode_calls": 0, + "role_probe_examples": 3704, + "task_loss_queries": 0, + "terminal_outcome_observations": 896 + }, + "critic_l2_after_training": 0.47546476125717163, + "daily_success": [ + 0.0, + 0.0, + 0.0, + 0.015625, + 0.0, + 0.046875, + 0.375, + 0.984375, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "early_success": 0.0, + "evaluation_active_state_episode_steps": 257, + "final_success": 1.0, + "late_success": 1.0, + "learning_gain": 1.0, + "role_cosine_after_training": 1.0000001192092896, + "training_wall_s": 0.19996124133467674 + }, + "outcome_training_lesion": { + "cost": { + "active_state_episode_steps": 17888, + "cursor_scalar_observations": 9260, + "maximum_state_episode_steps": 25088, + "reverse_mode_calls": 0, + "role_probe_examples": 4630, + "task_loss_queries": 0, + "terminal_outcome_observations": 896 + }, + "critic_l2_after_training": 0.2022681087255478, + "daily_success": [ + 0.0, + 0.0, + 0.0, + 0.015625, + 0.0, + 0.015625, + 0.078125, + 0.234375, + 0.59375, + 0.78125, + 0.984375, + 1.0, + 1.0, + 1.0 + ], + "early_success": 0.0, + "evaluation_active_state_episode_steps": 768, + "final_success": 1.0, + "late_success": 1.0, + "learning_gain": 1.0, + "role_cosine_after_training": 0.9417572021484375, + "training_wall_s": 0.23217474296689034 + }, + "plasticity_lesion": { + "cost": { + "active_state_episode_steps": 25088, + "cursor_scalar_observations": 12544, + "maximum_state_episode_steps": 25088, + "reverse_mode_calls": 0, + "role_probe_examples": 6272, + "task_loss_queries": 0, + "terminal_outcome_observations": 896 + }, + "critic_l2_after_training": 0.062091100960969925, + "daily_success": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "early_success": 0.0, + "evaluation_active_state_episode_steps": 7168, + "final_success": 0.0, + "late_success": 0.0, + "learning_gain": 0.0, + "role_cosine_after_training": 0.9967973232269287, + "training_wall_s": 0.21114370226860046 + } + }, + "config": { + "context_ar": 0.8, + "context_dim": 16, + "coupling_scale": 1.0, + "critic_eta": 0.03, + "days": 14, + "eligibility_decay": 0.8, + "episodes_per_day": 64, + "feedback": "performance_velocity", + "forward_eta": 0.1, + "gamma": 0.8, + "inertia": 0.65, + "kappa": 0.0, + "n_background": 30, + "n_minus": 5, + "n_plus": 5, + "perturb_every": 4, + "perturb_sigma": 0.03, + "predictor_eta": 0.2, + "process_noise": 0.12, + "steps_per_episode": 28, + "target": 0.8, + "terminal_reward": 1.0, + "vectorizer_eta": 0.03, + "velocity_reward_scale": 1.0 + }, + "finite": true, + "hardware": { + "device": "cpu", + "platform": "Linux-5.15.0-161-generic-x86_64-with-glibc2.35", + "threads": 1, + "torch_version": "2.10.0+cu128" + }, + "peak_rss_mib": 996.0546875, + "protocol": { + "confirmation_seeds_touched": false, + "d4_gate_sha256": "636c587e47287338cdca7d9558dc3bfb99a2cec615badb23337025d85b8128ef", + "failed_v2_gate_sha256": "e53f46cf456d60ce4d33586ace4f4619fb1a31a58082b2b72943d4ed10cd25fd", + "fixed_config": { + "critic_eta": 0.03, + "forward_eta": 0.1, + "gamma": 0.8, + "velocity_reward_scale": 1.0 + }, + "hyperparameter_selection": false, + "name": "oral_b_v2_cold_start_recovery_development_v1", + "old_r2_gate_sha256": "4f6f969ceae88afa2523e3472a3373522991f5ecaab69440830c07479d2d3597", + "protocol_sha256": "0621b47367074cb99ee74b332192878e76acbca11678b5238e1312adc2a2bcd9", + "selection_split": "development_validation" + }, + "provenance": { + "git_commit": "dae172b1425c646f0441609c28fc5156b50f820d", + "git_tracked_dirty": false, + "input_sha256": { + "base_dynamics": "d5a373314562af0daedf55baa04b975fe643236c7821be1560d2cfb4359688fc", + "confirmation_analyzer": "29a168143266bc7078e5229183bb7ea45283973592fecd1fcd715488f5d6242b", + "confirmation_runner": "0582ae7a0c173d2fef842ea57d75b77b23cb6f70391f7013ad27ec29e0fedfdf", + "d4_gate": "636c587e47287338cdca7d9558dc3bfb99a2cec615badb23337025d85b8128ef", + "development_analyzer": "2768b877132ece6d612d2a5f1e483c495b277e26fdf54fb8a716aa8de6fd67e2", + "failed_v2_gate": "e53f46cf456d60ce4d33586ace4f4619fb1a31a58082b2b72943d4ed10cd25fd", + "old_r2_gate": "4f6f969ceae88afa2523e3472a3373522991f5ecaab69440830c07479d2d3597", + "protocol": "0621b47367074cb99ee74b332192878e76acbca11678b5238e1312adc2a2bcd9", + "recovery_metrics": "371cb6a7e2192d25ddbfad69c0ce7a1150aa1c21ca98e141bedb2afc2acbdbaa", + "runner": "cea0ef658c698c376b1fda0cf2f7f910ee44f7bcde153f2fbaf18e66a2d3661a", + "v2_dynamics": "786c5aed3091644272ae0a740913ac3cc38cdb02a8cacb6dfc8496e36836cdae", + "v2_metrics": "421da9287b92e62c742f9eeefbfdccba582c6a4ad5cf92ade4709689120fa1ee" + }, + "tracked_inputs": { + "base_dynamics": true, + "confirmation_analyzer": true, + "confirmation_runner": true, + "d4_gate": true, + "development_analyzer": true, + "failed_v2_gate": true, + "old_r2_gate": true, + "protocol": true, + "recovery_metrics": true, + "runner": true, + "v2_dynamics": true, + "v2_metrics": true + } + }, + "schema_version": 3, + "signatures": { + "acute_outcome_lesion_outcome_balanced_acc": 0.5541079585665574, + "acute_outcome_lesion_role_aligned_separation": 0.01978198133532344, + "causal_role_sign_inversion_index": 0.040138957469204, + "challenge_episodes": 768, + "challenge_failure_count": 297, + "challenge_success_count": 471, + "challenge_success_fraction": 0.61328125, + "critic_contribution_value_prediction_corr": 0.9999999999991358, + "decoder_distance_residual_corr": 0.09986939764391377, + "mean_abs_raw_soma_corr": 0.9991346687315854, + "mean_abs_residual_soma_corr": 0.05678144772056838, + "mean_critic_expectedness_contribution": 0.322402930318503, + "nonterminal_training_events": 13189, + "raw_minus_residual_abs_soma_corr": 0.942353221011017, + "role_aligned_error_cv_corr": 0.352382055212796, + "role_aligned_velocity_cv_corr": 0.9984807554798858, + "surrounding_event_decoder_balanced_acc": 0.5451291756686398, + "target_success_fraction": { + "1.55": 1.0, + "1.6": 0.9921875, + "1.65": 0.9296875, + "1.7": 0.5625, + "1.75": 0.1875, + "1.8": 0.0078125 + }, + "terminal_outcome_separation_drop_under_acute_lesion": 0.40220903209568243, + "terminal_previous_soma_outcome_balanced_acc": 0.5670719938235862, + "terminal_residual_minus_previous_soma_acc": 0.4329280061764138, + "terminal_residual_outcome_balanced_acc": 1.0, + "terminal_role_aligned_outcome_separation": 0.42199101343100587, + "terminal_training_events": 896, + "velocity_minus_error_abs_cv_corr": 0.6460987002670898 + }, + "split": "development", + "wall_s": 2.6129846908152103, + "warmup": { + "critic_training_lesion": { + "batches": 100, + "examples": 6400, + "instruction_present": false, + "predictor_max_abs_error": 2.384185791015625e-07, + "rng_seed": 602400, + "role_cosine_after_warmup": 0.9931948781013489, + "role_cursor_scalar_observations": 12800, + "role_update_enabled": true + }, + "fixed_role": { + "batches": 100, + "examples": 6400, + "instruction_present": false, + "predictor_max_abs_error": 2.384185791015625e-07, + "rng_seed": 602400, + "role_cosine_after_warmup": -0.1393236517906189, + "role_cursor_scalar_observations": 12800, + "role_update_enabled": false + }, + "intact": { + "batches": 100, + "examples": 6400, + "instruction_present": false, + "predictor_max_abs_error": 2.384185791015625e-07, + "rng_seed": 602400, + "role_cosine_after_warmup": 0.9931948781013489, + "role_cursor_scalar_observations": 12800, + "role_update_enabled": true + }, + "oracle_role": { + "batches": 100, + "examples": 6400, + "instruction_present": false, + "predictor_max_abs_error": 2.384185791015625e-07, + "rng_seed": 602400, + "role_cosine_after_warmup": 1.0000001192092896, + "role_cursor_scalar_observations": 12800, + "role_update_enabled": false + }, + "outcome_training_lesion": { + "batches": 100, + "examples": 6400, + "instruction_present": false, + "predictor_max_abs_error": 2.384185791015625e-07, + "rng_seed": 602400, + "role_cosine_after_warmup": 0.9931948781013489, + "role_cursor_scalar_observations": 12800, + "role_update_enabled": true + }, + "plasticity_lesion": { + "batches": 100, + "examples": 6400, + "instruction_present": false, + "predictor_max_abs_error": 2.384185791015625e-07, + "rng_seed": 602400, + "role_cosine_after_warmup": 0.9931948781013489, + "role_cursor_scalar_observations": 12800, + "role_update_enabled": true + } + } +} diff --git a/results/bci_v2_recovery_dev/bci_v2_recovery_t25_m0.json b/results/bci_v2_recovery_dev/bci_v2_recovery_t25_m0.json new file mode 100644 index 0000000..6636e31 --- /dev/null +++ b/results/bci_v2_recovery_dev/bci_v2_recovery_t25_m0.json @@ -0,0 +1,430 @@ +{ + "args": { + "model_seed": 0, + "outdir": "results/bci_v2_recovery_dev", + "task_seed": 25 + }, + "assays": { + "challenge": { + "active_state_episode_steps_by_mode": { + "acute_critic_lesion": 3479, + "acute_outcome_lesion": 3479, + "intact": 3479 + }, + "episodes_per_target": 128, + "maximum_steps_per_episode": 28, + "selection_over_targets": false, + "targets": [ + 1.55, + 1.6, + 1.65, + 1.7, + 1.75, + 1.8 + ], + "trajectory_seeds": { + "1.55": 495000, + "1.6": 495001, + "1.65": 495002, + "1.7": 495003, + "1.75": 495004, + "1.8": 495005 + } + }, + "performance_evaluation_episodes": 256, + "performance_evaluation_seed": 460025 + }, + "conditions": { + "critic_training_lesion": { + "cost": { + "active_state_episode_steps": 16322, + "cursor_scalar_observations": 8606, + "maximum_state_episode_steps": 25088, + "reverse_mode_calls": 0, + "role_probe_examples": 4303, + "task_loss_queries": 0, + "terminal_outcome_observations": 896 + }, + "critic_l2_after_training": 0.0, + "daily_success": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.203125, + 0.578125, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "early_success": 0.0, + "evaluation_active_state_episode_steps": 256, + "final_success": 1.0, + "late_success": 1.0, + "learning_gain": 1.0, + "role_cosine_after_training": 0.956307053565979, + "training_wall_s": 0.19160666316747665 + }, + "fixed_role": { + "cost": { + "active_state_episode_steps": 25088, + "cursor_scalar_observations": 12544, + "maximum_state_episode_steps": 25088, + "reverse_mode_calls": 0, + "role_probe_examples": 6272, + "task_loss_queries": 0, + "terminal_outcome_observations": 896 + }, + "critic_l2_after_training": 0.062333203852176666, + "daily_success": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "early_success": 0.0, + "evaluation_active_state_episode_steps": 7168, + "final_success": 0.0, + "late_success": 0.0, + "learning_gain": 0.0, + "role_cosine_after_training": -0.1393236517906189, + "training_wall_s": 0.22667565941810608 + }, + "intact": { + "cost": { + "active_state_episode_steps": 13411, + "cursor_scalar_observations": 7336, + "maximum_state_episode_steps": 25088, + "reverse_mode_calls": 0, + "role_probe_examples": 3668, + "task_loss_queries": 0, + "terminal_outcome_observations": 896 + }, + "critic_l2_after_training": 0.5300180912017822, + "daily_success": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.046875, + 0.109375, + 0.515625, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "early_success": 0.0, + "evaluation_active_state_episode_steps": 256, + "final_success": 1.0, + "late_success": 1.0, + "learning_gain": 1.0, + "role_cosine_after_training": 0.9723086357116699, + "training_wall_s": 0.25713305175304413 + }, + "oracle_role": { + "cost": { + "active_state_episode_steps": 12800, + "cursor_scalar_observations": 7046, + "maximum_state_episode_steps": 25088, + "reverse_mode_calls": 0, + "role_probe_examples": 3523, + "task_loss_queries": 0, + "terminal_outcome_observations": 896 + }, + "critic_l2_after_training": 0.5326098799705505, + "daily_success": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.046875, + 0.21875, + 0.703125, + 0.984375, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "early_success": 0.0, + "evaluation_active_state_episode_steps": 257, + "final_success": 1.0, + "late_success": 1.0, + "learning_gain": 1.0, + "role_cosine_after_training": 1.0000001192092896, + "training_wall_s": 0.19247549399733543 + }, + "outcome_training_lesion": { + "cost": { + "active_state_episode_steps": 18794, + "cursor_scalar_observations": 9658, + "maximum_state_episode_steps": 25088, + "reverse_mode_calls": 0, + "role_probe_examples": 4829, + "task_loss_queries": 0, + "terminal_outcome_observations": 896 + }, + "critic_l2_after_training": 0.23738987743854523, + "daily_success": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.046875, + 0.0625, + 0.09375, + 0.390625, + 0.53125, + 0.78125, + 0.84375, + 0.84375, + 0.96875, + 1.0 + ], + "early_success": 0.0, + "evaluation_active_state_episode_steps": 836, + "final_success": 1.0, + "late_success": 0.9375, + "learning_gain": 0.9375, + "role_cosine_after_training": 0.9799865484237671, + "training_wall_s": 0.22869668155908585 + }, + "plasticity_lesion": { + "cost": { + "active_state_episode_steps": 25088, + "cursor_scalar_observations": 12544, + "maximum_state_episode_steps": 25088, + "reverse_mode_calls": 0, + "role_probe_examples": 6272, + "task_loss_queries": 0, + "terminal_outcome_observations": 896 + }, + "critic_l2_after_training": 0.06321071833372116, + "daily_success": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "early_success": 0.0, + "evaluation_active_state_episode_steps": 7168, + "final_success": 0.0, + "late_success": 0.0, + "learning_gain": 0.0, + "role_cosine_after_training": 0.9956421852111816, + "training_wall_s": 0.20200646668672562 + } + }, + "config": { + "context_ar": 0.8, + "context_dim": 16, + "coupling_scale": 1.0, + "critic_eta": 0.03, + "days": 14, + "eligibility_decay": 0.8, + "episodes_per_day": 64, + "feedback": "performance_velocity", + "forward_eta": 0.1, + "gamma": 0.8, + "inertia": 0.65, + "kappa": 0.0, + "n_background": 30, + "n_minus": 5, + "n_plus": 5, + "perturb_every": 4, + "perturb_sigma": 0.03, + "predictor_eta": 0.2, + "process_noise": 0.12, + "steps_per_episode": 28, + "target": 0.8, + "terminal_reward": 1.0, + "vectorizer_eta": 0.03, + "velocity_reward_scale": 1.0 + }, + "finite": true, + "hardware": { + "device": "cpu", + "platform": "Linux-5.15.0-161-generic-x86_64-with-glibc2.35", + "threads": 1, + "torch_version": "2.10.0+cu128" + }, + "peak_rss_mib": 994.16015625, + "protocol": { + "confirmation_seeds_touched": false, + "d4_gate_sha256": "636c587e47287338cdca7d9558dc3bfb99a2cec615badb23337025d85b8128ef", + "failed_v2_gate_sha256": "e53f46cf456d60ce4d33586ace4f4619fb1a31a58082b2b72943d4ed10cd25fd", + "fixed_config": { + "critic_eta": 0.03, + "forward_eta": 0.1, + "gamma": 0.8, + "velocity_reward_scale": 1.0 + }, + "hyperparameter_selection": false, + "name": "oral_b_v2_cold_start_recovery_development_v1", + "old_r2_gate_sha256": "4f6f969ceae88afa2523e3472a3373522991f5ecaab69440830c07479d2d3597", + "protocol_sha256": "0621b47367074cb99ee74b332192878e76acbca11678b5238e1312adc2a2bcd9", + "selection_split": "development_validation" + }, + "provenance": { + "git_commit": "dae172b1425c646f0441609c28fc5156b50f820d", + "git_tracked_dirty": false, + "input_sha256": { + "base_dynamics": "d5a373314562af0daedf55baa04b975fe643236c7821be1560d2cfb4359688fc", + "confirmation_analyzer": "29a168143266bc7078e5229183bb7ea45283973592fecd1fcd715488f5d6242b", + "confirmation_runner": "0582ae7a0c173d2fef842ea57d75b77b23cb6f70391f7013ad27ec29e0fedfdf", + "d4_gate": "636c587e47287338cdca7d9558dc3bfb99a2cec615badb23337025d85b8128ef", + "development_analyzer": "2768b877132ece6d612d2a5f1e483c495b277e26fdf54fb8a716aa8de6fd67e2", + "failed_v2_gate": "e53f46cf456d60ce4d33586ace4f4619fb1a31a58082b2b72943d4ed10cd25fd", + "old_r2_gate": "4f6f969ceae88afa2523e3472a3373522991f5ecaab69440830c07479d2d3597", + "protocol": "0621b47367074cb99ee74b332192878e76acbca11678b5238e1312adc2a2bcd9", + "recovery_metrics": "371cb6a7e2192d25ddbfad69c0ce7a1150aa1c21ca98e141bedb2afc2acbdbaa", + "runner": "cea0ef658c698c376b1fda0cf2f7f910ee44f7bcde153f2fbaf18e66a2d3661a", + "v2_dynamics": "786c5aed3091644272ae0a740913ac3cc38cdb02a8cacb6dfc8496e36836cdae", + "v2_metrics": "421da9287b92e62c742f9eeefbfdccba582c6a4ad5cf92ade4709689120fa1ee" + }, + "tracked_inputs": { + "base_dynamics": true, + "confirmation_analyzer": true, + "confirmation_runner": true, + "d4_gate": true, + "development_analyzer": true, + "failed_v2_gate": true, + "old_r2_gate": true, + "protocol": true, + "recovery_metrics": true, + "runner": true, + "v2_dynamics": true, + "v2_metrics": true + } + }, + "schema_version": 3, + "signatures": { + "acute_outcome_lesion_outcome_balanced_acc": 0.6177631578947369, + "acute_outcome_lesion_role_aligned_separation": 0.08405720592296978, + "causal_role_sign_inversion_index": 0.03978891586591343, + "challenge_episodes": 768, + "challenge_failure_count": 8, + "challenge_success_count": 760, + "challenge_success_fraction": 0.9895833333333334, + "critic_contribution_value_prediction_corr": 0.9999999999980205, + "decoder_distance_residual_corr": 0.14104127012712814, + "mean_abs_raw_soma_corr": 0.9991423341990776, + "mean_abs_residual_soma_corr": 0.05642313233065083, + "mean_critic_expectedness_contribution": 0.23241959506021517, + "nonterminal_training_events": 12515, + "raw_minus_residual_abs_soma_corr": 0.9427192018684267, + "role_aligned_error_cv_corr": 0.3284602854835664, + "role_aligned_velocity_cv_corr": 0.9987698528051169, + "surrounding_event_decoder_balanced_acc": 0.5518022758130953, + "target_success_fraction": { + "1.55": 1.0, + "1.6": 1.0, + "1.65": 1.0, + "1.7": 1.0, + "1.75": 1.0, + "1.8": 0.9375 + }, + "terminal_outcome_separation_drop_under_acute_lesion": 0.39362157579221996, + "terminal_previous_soma_outcome_balanced_acc": 0.46842105263157896, + "terminal_residual_minus_previous_soma_acc": 0.531578947368421, + "terminal_residual_outcome_balanced_acc": 1.0, + "terminal_role_aligned_outcome_separation": 0.47767878171518974, + "terminal_training_events": 896, + "velocity_minus_error_abs_cv_corr": 0.6703095673215504 + }, + "split": "development", + "wall_s": 2.5251883156597614, + "warmup": { + "critic_training_lesion": { + "batches": 100, + "examples": 6400, + "instruction_present": false, + "predictor_max_abs_error": 2.384185791015625e-07, + "rng_seed": 602500, + "role_cosine_after_warmup": 0.9958859086036682, + "role_cursor_scalar_observations": 12800, + "role_update_enabled": true + }, + "fixed_role": { + "batches": 100, + "examples": 6400, + "instruction_present": false, + "predictor_max_abs_error": 2.384185791015625e-07, + "rng_seed": 602500, + "role_cosine_after_warmup": -0.1393236517906189, + "role_cursor_scalar_observations": 12800, + "role_update_enabled": false + }, + "intact": { + "batches": 100, + "examples": 6400, + "instruction_present": false, + "predictor_max_abs_error": 2.384185791015625e-07, + "rng_seed": 602500, + "role_cosine_after_warmup": 0.9958859086036682, + "role_cursor_scalar_observations": 12800, + "role_update_enabled": true + }, + "oracle_role": { + "batches": 100, + "examples": 6400, + "instruction_present": false, + "predictor_max_abs_error": 2.384185791015625e-07, + "rng_seed": 602500, + "role_cosine_after_warmup": 1.0000001192092896, + "role_cursor_scalar_observations": 12800, + "role_update_enabled": false + }, + "outcome_training_lesion": { + "batches": 100, + "examples": 6400, + "instruction_present": false, + "predictor_max_abs_error": 2.384185791015625e-07, + "rng_seed": 602500, + "role_cosine_after_warmup": 0.9958859086036682, + "role_cursor_scalar_observations": 12800, + "role_update_enabled": true + }, + "plasticity_lesion": { + "batches": 100, + "examples": 6400, + "instruction_present": false, + "predictor_max_abs_error": 2.384185791015625e-07, + "rng_seed": 602500, + "role_cosine_after_warmup": 0.9958859086036682, + "role_cursor_scalar_observations": 12800, + "role_update_enabled": true + } + } +} diff --git a/results/bci_v2_recovery_dev_gate.json b/results/bci_v2_recovery_dev_gate.json new file mode 100644 index 0000000..b2ac485 --- /dev/null +++ b/results/bci_v2_recovery_dev_gate.json @@ -0,0 +1,113 @@ +{ + "challenge_success_fraction_by_task_seed": [ + 0.7096354166666666, + 0.61328125, + 0.9895833333333334 + ], + "checks_by_task_seed": { + "23": { + "challenge_fraction_between_0p15_and_0p85": true, + "critic_contribution_value_corr_at_least_0p95": true, + "critic_expectedness_at_least_0p05": true, + "decoder_distance_corr_at_least_0p02": true, + "final_success_at_least_0p70": true, + "fixed_role_gap_at_least_0p20": true, + "learning_gain_at_least_0p10": true, + "oracle_deficit_at_most_0p10": true, + "outcome_lesion_separation_drop_at_least_0p20": true, + "plasticity_lesion_retains_at_most_half_gain": true, + "raw_residual_corr_gap_at_least_0p20": true, + "residual_soma_corr_at_most_0p10": true, + "role_cosine_at_least_0p80": true, + "sign_inversion_at_least_0p01": true, + "surrounding_event_accuracy_at_least_0p52": true, + "terminal_outcome_accuracy_at_least_0p75": true, + "terminal_role_separation_at_least_0p20": true, + "velocity_advantage_at_least_0p05": true + }, + "24": { + "challenge_fraction_between_0p15_and_0p85": true, + "critic_contribution_value_corr_at_least_0p95": true, + "critic_expectedness_at_least_0p05": true, + "decoder_distance_corr_at_least_0p02": true, + "final_success_at_least_0p70": true, + "fixed_role_gap_at_least_0p20": true, + "learning_gain_at_least_0p10": true, + "oracle_deficit_at_most_0p10": true, + "outcome_lesion_separation_drop_at_least_0p20": true, + "plasticity_lesion_retains_at_most_half_gain": true, + "raw_residual_corr_gap_at_least_0p20": true, + "residual_soma_corr_at_most_0p10": true, + "role_cosine_at_least_0p80": true, + "sign_inversion_at_least_0p01": true, + "surrounding_event_accuracy_at_least_0p52": true, + "terminal_outcome_accuracy_at_least_0p75": true, + "terminal_role_separation_at_least_0p20": true, + "velocity_advantage_at_least_0p05": true + }, + "25": { + "challenge_fraction_between_0p15_and_0p85": false, + "critic_contribution_value_corr_at_least_0p95": true, + "critic_expectedness_at_least_0p05": true, + "decoder_distance_corr_at_least_0p02": true, + "final_success_at_least_0p70": true, + "fixed_role_gap_at_least_0p20": true, + "learning_gain_at_least_0p10": true, + "oracle_deficit_at_most_0p10": true, + "outcome_lesion_separation_drop_at_least_0p20": true, + "plasticity_lesion_retains_at_most_half_gain": true, + "raw_residual_corr_gap_at_least_0p20": true, + "residual_soma_corr_at_most_0p10": true, + "role_cosine_at_least_0p80": true, + "sign_inversion_at_least_0p01": true, + "surrounding_event_accuracy_at_least_0p52": true, + "terminal_outcome_accuracy_at_least_0p75": true, + "terminal_role_separation_at_least_0p20": true, + "velocity_advantage_at_least_0p05": true + } + }, + "complete_grid": true, + "confirmation_seeds_touched": false, + "fixed_config": { + "critic_eta": 0.03, + "forward_eta": 0.1, + "gamma": 0.8, + "velocity_reward_scale": 1.0 + }, + "input_sha256": { + "base_dynamics": "d5a373314562af0daedf55baa04b975fe643236c7821be1560d2cfb4359688fc", + "confirmation_analyzer": "29a168143266bc7078e5229183bb7ea45283973592fecd1fcd715488f5d6242b", + "confirmation_runner": "0582ae7a0c173d2fef842ea57d75b77b23cb6f70391f7013ad27ec29e0fedfdf", + "d4_gate": "636c587e47287338cdca7d9558dc3bfb99a2cec615badb23337025d85b8128ef", + "development_analyzer": "2768b877132ece6d612d2a5f1e483c495b277e26fdf54fb8a716aa8de6fd67e2", + "failed_v2_gate": "e53f46cf456d60ce4d33586ace4f4619fb1a31a58082b2b72943d4ed10cd25fd", + "old_r2_gate": "4f6f969ceae88afa2523e3472a3373522991f5ecaab69440830c07479d2d3597", + "protocol": "0621b47367074cb99ee74b332192878e76acbca11678b5238e1312adc2a2bcd9", + "recovery_metrics": "371cb6a7e2192d25ddbfad69c0ce7a1150aa1c21ca98e141bedb2afc2acbdbaa", + "runner": "cea0ef658c698c376b1fda0cf2f7f910ee44f7bcde153f2fbaf18e66a2d3661a", + "v2_dynamics": "786c5aed3091644272ae0a740913ac3cc38cdb02a8cacb6dfc8496e36836cdae", + "v2_metrics": "421da9287b92e62c742f9eeefbfdccba582c6a4ad5cf92ade4709689120fa1ee" + }, + "intact_final_by_task_seed": [ + 1.0, + 1.0, + 1.0 + ], + "protocol": "oral_b_v2_cold_start_recovery_development_v1", + "recovery_confirmation_opened": false, + "review_score_after": 7, + "review_score_before": 7, + "score_change_rule": "development validation never changes the formal score", + "source_commit": "dae172b1425c646f0441609c28fc5156b50f820d", + "source_sha256": { + "results/bci_v2_recovery_dev/bci_v2_recovery_t23_m0.json": "2879ac2b3ea6a37213556add0aa042c8c9d122e3ddf8685ff3bfbcae1d8e5eea", + "results/bci_v2_recovery_dev/bci_v2_recovery_t24_m0.json": "c5e25afcd11be4108682476a54b84350b68b035511b4bafce00d5ffbe193bef3", + "results/bci_v2_recovery_dev/bci_v2_recovery_t25_m0.json": "5710f5e1ba2656415b5806dc5d94f029aed3187505fbee7b5589ddd5eddce7dd" + }, + "status": "failed", + "terminal_outcome_accuracy_by_task_seed": [ + 1.0, + 1.0, + 1.0 + ] +} |
