diff options
| author | YurenHao0426 <Blackhao0426@gmail.com> | 2026-07-23 08:12:22 -0500 |
|---|---|---|
| committer | YurenHao0426 <Blackhao0426@gmail.com> | 2026-07-23 08:12:22 -0500 |
| commit | d289a46293e0b422c195746ad8b310d4b50ded23 (patch) | |
| tree | 1b5e7e57ca59ae32633a9ebc14bbfab886db7092 /experiments | |
| parent | 378e68dc424acb6b5a2082bc071314ce810d8ab5 (diff) | |
protocol: freeze calibration-split oral-B-v2 recovery
Diffstat (limited to 'experiments')
| -rw-r--r-- | experiments/analyze_bci_v2_calibrated_confirmation.py | 363 | ||||
| -rw-r--r-- | experiments/analyze_bci_v2_calibrated_development.py | 217 | ||||
| -rw-r--r-- | experiments/bci_v2_calibrated_confirmation.py | 143 | ||||
| -rw-r--r-- | experiments/bci_v2_calibrated_confirmation.sh | 16 | ||||
| -rw-r--r-- | experiments/bci_v2_calibrated_development.sh | 12 | ||||
| -rw-r--r-- | experiments/bci_v2_calibrated_run.py | 457 | ||||
| -rw-r--r-- | experiments/bci_v2_calibrated_smoke.py | 36 |
7 files changed, 1244 insertions, 0 deletions
diff --git a/experiments/analyze_bci_v2_calibrated_confirmation.py b/experiments/analyze_bci_v2_calibrated_confirmation.py new file mode 100644 index 0000000..9fd5eaf --- /dev/null +++ b/experiments/analyze_bci_v2_calibrated_confirmation.py @@ -0,0 +1,363 @@ +#!/usr/bin/env python3 +"""Audit untouched calibration-split oral-B-v2 confirmation.""" +import argparse +import glob +import hashlib +import json +import math +import os + +from experiments.analyze_bci_v2_recovery_confirmation import ( + checks, + summarize, + task_clusters, +) +from experiments.bci_v2_calibrated_confirmation import ( + MODEL_SEEDS, + TASK_SEEDS, +) +from experiments.bci_v2_calibrated_run import ( + CALIBRATION_EPISODES, + CHALLENGE_EPISODES, + TARGET_QUANTILES, + build_recovery_config, + source_paths, +) +from experiments.bci_v2_recovery_run import FIXED_CONFIG +from experiments.bci_v2_run import CONDITIONS + + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +RUNNER_PATH = os.path.join( + ROOT, "experiments", "bci_v2_calibrated_confirmation.py" +) + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def sha256(path): + digest = hashlib.sha256() + with open(path, "rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def finite_tree(value): + if isinstance(value, dict): + return all(finite_tree(item) for item in value.values()) + if isinstance(value, (list, tuple)): + return all(finite_tree(item) for item in value) + if isinstance(value, (int, float)): + return math.isfinite(value) + return True + + +def confirmation_paths(r1_gate): + paths = source_paths() + paths["development_runner"] = paths["runner"] + paths["runner"] = RUNNER_PATH + paths["r1_gate"] = os.path.abspath(r1_gate) + return paths + + +def validate(row, path, digests): + require(row.get("schema_version") == 4, f"{path}: schema") + args = row.get("args", {}) + task_seed = args.get("task_seed") + model_seed = args.get("model_seed") + require( + task_seed in TASK_SEEDS and model_seed in MODEL_SEEDS, + f"{path}: seeds", + ) + protocol = row.get("protocol", {}) + require( + protocol.get("name") + == "oral_b_v2_calibrated_recovery_confirmation_v1" + and protocol.get("split") == "untouched_confirmation" + and protocol.get("training_task_seed") == task_seed + and protocol.get("model_seed") == model_seed + and protocol.get("fixed_config") == FIXED_CONFIG + and protocol.get("calibration_uses_outcome_labels") is False + and protocol.get("no_further_selection") is True + and protocol.get("confirmation_grid_size") == 30 + and protocol.get("protocol_sha256") == digests["protocol"] + and protocol.get("r1_gate_sha256") == digests["r1_gate"], + f"{path}: protocol", + ) + provenance = row.get("provenance", {}) + require( + provenance.get("git_tracked_dirty") is False + and provenance.get("tracked_inputs") + and all(provenance["tracked_inputs"].values()) + and set(provenance["tracked_inputs"]) == set(digests) + and provenance.get("input_sha256") == digests, + f"{path}: provenance", + ) + require( + row.get("config") == vars(build_recovery_config()) + and row.get("split") == "untouched_confirmation" + and row.get("finite") is True + and finite_tree(row), + f"{path}: finite/config", + ) + require( + set(row.get("conditions", {})) == set(CONDITIONS), + f"{path}: conditions", + ) + for name in CONDITIONS: + warmup = row["warmup"][name] + condition = row["conditions"][name] + cost = condition["cost"] + require( + warmup["batches"] == 100 + and warmup["examples"] == 6400 + and warmup["instruction_present"] is False + and warmup["role_cursor_scalar_observations"] == 12800 + and warmup["predictor_max_abs_error"] <= 1e-5 + and len(condition["daily_success"]) == 14 + and cost["maximum_state_episode_steps"] == 25088 + and cost["cursor_scalar_observations"] + == 2 * cost["role_probe_examples"] + and cost["task_loss_queries"] == 0 + and cost["reverse_mode_calls"] == 0, + f"{path}: condition {name}", + ) + challenge = row["assays"]["challenge"] + calibration = challenge["calibration"] + targets = challenge["targets"] + require( + calibration["episodes"] == CALIBRATION_EPISODES + and calibration["quantiles"] == list(TARGET_QUANTILES) + and calibration["targets"] == targets + and calibration["uses_outcome_labels"] is False + and calibration["seed"] == 600_000 + task_seed + and challenge["episodes_per_target"] == CHALLENGE_EPISODES + and challenge["selection_over_evaluation_outcomes"] is False + and challenge["maximum_steps_per_episode"] == 28 + and len(targets) == len(TARGET_QUANTILES) + and all( + targets[index] < targets[index + 1] + for index in range(len(targets) - 1) + ) + and row["signatures"]["challenge_episodes"] + == len(TARGET_QUANTILES) * CHALLENGE_EPISODES, + f"{path}: calibration", + ) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--results", + default="results/bci_v2_calibrated_confirmation", + ) + parser.add_argument( + "--r1-gate", + default="results/bci_v2_calibrated_dev_gate.json", + ) + parser.add_argument( + "--out", + default="results/bci_v2_calibrated_confirmation_gate.json", + ) + args = parser.parse_args() + with open(args.r1_gate) as handle: + r1 = json.load(handle) + require( + r1.get("protocol") + == "oral_b_v2_calibrated_recovery_development_v1" + and r1.get("status") == "passed" + and r1.get("complete_grid") is True + and r1.get("fixed_config") == FIXED_CONFIG + and r1.get("confirmation_seeds_touched") is False + and r1.get("calibrated_confirmation_opened") is True, + "calibrated R1 gate", + ) + development_digests = { + name: sha256(path) + for name, path in source_paths().items() + } + require( + r1.get("input_sha256") == development_digests, + "R1 source binding", + ) + digests = { + name: sha256(path) + for name, path in confirmation_paths(args.r1_gate).items() + } + expected = { + f"bci_v2_calibrated_confirm_t{task_seed}_m{model_seed}.json" + for task_seed in TASK_SEEDS + for model_seed in MODEL_SEEDS + } + observed = { + os.path.basename(path) + for path in glob.glob(os.path.join(args.results, "*.json")) + } + require( + observed == expected, + ( + f"confirmation drift: missing={sorted(expected-observed)}, " + f"extra={sorted(observed-expected)}" + ), + ) + records = {} + commits = set() + source_sha256 = {} + for task_seed in TASK_SEEDS: + for model_seed in MODEL_SEEDS: + path = os.path.join( + args.results, + ( + f"bci_v2_calibrated_confirm_t{task_seed}" + f"_m{model_seed}.json" + ), + ) + with open(path) as handle: + row = json.load(handle) + validate(row, path, digests) + records[(task_seed, model_seed)] = row + commits.add(row["provenance"]["git_commit"]) + source_sha256[path] = sha256(path) + require(len(commits) == 1, "confirmation source revision drift") + + getters = { + "intact_final": lambda row: + row["conditions"]["intact"]["final_success"], + "intact_gain": lambda row: + row["conditions"]["intact"]["learning_gain"], + "fixed_role_gap": lambda row: ( + row["conditions"]["intact"]["final_success"] + - row["conditions"]["fixed_role"]["final_success"] + ), + "oracle_deficit": lambda row: ( + row["conditions"]["oracle_role"]["final_success"] + - row["conditions"]["intact"]["final_success"] + ), + "plasticity_half_margin": lambda row: ( + 0.5 * row["conditions"]["intact"]["learning_gain"] + - row["conditions"]["plasticity_lesion"]["learning_gain"] + ), + "role_cosine": lambda row: + row["conditions"]["intact"]["role_cosine_after_training"], + "critic_training_gap": lambda row: ( + row["conditions"]["intact"]["final_success"] + - row["conditions"]["critic_training_lesion"][ + "final_success" + ] + ), + "outcome_training_gap": lambda row: ( + row["conditions"]["intact"]["final_success"] + - row["conditions"]["outcome_training_lesion"][ + "final_success" + ] + ), + "residual_soma_corr": lambda row: + row["signatures"]["mean_abs_residual_soma_corr"], + "raw_residual_corr_gap": lambda row: + row["signatures"][ + "raw_minus_residual_abs_soma_corr" + ], + "surrounding_accuracy": lambda row: + row["signatures"][ + "surrounding_event_decoder_balanced_acc" + ], + "decoder_corr": lambda row: + row["signatures"]["decoder_distance_residual_corr"], + "sign_inversion": lambda row: + row["signatures"][ + "causal_role_sign_inversion_index" + ], + "velocity_advantage": lambda row: + row["signatures"][ + "velocity_minus_error_abs_cv_corr" + ], + "challenge_fraction": lambda row: + row["signatures"]["challenge_success_fraction"], + "terminal_accuracy": lambda row: + row["signatures"][ + "terminal_residual_outcome_balanced_acc" + ], + "terminal_separation": lambda row: + row["signatures"][ + "terminal_role_aligned_outcome_separation" + ], + "outcome_lesion_drop": lambda row: + row["signatures"][ + "terminal_outcome_separation_drop_under_acute_lesion" + ], + "critic_expectedness": lambda row: + row["signatures"][ + "mean_critic_expectedness_contribution" + ], + "critic_value_corr": lambda row: + row["signatures"][ + "critic_contribution_value_prediction_corr" + ], + } + clustered = { + name: task_clusters(records, getter) + for name, getter in getters.items() + } + metrics = { + name: summarize(values) + for name, values in clustered.items() + } + all_values = { + name: [getter(row) for row in records.values()] + for name, getter in getters.items() + } + gate_checks = checks(metrics, clustered, all_values) + passed = all( + value + for category in gate_checks.values() + for value in ( + [category] + if isinstance(category, bool) + else category.values() + ) + ) + output = { + "protocol": + "oral_b_v2_calibrated_recovery_confirmation_v1", + "status": "passed" if passed else "failed", + "complete_grid": True, + "fixed_config": FIXED_CONFIG, + "calibration_quantiles": list(TARGET_QUANTILES), + "checks": gate_checks, + "metrics": metrics, + "positive_sign_count": sum( + value > 0 for value in all_values["sign_inversion"] + ), + "source_commit": next(iter(commits)), + "source_sha256": source_sha256, + "input_sha256": digests, + "oral_b_v2_outcome_surprise_established": passed, + "all_prior_failures_preserved": True, + "old_oral_a_gate_remains_closed": True, + "new_oral_a_v2_protocol_may_be_frozen": passed, + "review_score_before": 7, + "review_score_after": 8 if passed else 7, + "score_change_rule": ( + "only a complete untouched calibrated R2 pass establishes " + "role-vectorized TD outcome surprise; prior failures and " + "the old oral-A gate remain unchanged" + ), + } + os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) + if os.path.exists(args.out): + with open(args.out) as handle: + existing = json.load(handle) + require(existing == output, "existing calibrated R2 gate differs") + else: + with open(args.out, "w") as handle: + json.dump(output, handle, indent=2, sort_keys=True) + handle.write("\n") + print(json.dumps(output, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/experiments/analyze_bci_v2_calibrated_development.py b/experiments/analyze_bci_v2_calibrated_development.py new file mode 100644 index 0000000..e8781d6 --- /dev/null +++ b/experiments/analyze_bci_v2_calibrated_development.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 +"""Audit calibration-split oral-B-v2 development validation.""" +import argparse +import glob +import hashlib +import json +import os + +from experiments.analyze_bci_v2_recovery_development import checks +from experiments.bci_v2_calibrated_run import ( + CALIBRATION_EPISODES, + CHALLENGE_EPISODES, + MODEL_SEEDS, + TARGET_QUANTILES, + TASK_SEEDS, + build_recovery_config, + source_paths, +) +from experiments.bci_v2_recovery_run import FIXED_CONFIG +from experiments.bci_v2_run import CONDITIONS + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def sha256(path): + digest = hashlib.sha256() + with open(path, "rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def validate(row, path, task_seed, digests): + require(row.get("schema_version") == 4, f"{path}: schema") + require( + row.get("args", {}).get("task_seed") == task_seed + and row["args"].get("model_seed") in MODEL_SEEDS, + f"{path}: seeds", + ) + protocol = row.get("protocol", {}) + require( + protocol.get("name") + == "oral_b_v2_calibrated_recovery_development_v1" + and protocol.get("selection_split") == "development_validation" + and protocol.get("hyperparameter_selection") is False + and protocol.get("confirmation_seeds_touched") is False + and protocol.get("fixed_config") == FIXED_CONFIG + and protocol.get("calibration_uses_outcome_labels") is False + and protocol.get("protocol_sha256") == digests["protocol"] + and protocol.get("failed_target_gate_sha256") + == digests["failed_target_gate"], + f"{path}: protocol", + ) + provenance = row.get("provenance", {}) + require( + provenance.get("git_tracked_dirty") is False + and provenance.get("tracked_inputs") + and all(provenance["tracked_inputs"].values()) + and set(provenance["tracked_inputs"]) == set(digests) + and provenance.get("input_sha256") == digests, + f"{path}: provenance", + ) + require( + row.get("config") == vars(build_recovery_config()) + and row.get("split") == "development_validation" + and row.get("finite") is True, + f"{path}: config/split", + ) + require( + set(row.get("conditions", {})) == set(CONDITIONS), + f"{path}: conditions", + ) + for name in CONDITIONS: + warmup = row["warmup"][name] + condition = row["conditions"][name] + cost = condition["cost"] + require( + warmup["batches"] == 100 + and warmup["examples"] == 6400 + and warmup["instruction_present"] is False + and warmup["role_cursor_scalar_observations"] == 12800 + and warmup["predictor_max_abs_error"] <= 1e-5 + and len(condition["daily_success"]) == 14 + and cost["maximum_state_episode_steps"] == 25088 + and cost["cursor_scalar_observations"] + == 2 * cost["role_probe_examples"] + and cost["task_loss_queries"] == 0 + and cost["reverse_mode_calls"] == 0, + f"{path}: condition {name}", + ) + challenge = row["assays"]["challenge"] + calibration = challenge["calibration"] + targets = challenge["targets"] + require( + calibration["episodes"] == CALIBRATION_EPISODES + and calibration["quantiles"] == list(TARGET_QUANTILES) + and calibration["targets"] == targets + and calibration["uses_outcome_labels"] is False + and calibration["seed"] == 550_000 + task_seed + and challenge["episodes_per_target"] == CHALLENGE_EPISODES + and challenge["selection_over_evaluation_outcomes"] is False + and challenge["maximum_steps_per_episode"] == 28 + and len(targets) == len(TARGET_QUANTILES) + and all( + targets[index] < targets[index + 1] + for index in range(len(targets) - 1) + ) + and row["signatures"]["challenge_episodes"] + == len(TARGET_QUANTILES) * CHALLENGE_EPISODES, + f"{path}: calibration", + ) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--results", default="results/bci_v2_calibrated_dev" + ) + parser.add_argument( + "--out", default="results/bci_v2_calibrated_dev_gate.json" + ) + args = parser.parse_args() + digests = { + name: sha256(path) + for name, path in source_paths().items() + } + expected = { + f"bci_v2_calibrated_t{task_seed}_m0.json" + for task_seed in TASK_SEEDS + } + observed = { + os.path.basename(path) + for path in glob.glob(os.path.join(args.results, "*.json")) + } + require( + observed == expected, + ( + f"calibrated grid drift: missing={sorted(expected-observed)}, " + f"extra={sorted(observed-expected)}" + ), + ) + records = {} + commits = set() + source_sha256 = {} + for task_seed in TASK_SEEDS: + path = os.path.join( + args.results, + f"bci_v2_calibrated_t{task_seed}_m0.json", + ) + with open(path) as handle: + row = json.load(handle) + validate(row, path, task_seed, digests) + records[task_seed] = row + commits.add(row["provenance"]["git_commit"]) + source_sha256[path] = sha256(path) + require(len(commits) == 1, "calibrated source revision drift") + checks_by_seed = { + str(seed): checks(records[seed]) for seed in TASK_SEEDS + } + passed = all( + value for seed_checks in checks_by_seed.values() + for value in seed_checks.values() + ) + output = { + "protocol": + "oral_b_v2_calibrated_recovery_development_v1", + "status": "passed" if passed else "failed", + "complete_grid": True, + "fixed_config": FIXED_CONFIG, + "calibration_quantiles": list(TARGET_QUANTILES), + "checks_by_task_seed": checks_by_seed, + "calibrated_targets_by_task_seed": { + str(seed): records[seed]["assays"]["challenge"]["targets"] + for seed in TASK_SEEDS + }, + "intact_final_by_task_seed": [ + records[seed]["conditions"]["intact"]["final_success"] + for seed in TASK_SEEDS + ], + "challenge_success_fraction_by_task_seed": [ + records[seed]["signatures"]["challenge_success_fraction"] + for seed in TASK_SEEDS + ], + "terminal_outcome_accuracy_by_task_seed": [ + records[seed]["signatures"][ + "terminal_residual_outcome_balanced_acc" + ] + for seed in TASK_SEEDS + ], + "confirmation_seeds_touched": False, + "calibrated_confirmation_opened": passed, + "source_commit": next(iter(commits)), + "source_sha256": source_sha256, + "input_sha256": digests, + "review_score_before": 7, + "review_score_after": 7, + "score_change_rule": ( + "development validation never changes the formal score" + ), + } + os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) + if os.path.exists(args.out): + with open(args.out) as handle: + existing = json.load(handle) + require(existing == output, "existing calibrated R1 gate differs") + else: + with open(args.out, "w") as handle: + json.dump(output, handle, indent=2, sort_keys=True) + handle.write("\n") + print(json.dumps(output, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/experiments/bci_v2_calibrated_confirmation.py b/experiments/bci_v2_calibrated_confirmation.py new file mode 100644 index 0000000..ae79708 --- /dev/null +++ b/experiments/bci_v2_calibrated_confirmation.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +"""Run one untouched calibration-split oral-B-v2 confirmation cell.""" +import argparse +import json +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from experiments.bci_v2_calibrated_run import ( + PROTOCOL_PATH, + finite_tree, + provenance, + require_parent_gates, + run_cell, + sha256, + source_paths, +) +from experiments.bci_v2_recovery_run import FIXED_CONFIG + + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +R1_GATE_PATH = os.path.join( + ROOT, "results", "bci_v2_calibrated_dev_gate.json" +) +TASK_SEEDS = tuple(range(30, 36)) +MODEL_SEEDS = tuple(range(5)) + + +def require_r1_gate(r1): + digests = { + name: sha256(path) + for name, path in source_paths().items() + } + if not ( + r1.get("protocol") + == "oral_b_v2_calibrated_recovery_development_v1" + and r1.get("status") == "passed" + and r1.get("complete_grid") is True + and r1.get("fixed_config") == FIXED_CONFIG + and r1.get("confirmation_seeds_touched") is False + and r1.get("calibrated_confirmation_opened") is True + and r1.get("review_score_after") == 7 + and r1.get("input_sha256") == digests + ): + raise ValueError( + "calibrated R2 requires the complete eligible R1 gate" + ) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--task-seed", type=int, choices=TASK_SEEDS, required=True + ) + parser.add_argument( + "--model-seed", type=int, choices=MODEL_SEEDS, required=True + ) + parser.add_argument( + "--r1-gate", + default="results/bci_v2_calibrated_dev_gate.json", + ) + parser.add_argument( + "--outdir", default="results/bci_v2_calibrated_confirmation" + ) + args = parser.parse_args() + require_parent_gates() + with open(args.r1_gate) as handle: + r1 = json.load(handle) + require_r1_gate(r1) + source = provenance({ + "development_runner": os.path.join( + ROOT, "experiments", "bci_v2_calibrated_run.py" + ), + "runner": os.path.abspath(__file__), + "r1_gate": os.path.abspath(args.r1_gate), + }) + if ( + source["git_tracked_dirty"] + or not all(source["tracked_inputs"].values()) + ): + raise RuntimeError( + "calibrated R2 requires clean, tracked, frozen inputs" + ) + cell = run_cell( + args.task_seed, + args.model_seed, + split="untouched_confirmation", + performance_seed_offset=590_000, + calibration_seed_offset=600_000, + challenge_seed_offset=610_000, + ) + result = { + "schema_version": 4, + "protocol": { + "name": + "oral_b_v2_calibrated_recovery_confirmation_v1", + "split": "untouched_confirmation", + "training_task_seed": args.task_seed, + "model_seed": args.model_seed, + "fixed_config": FIXED_CONFIG, + "calibration_uses_outcome_labels": False, + "no_further_selection": True, + "confirmation_grid_size": ( + len(TASK_SEEDS) * len(MODEL_SEEDS) + ), + "protocol_sha256": sha256(PROTOCOL_PATH), + "r1_gate_sha256": sha256(args.r1_gate), + }, + "args": vars(args), + "provenance": source, + **cell, + } + result["finite"] = finite_tree(result) + if not result["finite"]: + raise RuntimeError("non-finite calibrated confirmation record") + os.makedirs(args.outdir, exist_ok=True) + path = os.path.join( + args.outdir, + ( + f"bci_v2_calibrated_confirm_t{args.task_seed}" + f"_m{args.model_seed}.json" + ), + ) + if os.path.exists(path): + raise FileExistsError(f"refusing to overwrite {path}") + with open(path, "w") as handle: + json.dump(result, handle, indent=2, sort_keys=True) + handle.write("\n") + print(json.dumps({ + "path": path, + "intact_final": result["conditions"]["intact"]["final_success"], + "challenge_success_fraction": result["signatures"][ + "challenge_success_fraction" + ], + "terminal_outcome_accuracy": result["signatures"][ + "terminal_residual_outcome_balanced_acc" + ], + "finite": result["finite"], + }, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/experiments/bci_v2_calibrated_confirmation.sh b/experiments/bci_v2_calibrated_confirmation.sh new file mode 100644 index 0000000..73e8580 --- /dev/null +++ b/experiments/bci_v2_calibrated_confirmation.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Frozen calibrated R2: 6 untouched task seeds x 5 model seeds. +set -euo pipefail + +export OMP_NUM_THREADS=1 +export MKL_NUM_THREADS=1 + +for task_seed in 30 31 32 33 34 35; do + for model_seed in 0 1 2 3 4; do + python experiments/bci_v2_calibrated_confirmation.py \ + --task-seed "$task_seed" \ + --model-seed "$model_seed" + done +done + +PYTHONPATH=. python experiments/analyze_bci_v2_calibrated_confirmation.py diff --git a/experiments/bci_v2_calibrated_development.sh b/experiments/bci_v2_calibrated_development.sh new file mode 100644 index 0000000..0a9f004 --- /dev/null +++ b/experiments/bci_v2_calibrated_development.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# Frozen calibrated R1: one fixed mechanism x 3 new validation seeds. +set -euo pipefail + +export OMP_NUM_THREADS=1 +export MKL_NUM_THREADS=1 + +for task_seed in 26 27 28; do + python experiments/bci_v2_calibrated_run.py --task-seed "$task_seed" +done + +PYTHONPATH=. python experiments/analyze_bci_v2_calibrated_development.py diff --git a/experiments/bci_v2_calibrated_run.py b/experiments/bci_v2_calibrated_run.py new file mode 100644 index 0000000..cb477dd --- /dev/null +++ b/experiments/bci_v2_calibrated_run.py @@ -0,0 +1,457 @@ +#!/usr/bin/env python3 +"""Run one calibration-split oral-B-v2 development cell.""" +import argparse +from dataclasses import replace +import hashlib +import json +import os +import platform +import resource +import subprocess +import sys +import time + +import torch + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from experiments.bci_v2_recovery_run import ( + FIXED_CONFIG, + build_recovery_config, +) +from experiments.bci_v2_run import ( + ACUTE_MODES, + CONDITIONS, + evaluate_performance, + finite_tree, + neutral_warmup, + train, +) +from sdil.bci import generate_trajectories +from sdil.bci_v2 import BCIV2, run_day_v2 +from sdil.bci_v2_recovery_metrics import ( + annotate_target_events, + recovery_signature_metrics, +) + + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +PROTOCOL_PATH = os.path.join( + ROOT, "ORAL_B_V2_CALIBRATED_RECOVERY.md" +) +D4_GATE_PATH = os.path.join( + ROOT, "results", "kp_dynamic_projection_confirmation_gate.json" +) +OLD_R2_GATE_PATH = os.path.join( + ROOT, "results", "bci_td_confirmation_gate.json" +) +FAILED_V2_GATE_PATH = os.path.join( + ROOT, "results", "bci_v2_dev_gate.json" +) +FAILED_TARGET_GATE_PATH = os.path.join( + ROOT, "results", "bci_v2_recovery_dev_gate.json" +) +TASK_SEEDS = (26, 27, 28) +MODEL_SEEDS = (0,) +CALIBRATION_EPISODES = 512 +TARGET_QUANTILES = (0.20, 0.35, 0.50, 0.65, 0.80) +CHALLENGE_EPISODES = 128 +UNCENSORED_TARGET = 2.1 + + +def sha256(path): + digest = hashlib.sha256() + with open(path, "rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def source_paths(): + return { + "runner": os.path.abspath(__file__), + "development_analyzer": os.path.join( + ROOT, + "experiments", + "analyze_bci_v2_calibrated_development.py", + ), + "confirmation_runner": os.path.join( + ROOT, "experiments", "bci_v2_calibrated_confirmation.py" + ), + "confirmation_analyzer": os.path.join( + ROOT, + "experiments", + "analyze_bci_v2_calibrated_confirmation.py", + ), + "common_runner": os.path.join( + ROOT, "experiments", "bci_v2_run.py" + ), + "recovery_runner": os.path.join( + ROOT, "experiments", "bci_v2_recovery_run.py" + ), + "recovery_development_analyzer": os.path.join( + ROOT, + "experiments", + "analyze_bci_v2_recovery_development.py", + ), + "recovery_confirmation_analyzer": os.path.join( + ROOT, + "experiments", + "analyze_bci_v2_recovery_confirmation.py", + ), + "base_dynamics": os.path.join(ROOT, "sdil", "bci.py"), + "v2_dynamics": os.path.join(ROOT, "sdil", "bci_v2.py"), + "v2_metrics": os.path.join(ROOT, "sdil", "bci_v2_metrics.py"), + "recovery_metrics": os.path.join( + ROOT, "sdil", "bci_v2_recovery_metrics.py" + ), + "protocol": PROTOCOL_PATH, + "d4_gate": D4_GATE_PATH, + "old_r2_gate": OLD_R2_GATE_PATH, + "failed_v2_gate": FAILED_V2_GATE_PATH, + "failed_target_gate": FAILED_TARGET_GATE_PATH, + } + + +def provenance(extra_paths=None): + def run(command): + return subprocess.run( + command, + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + paths = source_paths() + if extra_paths: + paths.update(extra_paths) + relative = { + name: os.path.relpath(path, ROOT) + for name, path in paths.items() + } + tracked = { + name: subprocess.run( + ["git", "ls-files", "--error-unmatch", path], + cwd=ROOT, + capture_output=True, + ).returncode == 0 + for name, path in relative.items() + } + return { + "git_commit": run(["git", "rev-parse", "HEAD"]), + "git_tracked_dirty": bool(run([ + "git", "status", "--porcelain", "--untracked-files=no" + ])), + "tracked_inputs": tracked, + "input_sha256": { + name: sha256(path) for name, path in paths.items() + }, + } + + +def require_parent_gates(): + with open(D4_GATE_PATH) as handle: + d4 = json.load(handle) + with open(OLD_R2_GATE_PATH) as handle: + old_r2 = json.load(handle) + with open(FAILED_V2_GATE_PATH) as handle: + failed_v2 = json.load(handle) + with open(FAILED_TARGET_GATE_PATH) as handle: + failed_target = json.load(handle) + if not ( + d4.get("status") == "passed" + and d4.get("review_score_after") == 7 + and old_r2.get("status") == "failed" + and failed_v2.get("status") == "failed" + and failed_target.get("protocol") + == "oral_b_v2_cold_start_recovery_development_v1" + and failed_target.get("status") == "failed" + and failed_target.get("recovery_confirmation_opened") is False + and failed_target.get("review_score_after") == 7 + ): + raise ValueError( + "calibrated recovery requires D4 and all preserved failures" + ) + + +def calibrate_targets(model, task_seed, seed_offset): + calibration_seed = seed_offset + task_seed + calibration_cfg = replace(model.cfg, target=UNCENSORED_TARGET) + trajectories = generate_trajectories( + calibration_cfg, + calibration_seed, + days=1, + episodes=CALIBRATION_EPISODES, + ) + calibration_model = model.clone() + calibration_model.cfg = calibration_cfg + report = run_day_v2( + calibration_model, + trajectories, + 0, + horizon=calibration_cfg.steps_per_episode, + plasticity_gain=0.0, + learn_role=False, + probe_role=False, + learn_predictor=False, + learn_critic=False, + critic_enabled=False, + terminal_outcome_enabled=False, + collect=True, + ) + if bool(report["success"].any()): + raise RuntimeError("uncensored calibration target was reached") + cursors = torch.stack([ + event["cursor"].double() for event in report["events"] + ]) + maxima = cursors.max(0).values + quantiles = torch.tensor(TARGET_QUANTILES, dtype=torch.float64) + targets = torch.quantile(maxima, quantiles).tolist() + if not all( + targets[index] < targets[index + 1] + for index in range(len(targets) - 1) + ): + raise RuntimeError("calibrated targets must be strictly ordered") + return tuple(targets), { + "seed": calibration_seed, + "episodes": CALIBRATION_EPISODES, + "quantiles": list(TARGET_QUANTILES), + "targets": targets, + "uses_outcome_labels": False, + "active_state_episode_steps": report["active_transitions"], + "maximum_cursor_summary": { + "minimum": maxima.min().item(), + "median": maxima.median().item(), + "maximum": maxima.max().item(), + }, + } + + +def evaluate_calibrated_ladder( + model, + task_seed, + calibration_seed_offset, + challenge_seed_offset, +): + targets, calibration = calibrate_targets( + model, task_seed, calibration_seed_offset + ) + mode_events = {name: [] for name in ACUTE_MODES} + costs = {name: 0 for name in ACUTE_MODES} + seeds = {} + for target_index, target in enumerate(targets, start=1): + challenge_cfg = replace(model.cfg, target=target) + trajectory_seed = ( + challenge_seed_offset + + 1_000 * task_seed + + target_index - 1 + ) + seeds[str(target_index)] = trajectory_seed + trajectories = generate_trajectories( + challenge_cfg, + trajectory_seed, + days=1, + episodes=CHALLENGE_EPISODES, + ) + for mode in ACUTE_MODES: + settings = { + "intact": {}, + "acute_critic_lesion": {"critic_enabled": False}, + "acute_outcome_lesion": { + "terminal_outcome_enabled": False + }, + }[mode] + assay_model = model.clone() + assay_model.cfg = challenge_cfg + report = run_day_v2( + assay_model, + trajectories, + 0, + horizon=challenge_cfg.steps_per_episode, + plasticity_gain=0.0, + learn_role=False, + probe_role=False, + learn_predictor=False, + learn_critic=False, + collect=True, + **settings, + ) + annotate_target_events( + report["events"], + 0, + report["success"], + ( + 3_000_000 + + (target_index - 1) * CHALLENGE_EPISODES + ), + target_index, + ) + mode_events[mode].extend(report["events"]) + costs[mode] += report["active_transitions"] + return targets, mode_events, { + "calibration": calibration, + "targets": list(targets), + "episodes_per_target": CHALLENGE_EPISODES, + "trajectory_seeds": seeds, + "active_state_episode_steps_by_mode": costs, + "selection_over_evaluation_outcomes": False, + "maximum_steps_per_episode": model.cfg.steps_per_episode, + } + + +def run_cell( + task_seed, + model_seed, + *, + split, + performance_seed_offset, + calibration_seed_offset, + challenge_seed_offset, +): + cfg = build_recovery_config() + trajectories = generate_trajectories(cfg, task_seed) + performance_seed = performance_seed_offset + task_seed + performance_trajectories = generate_trajectories( + cfg, performance_seed, days=1, episodes=256 + ) + initial = BCIV2(cfg, model_seed) + conditions = {} + trained = {} + warmups = {} + training_events = None + started = time.perf_counter() + for name in CONDITIONS: + model, warmup = neutral_warmup( + initial, task_seed, model_seed, name + ) + events, report = train( + model, trajectories, name, collect=name == "intact" + ) + warmups[name] = warmup + conditions[name] = report + trained[name] = model + if name == "intact": + training_events = events + for name in CONDITIONS: + report = evaluate_performance( + trained[name], performance_trajectories + ) + conditions[name]["final_success"] = report["success_rate"] + conditions[name]["evaluation_active_state_episode_steps"] = ( + report["active_transitions"] + ) + targets, challenge_events, challenge_cost = ( + evaluate_calibrated_ladder( + trained["intact"], + task_seed, + calibration_seed_offset, + challenge_seed_offset, + ) + ) + signatures = recovery_signature_metrics( + training_events, + challenge_events, + cfg, + trained["intact"].role, + targets, + ) + return { + "config": vars(cfg), + "warmup": warmups, + "conditions": conditions, + "signatures": signatures, + "assays": { + "performance_evaluation_seed": performance_seed, + "performance_evaluation_episodes": 256, + "challenge": challenge_cost, + }, + "wall_s": time.perf_counter() - started, + "peak_rss_mib": ( + resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024 + ), + "hardware": { + "device": "cpu", + "platform": platform.platform(), + "torch_version": torch.__version__, + "threads": torch.get_num_threads(), + }, + "split": split, + } + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--task-seed", type=int, choices=TASK_SEEDS, required=True + ) + parser.add_argument( + "--model-seed", type=int, choices=MODEL_SEEDS, default=0 + ) + parser.add_argument( + "--outdir", default="results/bci_v2_calibrated_dev" + ) + args = parser.parse_args() + require_parent_gates() + source = provenance() + if ( + source["git_tracked_dirty"] + or not all(source["tracked_inputs"].values()) + ): + raise RuntimeError( + "calibrated R1 requires clean, tracked, frozen inputs" + ) + cell = run_cell( + args.task_seed, + args.model_seed, + split="development_validation", + performance_seed_offset=540_000, + calibration_seed_offset=550_000, + challenge_seed_offset=560_000, + ) + result = { + "schema_version": 4, + "protocol": { + "name": + "oral_b_v2_calibrated_recovery_development_v1", + "selection_split": "development_validation", + "hyperparameter_selection": False, + "confirmation_seeds_touched": False, + "fixed_config": FIXED_CONFIG, + "calibration_uses_outcome_labels": False, + "protocol_sha256": source["input_sha256"]["protocol"], + "failed_target_gate_sha256": source["input_sha256"][ + "failed_target_gate" + ], + }, + "args": vars(args), + "provenance": source, + **cell, + } + result["finite"] = finite_tree(result) + if not result["finite"]: + raise RuntimeError("non-finite calibrated development record") + os.makedirs(args.outdir, exist_ok=True) + path = os.path.join( + args.outdir, + f"bci_v2_calibrated_t{args.task_seed}_m0.json", + ) + if os.path.exists(path): + raise FileExistsError(f"refusing to overwrite {path}") + with open(path, "w") as handle: + json.dump(result, handle, indent=2, sort_keys=True) + handle.write("\n") + print(json.dumps({ + "path": path, + "intact_final": result["conditions"]["intact"]["final_success"], + "challenge_success_fraction": result["signatures"][ + "challenge_success_fraction" + ], + "terminal_outcome_accuracy": result["signatures"][ + "terminal_residual_outcome_balanced_acc" + ], + "finite": result["finite"], + }, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/experiments/bci_v2_calibrated_smoke.py b/experiments/bci_v2_calibrated_smoke.py new file mode 100644 index 0000000..1c05783 --- /dev/null +++ b/experiments/bci_v2_calibrated_smoke.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +"""Endpoint-free checks for calibration-split target selection.""" +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from experiments.bci_v2_calibrated_run import ( + CALIBRATION_EPISODES, + TARGET_QUANTILES, + calibrate_targets, +) +from experiments.bci_v2_recovery_run import build_recovery_config +from sdil.bci_v2 import BCIV2 + + +def check_calibration_without_outcomes(): + cfg = build_recovery_config() + model = BCIV2(cfg, model_seed=901) + targets, report = calibrate_targets( + model, task_seed=902, seed_offset=900_000 + ) + assert report["episodes"] == CALIBRATION_EPISODES + assert report["quantiles"] == list(TARGET_QUANTILES) + assert report["uses_outcome_labels"] is False + assert len(targets) == len(TARGET_QUANTILES) + assert all( + targets[index] < targets[index + 1] + for index in range(len(targets) - 1) + ) + assert all(-2.0 < target < 2.0 for target in targets) + print("calibration targets use only independent cursor quantiles: exact") + + +if __name__ == "__main__": + check_calibration_without_outcomes() + print("ALL CALIBRATION-SPLIT MECHANICS CHECKS PASSED") |
