protocol: freeze independent oral-B-v2 gates

1 files changed, 370 insertions, 0 deletions

diff --git a/experiments/analyze_bci_v2_development.py b/experiments/analyze_bci_v2_development.py
new file mode 100644
index 0000000..4acf321
--- /dev/null
+++ b/experiments/analyze_bci_v2_development.py
@@ -0,0 +1,370 @@
+#!/usr/bin/env python3
+"""Audit and select the frozen oral-B-v2 development grid."""
+import argparse
+import glob
+import hashlib
+import json
+import os
+import statistics
+
+from experiments.bci_v2_run import (
+    CHALLENGE_EPISODES,
+    CONDITIONS,
+    CRITIC_ETAS,
+    FORWARD_ETAS,
+    GAMMAS,
+    HORIZONS,
+    MODEL_SEEDS,
+    PROTOCOL_PATH,
+    TASK_SEEDS,
+    build_config,
+    source_paths,
+)
+
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+RUNNER_PATH = os.path.join(ROOT, "experiments", "bci_v2_run.py")
+
+
+def require(condition, message):
+    if not condition:
+        raise ValueError(message)
+
+
+def sha256(path):
+    digest = hashlib.sha256()
+    with open(path, "rb") as handle:
+        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def tag(value):
+    return str(value).replace(".", "p")
+
+
+def filename(candidate, task_seed):
+    forward_eta, gamma, critic_eta = candidate
+    return (
+        f"bci_v2_e{tag(forward_eta)}_g{tag(gamma)}"
+        f"_c{tag(critic_eta)}_t{task_seed}_m0.json"
+    )
+
+
+def candidate_checks(row):
+    conditions = row["conditions"]
+    signatures = row["signatures"]
+    intact = conditions["intact"]
+    return {
+        "final_success_at_least_0p70":
+            intact["final_success"] >= 0.70,
+        "learning_gain_at_least_0p10":
+            intact["learning_gain"] >= 0.10,
+        "fixed_role_final_gap_at_least_0p20": (
+            intact["final_success"]
+            - conditions["fixed_role"]["final_success"]
+        ) >= 0.20,
+        "oracle_final_deficit_at_most_0p10": (
+            conditions["oracle_role"]["final_success"]
+            - intact["final_success"]
+        ) <= 0.10,
+        "plasticity_lesion_retains_at_most_half_gain":
+            conditions["plasticity_lesion"]["learning_gain"]
+            <= 0.5 * intact["learning_gain"],
+        "learned_role_cosine_at_least_0p80":
+            intact["role_cosine_after_training"] >= 0.80,
+        "challenge_success_fraction_between_0p15_and_0p85":
+            0.15 <= signatures["challenge_success_fraction"] <= 0.85,
+        "terminal_outcome_accuracy_at_least_0p65":
+            signatures[
+                "terminal_residual_outcome_balanced_acc"
+            ] >= 0.65,
+        "terminal_role_separation_at_least_0p03":
+            signatures[
+                "terminal_role_aligned_outcome_separation"
+            ] >= 0.03,
+        "acute_outcome_lesion_separation_drop_at_least_0p01":
+            signatures[
+                "terminal_outcome_separation_drop_under_acute_lesion"
+            ] >= 0.01,
+        "critic_expectedness_contribution_at_least_0p005":
+            signatures[
+                "mean_critic_expectedness_contribution"
+            ] >= 0.005,
+        "critic_contribution_tracks_value_at_least_0p95":
+            signatures[
+                "critic_contribution_value_prediction_corr"
+            ] >= 0.95,
+        "residual_soma_corr_at_most_0p10":
+            signatures["mean_abs_residual_soma_corr"] <= 0.10,
+        "raw_residual_corr_gap_at_least_0p20":
+            signatures[
+                "raw_minus_residual_abs_soma_corr"
+            ] >= 0.20,
+        "surrounding_event_accuracy_at_least_0p52":
+            signatures[
+                "surrounding_event_decoder_balanced_acc"
+            ] >= 0.52,
+        "decoder_distance_corr_at_least_0p02":
+            signatures[
+                "decoder_distance_residual_corr"
+            ] >= 0.02,
+        "sign_inversion_at_least_0p01":
+            signatures[
+                "causal_role_sign_inversion_index"
+            ] >= 0.01,
+        "velocity_advantage_at_least_0p05":
+            signatures[
+                "velocity_minus_error_abs_cv_corr"
+            ] >= 0.05,
+    }
+
+
+def validate(row, path, candidate, task_seed, digests):
+    require(row.get("schema_version") == 2, f"{path}: schema")
+    args = row.get("args", {})
+    require(args.get("task_seed") == task_seed, f"{path}: task seed")
+    require(args.get("model_seed") in MODEL_SEEDS, f"{path}: model seed")
+    require(
+        (
+            args.get("forward_eta"),
+            args.get("gamma"),
+            args.get("critic_eta"),
+        ) == candidate,
+        f"{path}: candidate",
+    )
+    protocol = row.get("protocol", {})
+    require(
+        protocol.get("name") == "oral_b_v2_development_v1",
+        f"{path}: protocol",
+    )
+    require(
+        protocol.get("selection_split") == "development"
+        and protocol.get("confirmation_seeds_touched") is False,
+        f"{path}: split",
+    )
+    require(protocol.get("grid_size") == 24, f"{path}: grid size")
+    require(
+        protocol.get("protocol_sha256") == digests["protocol"]
+        and protocol.get("d4_gate_sha256") == digests["d4_gate"]
+        and protocol.get("old_r2_gate_sha256")
+        == digests["old_r2_gate"],
+        f"{path}: parent digest",
+    )
+    provenance = row.get("provenance", {})
+    require(
+        provenance.get("git_tracked_dirty") is False,
+        f"{path}: dirty source",
+    )
+    require(
+        provenance.get("tracked_inputs")
+        and all(provenance["tracked_inputs"].values())
+        and set(provenance["tracked_inputs"]) == set(digests),
+        f"{path}: tracked inputs",
+    )
+    require(
+        provenance.get("input_sha256") == digests,
+        f"{path}: source digest drift",
+    )
+    require(
+        isinstance(provenance.get("git_commit"), str)
+        and len(provenance["git_commit"]) == 40,
+        f"{path}: source commit",
+    )
+    expected_cfg = vars(build_config(*candidate))
+    require(row.get("config") == expected_cfg, f"{path}: config")
+    require(row.get("finite") is True, f"{path}: finite")
+    require(row.get("split") == "development", f"{path}: cell split")
+    require(
+        set(row.get("conditions", {})) == set(CONDITIONS),
+        f"{path}: conditions",
+    )
+    for name in CONDITIONS:
+        warmup = row["warmup"][name]
+        require(
+            warmup["batches"] == 100
+            and warmup["examples"] == 6400
+            and warmup["instruction_present"] is False
+            and warmup["role_cursor_scalar_observations"] == 12800
+            and warmup["predictor_max_abs_error"] <= 1e-5,
+            f"{path}: warmup {name}",
+        )
+        condition = row["conditions"][name]
+        require(
+            len(condition["daily_success"]) == 14,
+            f"{path}: daily trajectory {name}",
+        )
+        cost = condition["cost"]
+        require(
+            cost["maximum_state_episode_steps"] == 25088
+            and cost["cursor_scalar_observations"]
+            == 2 * cost["role_probe_examples"]
+            and cost["terminal_outcome_observations"] == 896
+            and cost["task_loss_queries"] == 0
+            and cost["reverse_mode_calls"] == 0,
+            f"{path}: cost {name}",
+        )
+    challenge = row["assays"]["challenge"]
+    require(
+        challenge["horizons"] == list(HORIZONS)
+        and challenge["episodes_per_horizon"] == CHALLENGE_EPISODES
+        and challenge["selection_over_horizons"] is False,
+        f"{path}: challenge ladder",
+    )
+    signatures = row["signatures"]
+    require(
+        signatures["challenge_episodes"]
+        == len(HORIZONS) * CHALLENGE_EPISODES,
+        f"{path}: challenge count",
+    )
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--results", default="results/bci_v2_dev")
+    parser.add_argument(
+        "--out", default="results/bci_v2_dev_gate.json"
+    )
+    args = parser.parse_args()
+    paths = source_paths()
+    digests = {name: sha256(path) for name, path in paths.items()}
+    require(
+        os.path.abspath(paths["runner"]) == os.path.abspath(RUNNER_PATH),
+        "runner path",
+    )
+    candidates = [
+        (forward_eta, gamma, critic_eta)
+        for forward_eta in FORWARD_ETAS
+        for gamma in GAMMAS
+        for critic_eta in CRITIC_ETAS
+    ]
+    expected_names = {
+        filename(candidate, task_seed)
+        for candidate in candidates
+        for task_seed in TASK_SEEDS
+    }
+    observed_names = {
+        os.path.basename(path)
+        for path in glob.glob(os.path.join(args.results, "*.json"))
+    }
+    require(
+        observed_names == expected_names,
+        (
+            "development grid drift: "
+            f"missing={sorted(expected_names - observed_names)}, "
+            f"extra={sorted(observed_names - expected_names)}"
+        ),
+    )
+    records = {}
+    commits = set()
+    source_sha256 = {}
+    for candidate in candidates:
+        for task_seed in TASK_SEEDS:
+            path = os.path.join(
+                args.results, filename(candidate, task_seed)
+            )
+            with open(path) as handle:
+                row = json.load(handle)
+            validate(row, path, candidate, task_seed, digests)
+            records[(candidate, task_seed)] = row
+            commits.add(row["provenance"]["git_commit"])
+            source_sha256[path] = sha256(path)
+    require(len(commits) == 1, "development source revision drift")
+
+    summaries = []
+    eligible = []
+    for candidate in candidates:
+        rows = [records[(candidate, seed)] for seed in TASK_SEEDS]
+        checks = {
+            str(seed): candidate_checks(row)
+            for seed, row in zip(TASK_SEEDS, rows)
+        }
+        passed = all(
+            value for seed_checks in checks.values()
+            for value in seed_checks.values()
+        )
+        final_values = [
+            row["conditions"]["intact"]["final_success"]
+            for row in rows
+        ]
+        outcome_values = [
+            row["signatures"][
+                "terminal_residual_outcome_balanced_acc"
+            ]
+            for row in rows
+        ]
+        summary = {
+            "forward_eta": candidate[0],
+            "gamma": candidate[1],
+            "critic_eta": candidate[2],
+            "eligible": passed,
+            "checks_by_task_seed": checks,
+            "intact_final_by_task_seed": final_values,
+            "terminal_outcome_accuracy_by_task_seed": outcome_values,
+            "worst_intact_final": min(final_values),
+            "mean_intact_final": statistics.mean(final_values),
+            "worst_terminal_outcome_accuracy": min(outcome_values),
+        }
+        summaries.append(summary)
+        if passed:
+            eligible.append(summary)
+    selected = None
+    if eligible:
+        selected = sorted(
+            eligible,
+            key=lambda row: (
+                -row["worst_intact_final"],
+                -row["worst_terminal_outcome_accuracy"],
+                row["forward_eta"],
+                row["critic_eta"],
+                row["gamma"],
+            ),
+        )[0]
+    output = {
+        "protocol": "oral_b_v2_development_v1",
+        "status": "passed" if selected else "failed",
+        "complete_grid": True,
+        "grid_size": len(expected_names),
+        "candidate_summaries": summaries,
+        "selected": (
+            {
+                "forward_eta": selected["forward_eta"],
+                "gamma": selected["gamma"],
+                "critic_eta": selected["critic_eta"],
+                "worst_intact_final": selected[
+                    "worst_intact_final"
+                ],
+                "worst_terminal_outcome_accuracy": selected[
+                    "worst_terminal_outcome_accuracy"
+                ],
+            }
+            if selected else None
+        ),
+        "confirmation_seeds_touched": False,
+        "oral_b_v2_confirmation_opened": selected is not None,
+        "source_commit": next(iter(commits)),
+        "source_sha256": source_sha256,
+        "input_sha256": digests,
+        "review_score_before": 7,
+        "review_score_after": 7,
+        "score_change_rule": (
+            "development selection never changes the formal milestone score"
+        ),
+    }
+    os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True)
+    if os.path.exists(args.out):
+        with open(args.out) as handle:
+            existing = json.load(handle)
+        require(
+            existing == output,
+            "existing v2 development gate differs from deterministic audit",
+        )
+    else:
+        with open(args.out, "w") as handle:
+            json.dump(output, handle, indent=2, sort_keys=True)
+            handle.write("\n")
+    print(json.dumps(output, indent=2))
+
+
+if __name__ == "__main__":
+    main()