protocol: freeze BCI-v2-gated oral-A scaling

3 files changed, 478 insertions, 0 deletions

diff --git a/experiments/analyze_oral_a_dynamic_scaling_v2.py b/experiments/analyze_oral_a_dynamic_scaling_v2.py
new file mode 100644
index 0000000..c593929
--- /dev/null
+++ b/experiments/analyze_oral_a_dynamic_scaling_v2.py
@@ -0,0 +1,256 @@
+#!/usr/bin/env python3
+"""Audit the calibrated-BCI-v2-gated standard-depth confirmation panel."""
+import argparse
+import glob
+import hashlib
+import json
+import os
+import statistics
+import subprocess
+
+from analyze_oral_a_dynamic_scaling import (
+    DEPTHS,
+    METHODS,
+    SEEDS,
+    dynamic_invariants,
+    oral_a_checks,
+    require,
+    sha256,
+    validate_record,
+)
+from oral_a_dynamic_scaling_v2 import require_prerequisites
+
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+PROTOCOL_PATH = os.path.join(ROOT, "ORAL_A_RECOVERY_V2.md")
+RUNNER_PATH = os.path.join(
+    ROOT, "experiments", "oral_a_dynamic_scaling_v2.py")
+ANALYZER_PATH = os.path.abspath(__file__)
+LEGACY_RUNNER_PATH = os.path.join(
+    ROOT, "experiments", "oral_a_dynamic_scaling.py")
+LEGACY_ANALYZER_PATH = os.path.join(
+    ROOT, "experiments", "analyze_oral_a_dynamic_scaling.py")
+CONV_RUN_PATH = os.path.join(ROOT, "experiments", "conv_run.py")
+
+
+def git_blob_sha256(commit, relative_path):
+    content = subprocess.run(
+        ["git", "show", f"{commit}:{relative_path}"],
+        cwd=ROOT,
+        check=True,
+        capture_output=True,
+    ).stdout
+    return hashlib.sha256(content).hexdigest()
+
+
+def require_training_source_bound(commit):
+    paths = (
+        PROTOCOL_PATH,
+        RUNNER_PATH,
+        LEGACY_RUNNER_PATH,
+        CONV_RUN_PATH,
+    )
+    for path in paths:
+        relative = os.path.relpath(path, ROOT)
+        require(
+            git_blob_sha256(commit, relative) == sha256(path),
+            f"training source drift after {commit}: {relative}",
+        )
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--d4_results", default="results/kp_dynamic_projection_confirmation")
+    parser.add_argument(
+        "--new_results", default="results/oral_a_dynamic_scaling_v2")
+    parser.add_argument(
+        "--d4_gate",
+        default="results/kp_dynamic_projection_confirmation_gate.json")
+    parser.add_argument(
+        "--bci_v2_gate",
+        default="results/bci_v2_calibrated_confirmation_gate.json")
+    parser.add_argument(
+        "--out", default="results/oral_a_dynamic_scaling_v2_gate.json")
+    args = parser.parse_args()
+    prerequisite = require_prerequisites(args.d4_gate, args.bci_v2_gate)
+    with open(args.d4_gate) as handle:
+        d4 = json.load(handle)
+    with open(args.bci_v2_gate) as handle:
+        bci_v2 = json.load(handle)
+
+    expected_d4 = {
+        f"seed{seed}_{condition}.json"
+        for seed in SEEDS
+        for condition in ("clean_kp", "dynamic")
+    }
+    observed_d4 = {
+        os.path.basename(path)
+        for path in glob.glob(os.path.join(args.d4_results, "*.json"))
+    }
+    require(
+        observed_d4 == expected_d4,
+        f"D4 record drift: missing={sorted(expected_d4-observed_d4)}, "
+        f"extra={sorted(observed_d4-expected_d4)}",
+    )
+    expected_new = {
+        f"{method}_d{depth}_s{seed}.json"
+        for depth in DEPTHS
+        for seed in SEEDS
+        for method in METHODS
+        if not (depth == 20 and method in ("clean_kp", "dynamic"))
+    }
+    observed_new = {
+        os.path.basename(path)
+        for path in glob.glob(os.path.join(args.new_results, "*.json"))
+    }
+    require(
+        observed_new == expected_new,
+        f"new record drift: missing={sorted(expected_new-observed_new)}, "
+        f"extra={sorted(observed_new-expected_new)}",
+    )
+
+    records = {}
+    rows = {}
+    source_sha256 = {}
+    d4_commits = set()
+    new_commits = set()
+    for seed in SEEDS:
+        for method, condition in (
+            ("clean_kp", "clean_kp"),
+            ("dynamic", "dynamic"),
+        ):
+            path = os.path.join(
+                args.d4_results, f"seed{seed}_{condition}.json")
+            with open(path) as handle:
+                record = json.load(handle)
+            records[(method, 20, seed)] = record
+            rows[(method, 20, seed)] = validate_record(
+                record, path, method, 20, seed)
+            d4_commits.add(rows[(method, 20, seed)]["source_commit"])
+            source_sha256[path] = sha256(path)
+    for depth in DEPTHS:
+        for seed in SEEDS:
+            for method in METHODS:
+                if depth == 20 and method in ("clean_kp", "dynamic"):
+                    continue
+                path = os.path.join(
+                    args.new_results, f"{method}_d{depth}_s{seed}.json")
+                with open(path) as handle:
+                    record = json.load(handle)
+                records[(method, depth, seed)] = record
+                rows[(method, depth, seed)] = validate_record(
+                    record, path, method, depth, seed)
+                new_commits.add(rows[(method, depth, seed)]["source_commit"])
+                source_sha256[path] = sha256(path)
+
+    require(len(rows) == 60, "oral-A grid must contain exactly 60 records")
+    require(
+        len(d4_commits) == 1 and len(new_commits) == 1,
+        "source revision drift",
+    )
+    require(
+        next(iter(d4_commits)) == d4["metrics"]["source_commit"],
+        "D4 source does not match its gate",
+    )
+    new_commit = next(iter(new_commits))
+    require_training_source_bound(new_commit)
+    require(
+        new_commit == prerequisite["git_commit"],
+        "analysis must run at the exact clean training revision",
+    )
+
+    failures = []
+    mechanism = {}
+    for depth in DEPTHS:
+        for seed in SEEDS:
+            dynamic = records[("dynamic", depth, seed)]
+            mechanism[f"d{depth}_s{seed}"] = dynamic_invariants(
+                dynamic, depth, seed, failures)
+            bp_macs = rows[("bp", depth, seed)]["total_macs"]
+            for method in ("clean_kp", "dynamic"):
+                if rows[(method, depth, seed)]["total_macs"] > 1.34 * bp_macs:
+                    failures.append(f"{method}_d{depth}_s{seed}:mac_cost")
+            if rows[("dynamic", depth, seed)]["peak_memory"] > 8 * 1024 ** 3:
+                failures.append(f"dynamic_d{depth}_s{seed}:peak_memory")
+            for method in ("dfa", "clean_kp", "dynamic"):
+                if rows[(method, depth, seed)]["logical_queries"] != 0:
+                    failures.append(f"{method}_d{depth}_s{seed}:queries")
+
+    accuracies = {
+        method: {
+            depth: [
+                rows[(method, depth, seed)]["accuracy"] for seed in SEEDS
+            ]
+            for depth in DEPTHS
+        }
+        for method in METHODS
+    }
+    alignments = {
+        depth: [
+            rows[("dynamic", depth, seed)]["early_alignment"]
+            for seed in SEEDS
+        ]
+        for depth in DEPTHS
+    }
+    checks, derived = oral_a_checks(accuracies, alignments, failures)
+    passed = all(checks.values())
+    output = {
+        "protocol": "oral_a_dynamic_innovation_scaling_recovery_v2",
+        "status": "passed" if passed else "failed",
+        "complete_grid": True,
+        "checks": checks,
+        "metrics": {
+            "accuracy_by_method_depth_seed": accuracies,
+            "mean_accuracy": {
+                method: {
+                    str(depth): statistics.mean(values)
+                    for depth, values in by_depth.items()
+                }
+                for method, by_depth in accuracies.items()
+            },
+            **derived,
+            "dynamic_alignment": alignments,
+            "mechanism": mechanism,
+            "invariant_failures": failures,
+        },
+        "d4_source_commit": next(iter(d4_commits)),
+        "new_source_commit": new_commit,
+        "source_sha256": source_sha256,
+        "protocol_sha256": sha256(PROTOCOL_PATH),
+        "runner_sha256": sha256(RUNNER_PATH),
+        "analyzer_sha256": sha256(ANALYZER_PATH),
+        "legacy_runner_sha256": sha256(LEGACY_RUNNER_PATH),
+        "legacy_analyzer_sha256": sha256(LEGACY_ANALYZER_PATH),
+        "conv_run_sha256": sha256(CONV_RUN_PATH),
+        "d4_gate_sha256": prerequisite["d4_gate_sha256"],
+        "bci_v2_gate_sha256": prerequisite["bci_v2_gate_sha256"],
+        "bci_v2_protocol": bci_v2["protocol"],
+        "test_evaluations": 60,
+        "standard_depth_scaling_established": passed,
+        "old_oral_a_failures_preserved": True,
+        "review_score_before": 8,
+        "review_score_after": 9 if passed else 8,
+        "score_change_rule": (
+            "only the complete calibrated-BCI-v2-gated 60-cell "
+            "standard-ResNet panel establishes oral-A scaling"
+        ),
+    }
+    os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True)
+    if os.path.exists(args.out):
+        with open(args.out) as handle:
+            existing = json.load(handle)
+        require(
+            existing == output,
+            "existing oral-A recovery-v2 gate differs from re-audit",
+        )
+        print(json.dumps(output, indent=2))
+        return
+    with open(args.out, "w") as handle:
+        json.dump(output, handle, indent=2, sort_keys=True)
+        handle.write("\n")
+    print(json.dumps(output, indent=2))
+
+
+if __name__ == "__main__":
+    main()
diff --git a/experiments/oral_a_dynamic_scaling_v2.py b/experiments/oral_a_dynamic_scaling_v2.py
new file mode 100644
index 0000000..2e58720
--- /dev/null
+++ b/experiments/oral_a_dynamic_scaling_v2.py
@@ -0,0 +1,149 @@
+#!/usr/bin/env python3
+"""Run shards of the calibrated-BCI-v2-gated standard-depth panel."""
+import argparse
+import hashlib
+import json
+import os
+import subprocess
+
+from oral_a_dynamic_scaling import (
+    DEPTHS,
+    METHODS,
+    SEEDS,
+    existing_matches,
+    jobs,
+)
+
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+PROTOCOL_PATH = os.path.join(ROOT, "ORAL_A_RECOVERY_V2.md")
+LEGACY_RUNNER_PATH = os.path.join(
+    ROOT, "experiments", "oral_a_dynamic_scaling.py")
+
+
+def sha256(path):
+    digest = hashlib.sha256()
+    with open(path, "rb") as handle:
+        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def git_output(*args):
+    return subprocess.run(
+        ["git", *args], cwd=ROOT, check=True, capture_output=True,
+        text=True).stdout.strip()
+
+
+def require_prerequisites(d4_path, bci_v2_path):
+    with open(d4_path) as handle:
+        d4 = json.load(handle)
+    with open(bci_v2_path) as handle:
+        bci_v2 = json.load(handle)
+    if not (
+        d4.get("protocol") == "kp_dynamic_neutral_projection_confirmation_v1"
+        and d4.get("status") == "passed"
+        and d4.get("review_score_after") == 7
+    ):
+        raise ValueError("oral-A recovery v2 requires the complete D4 pass")
+    if not (
+        bci_v2.get("protocol")
+        == "oral_b_v2_calibrated_recovery_confirmation_v1"
+        and bci_v2.get("status") == "passed"
+        and bci_v2.get("complete_grid") is True
+        and bci_v2.get("oral_b_v2_outcome_surprise_established") is True
+        and bci_v2.get("review_score_after") == 8
+        and bci_v2.get("new_oral_a_v2_protocol_may_be_frozen") is True
+        and bci_v2.get("old_oral_a_gate_remains_closed") is True
+        and bci_v2.get("all_prior_failures_preserved") is True
+    ):
+        raise ValueError(
+            "oral-A recovery v2 requires the complete calibrated BCI-v2 pass")
+
+    paths = [
+        os.path.abspath(__file__),
+        PROTOCOL_PATH,
+        LEGACY_RUNNER_PATH,
+        os.path.abspath(d4_path),
+        os.path.abspath(bci_v2_path),
+        os.path.join(ROOT, "experiments", "conv_run.py"),
+    ]
+    relative = [os.path.relpath(path, ROOT) for path in paths]
+    tracked = all(
+        subprocess.run(
+            ["git", "ls-files", "--error-unmatch", path],
+            cwd=ROOT,
+            capture_output=True,
+        ).returncode == 0
+        for path in relative
+    )
+    dirty = bool(git_output(
+        "status", "--porcelain", "--untracked-files=no"))
+    if dirty or not tracked:
+        raise RuntimeError(
+            "oral-A recovery v2 requires a clean tracked source")
+    return {
+        "git_commit": git_output("rev-parse", "HEAD"),
+        "protocol_sha256": sha256(PROTOCOL_PATH),
+        "d4_gate_sha256": sha256(d4_path),
+        "bci_v2_gate_sha256": sha256(bci_v2_path),
+        "legacy_runner_sha256": sha256(LEGACY_RUNNER_PATH),
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--d4_gate",
+        default="results/kp_dynamic_projection_confirmation_gate.json",
+    )
+    parser.add_argument(
+        "--bci_v2_gate",
+        default="results/bci_v2_calibrated_confirmation_gate.json",
+    )
+    parser.add_argument("--device", default="cuda")
+    parser.add_argument("--method", choices=("all",) + METHODS, default="all")
+    parser.add_argument("--depth", type=int, choices=DEPTHS)
+    parser.add_argument("--seed", type=int, choices=SEEDS)
+    parser.add_argument("--shard-index", type=int, default=0)
+    parser.add_argument("--num-shards", type=int, default=1)
+    parser.add_argument(
+        "--outdir", default="results/oral_a_dynamic_scaling_v2")
+    parser.add_argument("--dry-run", action="store_true")
+    args = parser.parse_args()
+    if not 0 <= args.shard_index < args.num_shards:
+        raise ValueError("invalid shard index")
+    source = require_prerequisites(args.d4_gate, args.bci_v2_gate)
+    selected = jobs(args.device, args.outdir)
+    if args.method != "all":
+        selected = [job for job in selected if job[0] == args.method]
+    if args.depth is not None:
+        selected = [job for job in selected if job[1] == args.depth]
+    if args.seed is not None:
+        selected = [job for job in selected if job[2] == args.seed]
+    selected = [
+        job
+        for index, job in enumerate(selected)
+        if index % args.num_shards == args.shard_index
+    ]
+    if not selected:
+        raise ValueError("no oral-A recovery-v2 jobs match the requested shard")
+
+    os.makedirs(args.outdir, exist_ok=True)
+    for method, depth, seed, path, command in selected:
+        tag = f"{method}_d{depth}_s{seed}"
+        if os.path.exists(path):
+            if not existing_matches(
+                path, method, depth, seed, source["git_commit"]
+            ):
+                raise RuntimeError(
+                    f"refusing mismatched existing record: {path}")
+            print(f"preserving {tag}", flush=True)
+            continue
+        print(tag, " ".join(command), flush=True)
+        if not args.dry_run:
+            subprocess.run(command, cwd=ROOT, check=True)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/experiments/oral_a_dynamic_scaling_v2_smoke.py b/experiments/oral_a_dynamic_scaling_v2_smoke.py
new file mode 100644
index 0000000..2fa72dd
--- /dev/null
+++ b/experiments/oral_a_dynamic_scaling_v2_smoke.py
@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+"""Endpoint-free audit of the oral-A recovery-v2 frozen funnel."""
+import copy
+import json
+import os
+import tempfile
+
+from analyze_oral_a_dynamic_scaling import oral_a_checks
+from oral_a_dynamic_scaling import DEPTHS, METHODS, SEEDS, jobs
+from oral_a_dynamic_scaling_v2 import require_prerequisites
+
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+
+def main():
+    generated = jobs("cuda", "results/oral_a_dynamic_scaling_v2")
+    assert len(generated) == 50
+    keys = [(method, depth, seed) for method, depth, seed, _, _ in generated]
+    assert len(set(keys)) == 50
+    expected = {
+        (method, depth, seed)
+        for depth in DEPTHS
+        for seed in SEEDS
+        for method in METHODS
+        if not (depth == 20 and method in ("clean_kp", "dynamic"))
+    }
+    assert set(keys) == expected
+    for method, depth, seed, path, command in generated:
+        assert path.endswith(f"{method}_d{depth}_s{seed}.json")
+        assert "--epochs" in command and command[command.index("--epochs")+1] == "200"
+        assert "--eval_every" in command
+        assert command[command.index("--eval_every")+1] == "0"
+        assert "--eval_split" in command
+        assert command[command.index("--eval_split")+1] == "test"
+
+    passing = {
+        "bp": {20: [0.91] * 5, 32: [0.92] * 5, 56: [0.93] * 5},
+        "dfa": {20: [0.80] * 5, 32: [0.81] * 5, 56: [0.84] * 5},
+        "clean_kp": {20: [0.91] * 5, 32: [0.92] * 5, 56: [0.93] * 5},
+        "dynamic": {20: [0.91] * 5, 32: [0.92] * 5, 56: [0.93] * 5},
+    }
+    alignments = {20: [0.95] * 5, 32: [0.93] * 5, 56: [0.90] * 5}
+    checks, _ = oral_a_checks(passing, alignments, [])
+    assert all(checks.values())
+    failing = copy.deepcopy(passing)
+    failing["dynamic"][56] = [0.86] * 5
+    checks, _ = oral_a_checks(failing, alignments, [])
+    assert not all(checks.values())
+
+    d4_path = os.path.join(
+        ROOT, "results", "kp_dynamic_projection_confirmation_gate.json")
+    bci_path = os.path.join(
+        ROOT, "results", "bci_v2_calibrated_confirmation_gate.json")
+    with open(bci_path) as handle:
+        old_gate = json.load(handle)
+    old_gate["protocol"] = "oral_b_td_confirmation_v1"
+    with tempfile.TemporaryDirectory() as tmp:
+        fake = os.path.join(tmp, "old_r2.json")
+        with open(fake, "w") as handle:
+            json.dump(old_gate, handle)
+        try:
+            require_prerequisites(d4_path, fake)
+        except ValueError:
+            pass
+        else:
+            raise AssertionError("legacy oral-B gate must not open recovery v2")
+
+    print("oral-A recovery-v2 endpoint-free smoke passed")
+
+
+if __name__ == "__main__":
+    main()