protocol: freeze post-failure Oral-A v2 funnel

1 files changed, 116 insertions, 0 deletions

diff --git a/experiments/analyze_oral_a_v2_full.py b/experiments/analyze_oral_a_v2_full.py
new file mode 100644
index 0000000..16f7e78
--- /dev/null
+++ b/experiments/analyze_oral_a_v2_full.py
@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+"""Apply the frozen Oral-A-v2 full-validation gate."""
+import argparse
+import json
+import math
+import os
+
+
+SPLIT_HASH = "8328b206a97c420e49e54e3eca4abe3274c4756b084355784ea3fb8059e4515b"
+
+
+def finite_tree(value):
+    if isinstance(value, dict):
+        return all(finite_tree(item) for item in value.values())
+    if isinstance(value, list):
+        return all(finite_tree(item) for item in value)
+    if isinstance(value, float):
+        return math.isfinite(value)
+    return True
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--run", default="results/oral_a_v2_dev/sdil_full_r20_s0.json")
+    parser.add_argument(
+        "--selection", default="results/oral_a_v2_calibration_gate.json")
+    parser.add_argument(
+        "--bp", default="results/oral_a_dev/bp_reference_primary.json")
+    parser.add_argument(
+        "--dfa", default="results/oral_a_dev/dfa_full_r20_s0.json")
+    parser.add_argument("--out", default="results/oral_a_v2_full_gate.json")
+    args = parser.parse_args()
+    with open(args.run) as handle:
+        run = json.load(handle)
+    with open(args.selection) as handle:
+        selection = json.load(handle)
+    with open(args.bp) as handle:
+        bp = json.load(handle)
+    with open(args.dfa) as handle:
+        dfa = json.load(handle)
+    if selection["status"] != "passed":
+        raise ValueError("v2 causal-capture gate did not pass")
+    chosen = selection["selected"]["channel_subspace"]
+    expected = {
+        "mode": "sdil", "depth": 20, "width": 16, "seed": 0,
+        "epochs": 200, "val_examples": 5000, "lr": 0.03,
+        "output_lr": 0.1, "lr_schedule": "step",
+        "lr_milestones": "100,150", "lr_gamma": 0.1,
+        "a_scale": 0.0, "eta_A": chosen["eta_A"],
+        "a_warmup_steps": 400, "apical_calibration_mode": "channel_subspace",
+        "pert_sigma": 0.01, "pert_directions": 1, "pert_every": 4,
+        "normalization": "batchnorm", "vectorizer_mode": "channel_gated",
+    }
+    for key, value in expected.items():
+        if run["args"].get(key) != value:
+            raise ValueError(
+                f"v2 run {key}={run['args'].get(key)!r}, expected {value!r}")
+    if run["provenance"]["git_tracked_dirty"]:
+        raise ValueError("tracked-dirty v2 result")
+    if run["split"]["validation_index_sha256"] != SPLIT_HASH:
+        raise ValueError("v2 split drift")
+    if run["evaluation_protocol"]["test_evaluations"] != 0:
+        raise ValueError("test endpoint touched during v2 development")
+    values = run["diagnostics"]["teaching_negative_gradient_cosine"]
+    early_count = max(1, len(values) // 3)
+    early = sum(values[:early_count]) / early_count
+    sdil_accuracy = run["final"]["accuracy"]
+    bp_accuracy = bp["final"]["accuracy"]
+    dfa_accuracy = dfa["final"]["accuracy"]
+    checks = {
+        "all_metrics_finite": run["final"]["finite"] and finite_tree(run),
+        "bp_reference_at_least_90pct": bp_accuracy >= 0.90,
+        "sdil_within_5pt_of_bp": sdil_accuracy >= bp_accuracy - 0.05,
+        "sdil_at_least_2pt_above_dfa": sdil_accuracy >= dfa_accuracy + 0.02,
+        "early_third_alignment_at_least_0.05": early >= 0.05,
+        "sdil_macs_no_more_than_bp": (
+            run["work"]["total_macs_estimate"]
+            <= bp["work"]["total_macs_estimate"]),
+    }
+    passed = all(checks.values())
+    output = {
+        "protocol": "oral_a_v2_full_validation_v1",
+        "status": "passed" if passed else "failed",
+        "checks": checks,
+        "metrics": {
+            "sdil_validation_accuracy": sdil_accuracy,
+            "bp_validation_accuracy": bp_accuracy,
+            "dfa_validation_accuracy": dfa_accuracy,
+            "early_third_alignment": early,
+            "sdil_total_macs": run["work"]["total_macs_estimate"],
+            "bp_total_macs": bp["work"]["total_macs_estimate"],
+        },
+        "sources": {"sdil": args.run, "bp": args.bp, "dfa": args.dfa},
+        "confirmation_test_seeds_touched": False,
+        "review_score_before": 5,
+        "review_score_after": 6 if passed else 5,
+        "review_score_rationale": (
+            "full standard-scale validation gate passed; independent depth "
+            "confirmation still required"
+            if passed else
+            "full standard-scale validation gate failed; controlled evidence "
+            "remains unchanged"),
+    }
+    os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True)
+    with open(args.out, "w") as handle:
+        json.dump(output, handle, indent=2, sort_keys=True)
+        handle.write("\n")
+    print(json.dumps({
+        "status": output["status"], "checks": checks,
+        "metrics": output["metrics"],
+    }, indent=2))
+
+
+if __name__ == "__main__":
+    main()