1 files changed, 84 insertions, 0 deletions
diff --git a/experiments/analyze_oral_a_short.py b/experiments/analyze_oral_a_short.py
new file mode 100755
index 0000000..375718a
--- /dev/null
+++ b/experiments/analyze_oral_a_short.py
@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+"""Apply the frozen A2b selector and advancement gate."""
+import argparse
+import glob
+import json
+import math
+import os
+
+
+def read(path):
+    with open(path) as handle:
+        record = json.load(handle)
+    args = record["args"]
+    expected = {
+        "depth": 20, "width": 16, "seed": 0, "epochs": 20,
+        "train_limit": 10000, "val_examples": 5000,
+        "eval_split": "validation", "normalization": "batchnorm",
+    }
+    for key, value in expected.items():
+        if args.get(key) != value:
+            raise ValueError(f"{path}: {key} drift")
+    if record["provenance"]["git_tracked_dirty"]:
+        raise ValueError(f"tracked-dirty result: {path}")
+    if record["evaluation_protocol"]["test_evaluations"]:
+        raise ValueError(f"short screen touched test: {path}")
+    accuracy = float(record["final"]["accuracy"])
+    loss = float(record["final"]["loss"])
+    return {
+        "path": path, "mode": args["mode"], "lr": float(args["lr"]),
+        "vectorizer_mode": args.get("vectorizer_mode"),
+        "a_scale": float(args.get("a_scale", 0.0)),
+        "eta_A": float(args.get("eta_A", 0.0)),
+        "accuracy": accuracy, "loss": loss,
+        "finite": record["final"]["finite"] and math.isfinite(accuracy + loss),
+        "total_macs": record["work"]["total_macs_estimate"],
+        "source_commit": record["provenance"]["git_commit"],
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--input", default="results/oral_a_short")
+    parser.add_argument("--out", default="results/oral_a_short_selection.json")
+    args = parser.parse_args()
+    rows = [read(path) for path in sorted(glob.glob(os.path.join(args.input, "*.json")))]
+    if len(rows) != 10:
+        raise ValueError(f"expected 10 A2b runs, found {len(rows)}")
+    if len({row["source_commit"] for row in rows}) != 1:
+        raise ValueError("A2b source commits differ")
+    bp = [row for row in rows if row["mode"] == "bp"]
+    dfa = [row for row in rows if row["mode"] == "dfa"]
+    sdil = [row for row in rows if row["mode"] == "sdil"]
+    if len(bp) != 1 or len(dfa) != 3 or len(sdil) != 6:
+        raise ValueError("A2b method grid is incomplete")
+    selected_dfa = sorted(
+        dfa, key=lambda row: (-row["accuracy"], row["total_macs"], row["lr"]))[0]
+    best_accuracy = max(row["accuracy"] for row in sdil if row["finite"])
+    gated_near_best = [row for row in sdil if row["finite"]
+                       and row["vectorizer_mode"] == "channel_gated"
+                       and row["accuracy"] >= best_accuracy - 0.005]
+    pool = gated_near_best or [row for row in sdil if row["finite"]]
+    selected_sdil = sorted(
+        pool, key=lambda row: (-row["accuracy"], row["total_macs"], row["lr"]))[0]
+    passed = (bp[0]["finite"] and selected_dfa["finite"] and selected_sdil["finite"]
+              and bp[0]["accuracy"] >= 0.50
+              and selected_sdil["accuracy"] >= 0.35
+              and selected_sdil["accuracy"] >= selected_dfa["accuracy"] - 0.10)
+    output = {
+        "protocol": "oral_a_A2b_v1",
+        "status": "selected" if passed else "failed_advancement_gate",
+        "rows": rows, "selected_bp": bp[0],
+        "selected_dfa": selected_dfa, "selected_sdil": selected_sdil,
+        "confirmation_test_seeds_touched": False,
+    }
+    os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True)
+    with open(args.out, "w") as handle:
+        json.dump(output, handle, indent=2, sort_keys=True)
+        handle.write("\n")
+    print(json.dumps({key: output[key] for key in (
+        "status", "selected_bp", "selected_dfa", "selected_sdil")}, indent=2))
+
+
+if __name__ == "__main__":
+    main()