experiments: freeze oral A confirmation gates

5 files changed, 380 insertions, 1 deletions

diff --git a/ORAL_A.md b/ORAL_A.md
index e3427c4..1a07708 100644
--- a/ORAL_A.md
+++ b/ORAL_A.md
@@ -118,7 +118,9 @@ No recovery branch follows an A3 failure.
 
 Only after A3 passes, run exact BP, selected DFA, and selected SDIL at depths
 20, 32, and 56 for model/data-loader seeds 10--14. Hyperparameters and epoch
-budgets are copied without depth-specific tuning. The CIFAR-10 test set is
+budgets are copied without depth-specific tuning. Confirmation refits on all
+50,000 original training examples (`val_examples=0`) after development has
+closed. The CIFAR-10 test set is
 evaluated exactly once at the end of every run; no intermediate test metric is
 logged or used for selection.
 
diff --git a/experiments/analyze_oral_a_confirmation.py b/experiments/analyze_oral_a_confirmation.py
new file mode 100755
index 0000000..5de05cb
--- /dev/null
+++ b/experiments/analyze_oral_a_confirmation.py
@@ -0,0 +1,146 @@
+#!/usr/bin/env python3
+"""Audit all 45 A4 trajectories and apply the frozen Oral-A gate."""
+import argparse
+import glob
+import json
+import math
+import os
+import statistics
+
+
+METHODS = ("bp", "dfa", "sdil")
+DEPTHS = (20, 32, 56)
+SEEDS = tuple(range(10, 15))
+
+
+def read(path):
+    with open(path) as handle:
+        record = json.load(handle)
+    args = record["args"]
+    key = (args["mode"], int(args["depth"]), int(args["seed"]))
+    if key[0] not in METHODS or key[1] not in DEPTHS or key[2] not in SEEDS:
+        raise ValueError(f"unexpected A4 key {key}: {path}")
+    expected = {
+        "width": 16, "epochs": 200, "val_examples": 0,
+        "eval_split": "test", "eval_every": 0, "normalization": "batchnorm",
+    }
+    for name, value in expected.items():
+        if args.get(name) != value:
+            raise ValueError(f"{path}: {name} drift")
+    if record["provenance"]["git_tracked_dirty"]:
+        raise ValueError(f"tracked-dirty result: {path}")
+    protocol = record["evaluation_protocol"]
+    if protocol["test_evaluations"] != 1 or protocol["test_used_for_selection"]:
+        raise ValueError(f"invalid test protocol: {path}")
+    accuracy = float(record["final"]["accuracy"])
+    loss = float(record["final"]["loss"])
+    alignment = (None if record["diagnostics"] is None
+                 else float(record["diagnostics"]["early_third_mean"]))
+    return key, {
+        "path": path, "accuracy": accuracy, "loss": loss,
+        "finite": record["final"]["finite"] and math.isfinite(accuracy + loss),
+        "early_third_alignment": alignment,
+        "total_macs": record["work"]["total_macs_estimate"],
+        "logical_queries": record["work"]["logical_batch_loss_queries"],
+        "peak_memory": record["hardware"]["peak_memory_allocated_bytes"],
+        "source_commit": record["provenance"]["git_commit"],
+    }
+
+
+def mean_sd(values):
+    return {"mean": statistics.mean(values), "sample_sd": statistics.stdev(values)}
+
+
+def frontier_members(points, cost):
+    members = []
+    for candidate in points:
+        dominated = any(
+            other["accuracy"] >= candidate["accuracy"]
+            and other[cost] <= candidate[cost]
+            and (other["accuracy"] > candidate["accuracy"]
+                 or other[cost] < candidate[cost])
+            for other in points if other is not candidate)
+        if not dominated:
+            members.append(candidate["name"])
+    return members
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--input", default="results/oral_a_confirm")
+    parser.add_argument("--out", default="results/oral_a_confirmation.json")
+    args = parser.parse_args()
+    rows = dict(read(path) for path in sorted(glob.glob(os.path.join(args.input, "*.json"))))
+    expected = {(method, depth, seed) for method in METHODS
+                for depth in DEPTHS for seed in SEEDS}
+    if set(rows) != expected or len(rows) != 45:
+        raise ValueError(f"incomplete A4 grid: missing={expected-set(rows)}, extra={set(rows)-expected}")
+    summaries = {}
+    for method in METHODS:
+        for depth in DEPTHS:
+            group = [rows[(method, depth, seed)] for seed in SEEDS]
+            summaries[f"{method}_d{depth}"] = {
+                "accuracy": mean_sd([row["accuracy"] for row in group]),
+                "total_macs": mean_sd([row["total_macs"] for row in group]),
+                "peak_memory": mean_sd([row["peak_memory"] for row in group]),
+                "logical_queries": mean_sd([row["logical_queries"] for row in group]),
+                "early_third_alignment": (None if method == "bp" else mean_sd(
+                    [row["early_third_alignment"] for row in group])),
+            }
+    near_bp = all(
+        summaries[f"sdil_d{depth}"]["accuracy"]["mean"]
+        >= summaries[f"bp_d{depth}"]["accuracy"]["mean"] - 0.02
+        for depth in DEPTHS)
+    d56_advantage = (
+        summaries["sdil_d56"]["accuracy"]["mean"]
+        >= summaries["dfa_d56"]["accuracy"]["mean"] + 0.02)
+    paired_depth_changes = [
+        rows[("sdil", 56, seed)]["accuracy"] - rows[("sdil", 20, seed)]["accuracy"]
+        for seed in SEEDS]
+    depth_preserved = statistics.mean(paired_depth_changes) >= -0.02
+    align20 = summaries["sdil_d20"]["early_third_alignment"]["mean"]
+    align56 = summaries["sdil_d56"]["early_third_alignment"]["mean"]
+    alignment_gate = align56 >= 0.05 and align56 >= 0.30 * align20
+    local_points = []
+    for method in ("dfa", "sdil"):
+        for depth in DEPTHS:
+            summary = summaries[f"{method}_d{depth}"]
+            local_points.append({
+                "name": f"{method}_d{depth}",
+                "accuracy": summary["accuracy"]["mean"],
+                "total_macs": summary["total_macs"]["mean"],
+                "logical_queries": summary["logical_queries"]["mean"],
+                "peak_memory": summary["peak_memory"]["mean"],
+            })
+    frontiers = {cost: frontier_members(local_points, cost) for cost in (
+        "total_macs", "logical_queries", "peak_memory")}
+    nondominated = all(
+        any(name.startswith("sdil_") for name in members)
+        for members in frontiers.values())
+    checks = {
+        "all_finite": all(row["finite"] for row in rows.values()),
+        "within_2pt_bp_each_depth": near_bp,
+        "d56_beats_dfa_2pt": d56_advantage,
+        "paired_d20_to_d56_no_worse_than_minus_2pt": depth_preserved,
+        "d56_alignment_level_and_retention": alignment_gate,
+        "sdil_on_each_local_cost_frontier": nondominated,
+    }
+    output = {
+        "protocol": "oral_a_A4_v1",
+        "status": "passed" if all(checks.values()) else "failed",
+        "checks": checks, "summaries": summaries,
+        "local_frontiers": frontiers,
+        "paired_sdil_d20_to_d56_accuracy_change": mean_sd(paired_depth_changes),
+        "rows": {f"{method}_d{depth}_s{seed}": row
+                 for (method, depth, seed), row in rows.items()},
+    }
+    os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True)
+    with open(args.out, "w") as handle:
+        json.dump(output, handle, indent=2, sort_keys=True)
+        handle.write("\n")
+    print(json.dumps({"status": output["status"], "checks": checks,
+                      "summaries": summaries}, indent=2))
+
+
+if __name__ == "__main__":
+    main()
diff --git a/experiments/analyze_oral_a_full.py b/experiments/analyze_oral_a_full.py
new file mode 100755
index 0000000..f22d55c
--- /dev/null
+++ b/experiments/analyze_oral_a_full.py
@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+"""Apply the frozen A3 full seed-0 ResNet-20 advancement gate."""
+import argparse
+import json
+import math
+import os
+
+
+SPLIT_HASH = "8328b206a97c420e49e54e3eca4abe3274c4756b084355784ea3fb8059e4515b"
+
+
+def read_result(path, mode):
+    with open(path) as handle:
+        record = json.load(handle)
+    args = record["args"]
+    expected = {
+        "mode": mode, "depth": 20, "width": 16, "seed": 0,
+        "epochs": 200, "val_examples": 5000,
+        "eval_split": "validation", "normalization": "batchnorm",
+    }
+    for key, value in expected.items():
+        if args.get(key) != value:
+            raise ValueError(f"{path}: {key} drift")
+    if record["provenance"]["git_tracked_dirty"]:
+        raise ValueError(f"tracked-dirty result: {path}")
+    if record["split"]["validation_index_sha256"] != SPLIT_HASH:
+        raise ValueError(f"split drift: {path}")
+    if record["evaluation_protocol"]["test_evaluations"]:
+        raise ValueError(f"A3 touched test: {path}")
+    alignment = None
+    if record["diagnostics"] is not None:
+        alignment = float(record["diagnostics"]["early_third_mean"])
+    accuracy = float(record["final"]["accuracy"])
+    loss = float(record["final"]["loss"])
+    return {
+        "mode": mode, "path": path, "accuracy": accuracy, "loss": loss,
+        "finite": record["final"]["finite"] and math.isfinite(accuracy + loss),
+        "early_third_alignment": alignment,
+        "total_macs": record["work"]["total_macs_estimate"],
+        "peak_memory": record["hardware"]["peak_memory_allocated_bytes"],
+        "source_commit": record["provenance"]["git_commit"],
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--bp_selection", default="results/oral_a_bp_selection.json")
+    parser.add_argument("--dfa", default="results/oral_a_dev/dfa_full_r20_s0.json")
+    parser.add_argument("--sdil", default="results/oral_a_dev/sdil_full_r20_s0.json")
+    parser.add_argument("--out", default="results/oral_a_full_gate.json")
+    args = parser.parse_args()
+    with open(args.bp_selection) as handle:
+        bp_selection = json.load(handle)
+    if not bp_selection["status"].startswith("passed_"):
+        raise ValueError("A1 BP reference is not viable")
+    bp = read_result(bp_selection["selected"]["path"], "bp")
+    dfa = read_result(args.dfa, "dfa")
+    sdil = read_result(args.sdil, "sdil")
+    checks = {
+        "bp_at_least_90": bp["accuracy"] >= 0.90,
+        "sdil_within_5pt_bp": sdil["accuracy"] >= bp["accuracy"] - 0.05,
+        "sdil_beats_dfa_2pt": sdil["accuracy"] >= dfa["accuracy"] + 0.02,
+        "sdil_alignment_at_least_0p05": (
+            sdil["early_third_alignment"] is not None
+            and sdil["early_third_alignment"] >= 0.05),
+        "all_finite": all(row["finite"] for row in (bp, dfa, sdil)),
+        "sdil_macs_no_more_than_bp": sdil["total_macs"] <= bp["total_macs"],
+    }
+    output = {
+        "protocol": "oral_a_A3_v1",
+        "status": "passed" if all(checks.values()) else "failed",
+        "checks": checks, "rows": [bp, dfa, sdil],
+        "confirmation_test_seeds_touched": False,
+    }
+    os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True)
+    with open(args.out, "w") as handle:
+        json.dump(output, handle, indent=2, sort_keys=True)
+        handle.write("\n")
+    print(json.dumps(output, indent=2))
+
+
+if __name__ == "__main__":
+    main()
diff --git a/experiments/oral_a_confirmation.py b/experiments/oral_a_confirmation.py
new file mode 100755
index 0000000..e8b0dba
--- /dev/null
+++ b/experiments/oral_a_confirmation.py
@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+"""Run a deterministic shard of the frozen A4 independent test panel."""
+import argparse
+import json
+import os
+import subprocess
+import sys
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--gate", default="results/oral_a_full_gate.json")
+    parser.add_argument("--bp_selection", default="results/oral_a_bp_selection.json")
+    parser.add_argument("--short_selection", default="results/oral_a_short_selection.json")
+    parser.add_argument("--device", default="cuda")
+    parser.add_argument("--shard_index", type=int, default=0)
+    parser.add_argument("--num_shards", type=int, default=1)
+    parser.add_argument("--dry_run", action="store_true")
+    args = parser.parse_args()
+    if not 0 <= args.shard_index < args.num_shards:
+        raise ValueError("invalid shard index")
+    with open(args.gate) as handle:
+        gate = json.load(handle)
+    with open(args.bp_selection) as handle:
+        bp_selection = json.load(handle)
+    with open(args.short_selection) as handle:
+        short = json.load(handle)
+    if gate["status"] != "passed":
+        raise ValueError("A3 did not pass; A4 test confirmation is prohibited")
+    if short["status"] != "selected" or not bp_selection["status"].startswith("passed_"):
+        raise ValueError("development selections are incomplete")
+    dfa = short["selected_dfa"]
+    sdil = short["selected_sdil"]
+    bp_variant = bp_selection["selected"]["variant"]
+    jobs = []
+    for depth in (20, 32, 56):
+        for seed in range(10, 15):
+            common = [
+                sys.executable, "experiments/conv_run.py", "--device", args.device,
+                "--depth", str(depth), "--width", "16", "--seed", str(seed),
+                "--loader_seed", str(seed), "--perturb_seed", str(1000 + seed),
+                "--batch_size", "128", "--epochs", "200", "--val_examples", "0",
+                "--eval_split", "test", "--eval_every", "0", "--augment_train", "1",
+                "--momentum", "0.9", "--normalization", "batchnorm",
+            ]
+            if bp_variant == "primary":
+                bp_schedule = [
+                    "--lr", "0.1", "--lr_schedule", "step",
+                    "--lr_milestones", "100,150", "--lr_gamma", "0.1",
+                    "--warmup_epochs", "0", "--weight_decay", "1e-4"]
+            else:
+                bp_schedule = [
+                    "--lr", "0.1", "--lr_schedule", "cosine",
+                    "--warmup_epochs", "5", "--weight_decay", "5e-4"]
+            jobs.append((f"bp_d{depth}_s{seed}", common + [
+                "--mode", "bp", *bp_schedule,
+                "--out", f"results/oral_a_confirm/bp_d{depth}_s{seed}.json"]))
+            local_schedule = [
+                "--output_lr", "0.1", "--lr_schedule", "step",
+                "--lr_milestones", "100,150", "--lr_gamma", "0.1",
+                "--warmup_epochs", "0", "--weight_decay", "1e-4",
+                "--alignment_probe", "32"]
+            jobs.append((f"dfa_d{depth}_s{seed}", common + local_schedule + [
+                "--mode", "dfa", "--lr", str(dfa["lr"]), "--a_scale", "1",
+                "--vectorizer_mode", "spatial_template",
+                "--out", f"results/oral_a_confirm/dfa_d{depth}_s{seed}.json"]))
+            jobs.append((f"sdil_d{depth}_s{seed}", common + local_schedule + [
+                "--mode", "sdil", "--lr", str(sdil["lr"]),
+                "--vectorizer_mode", sdil["vectorizer_mode"],
+                "--a_scale", str(sdil["a_scale"]), "--eta_A", str(sdil["eta_A"]),
+                "--a_warmup_steps", "100", "--pert_sigma", "0.01",
+                "--pert_directions", "1", "--pert_every", "4",
+                "--out", f"results/oral_a_confirm/sdil_d{depth}_s{seed}.json"]))
+    os.makedirs("results/oral_a_confirm", exist_ok=True)
+    for index, (tag, command) in enumerate(jobs):
+        if index % args.num_shards != args.shard_index:
+            continue
+        print(tag, " ".join(command), flush=True)
+        if not args.dry_run:
+            subprocess.run(command, check=True)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/experiments/oral_a_full_development.py b/experiments/oral_a_full_development.py
new file mode 100755
index 0000000..0d8b8eb
--- /dev/null
+++ b/experiments/oral_a_full_development.py
@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+"""Run a shard of the frozen A3 full ResNet-20 local-method development gate."""
+import argparse
+import json
+import os
+import subprocess
+import sys
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--short_selection", default="results/oral_a_short_selection.json")
+    parser.add_argument("--bp_selection", default="results/oral_a_bp_selection.json")
+    parser.add_argument("--device", default="cuda")
+    parser.add_argument("--shard_index", type=int, default=0)
+    parser.add_argument("--num_shards", type=int, default=1)
+    parser.add_argument("--dry_run", action="store_true")
+    args = parser.parse_args()
+    if not 0 <= args.shard_index < args.num_shards:
+        raise ValueError("invalid shard index")
+    with open(args.short_selection) as handle:
+        short = json.load(handle)
+    with open(args.bp_selection) as handle:
+        bp = json.load(handle)
+    if short["status"] != "selected":
+        raise ValueError("A2b advancement gate did not pass")
+    if not bp["status"].startswith("passed_"):
+        raise ValueError("A1 BP reference gate did not pass")
+    dfa = short["selected_dfa"]
+    sdil = short["selected_sdil"]
+    common = [
+        sys.executable, "experiments/conv_run.py", "--device", args.device,
+        "--depth", "20", "--width", "16", "--seed", "0", "--loader_seed", "0",
+        "--batch_size", "128", "--epochs", "200", "--val_examples", "5000",
+        "--split_seed", "2027", "--eval_split", "validation", "--eval_every", "20",
+        "--augment_train", "1", "--lr_schedule", "step", "--lr_milestones", "100,150",
+        "--lr_gamma", "0.1", "--warmup_epochs", "0", "--momentum", "0.9",
+        "--weight_decay", "1e-4", "--normalization", "batchnorm",
+        "--output_lr", "0.1", "--alignment_probe", "32",
+    ]
+    jobs = [
+        ("dfa", common + [
+            "--mode", "dfa", "--lr", str(dfa["lr"]), "--a_scale", "1",
+            "--vectorizer_mode", "spatial_template",
+            "--out", "results/oral_a_dev/dfa_full_r20_s0.json"]),
+        ("sdil", common + [
+            "--mode", "sdil", "--lr", str(sdil["lr"]),
+            "--vectorizer_mode", sdil["vectorizer_mode"],
+            "--a_scale", str(sdil["a_scale"]), "--eta_A", str(sdil["eta_A"]),
+            "--a_warmup_steps", "100", "--pert_sigma", "0.01",
+            "--pert_directions", "1", "--pert_every", "4",
+            "--out", "results/oral_a_dev/sdil_full_r20_s0.json"]),
+    ]
+    os.makedirs("results/oral_a_dev", exist_ok=True)
+    for index, (tag, command) in enumerate(jobs):
+        if index % args.num_shards != args.shard_index:
+            continue
+        print(tag, " ".join(command), flush=True)
+        if not args.dry_run:
+            subprocess.run(command, check=True)
+
+
+if __name__ == "__main__":
+    main()