experiments: freeze oral A development gates

7 files changed, 391 insertions, 2 deletions

diff --git a/experiments/analyze_oral_a_apical.py b/experiments/analyze_oral_a_apical.py
new file mode 100755
index 0000000..1d10edb
--- /dev/null
+++ b/experiments/analyze_oral_a_apical.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+"""Apply the frozen A2a selector to convolutional apical-screen records."""
+import argparse
+import glob
+import json
+import math
+import os
+
+
+MODES = ("spatial_template", "channel_gated")
+EXPECTED_SCALES = (0.0, 1.0)
+EXPECTED_RATES = (0.001, 0.01, 0.1)
+
+
+def load(path):
+    with open(path) as handle:
+        record = json.load(handle)
+    args = record["args"]
+    if record["provenance"]["git_tracked_dirty"]:
+        raise ValueError(f"tracked-dirty result: {path}")
+    expected = {
+        "mode": "sdil", "depth": 20, "width": 16, "seed": 0,
+        "epochs": 0, "train_limit": 10000, "val_examples": 5000,
+        "a_warmup_steps": 100, "pert_directions": 1, "pert_every": 4,
+        "normalization": "batchnorm",
+    }
+    for key, value in expected.items():
+        if args.get(key) != value:
+            raise ValueError(f"{path}: {key}={args.get(key)!r}, expected {value!r}")
+    if record["split"]["validation_index_sha256"] != (
+            "8328b206a97c420e49e54e3eca4abe3274c4756b084355784ea3fb8059e4515b"):
+        raise ValueError(f"split drift: {path}")
+    diagnostics = record.get("diagnostics")
+    warmup = record.get("apical_warmup", {}).get("last")
+    if diagnostics is None or warmup is None:
+        raise ValueError(f"missing diagnostics/warmup: {path}")
+    values = diagnostics["teaching_negative_gradient_cosine"]
+    early_count = max(1, len(values) // 3)
+    metrics = {
+        "early_third_alignment": sum(values[:early_count]) / early_count,
+        "all_layer_alignment": sum(values) / len(values),
+        "calibration_mse": warmup["calibration_mse"],
+        "prediction_target_cosine": warmup["prediction_target_cosine"],
+    }
+    finite = (record["final"]["finite"]
+              and all(math.isfinite(value) for value in metrics.values()))
+    return {
+        "path": path,
+        "source_commit": record["provenance"]["git_commit"],
+        "vectorizer_mode": args["vectorizer_mode"],
+        "a_scale": float(args["a_scale"]),
+        "eta_A": float(args["eta_A"]),
+        "metrics": metrics,
+        "eligible": finite and metrics["early_third_alignment"] > 0.0,
+        "vectorizer_parameters": record["architecture"]["vectorizer_parameters"],
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--input", default="results/oral_a_apical_screen")
+    parser.add_argument("--out", default="results/oral_a_apical_selection.json")
+    args = parser.parse_args()
+    paths = sorted(glob.glob(os.path.join(args.input, "*.json")))
+    rows = [load(path) for path in paths]
+    observed = {(row["vectorizer_mode"], row["a_scale"], row["eta_A"])
+                for row in rows}
+    expected = {(mode, scale, rate) for mode in MODES
+                for scale in EXPECTED_SCALES for rate in EXPECTED_RATES}
+    if observed != expected or len(rows) != len(expected):
+        raise ValueError(f"incomplete A2a grid: missing={expected-observed}, extra={observed-expected}")
+    if len({row["source_commit"] for row in rows}) != 1:
+        raise ValueError("A2a source commits differ")
+    selected = {}
+    for mode in MODES:
+        eligible = [row for row in rows
+                    if row["vectorizer_mode"] == mode and row["eligible"]]
+        if eligible:
+            eligible.sort(key=lambda row: (
+                -row["metrics"]["early_third_alignment"],
+                -row["metrics"]["all_layer_alignment"],
+                row["eta_A"], row["a_scale"]))
+            selected[mode] = eligible[0]
+    output = {
+        "protocol": "oral_a_A2a_v1",
+        "status": ("selected" if len(selected) == len(MODES)
+                   else "failed_no_eligible_family"),
+        "rows": rows,
+        "selected": selected,
+        "confirmation_test_seeds_touched": False,
+    }
+    os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True)
+    with open(args.out, "w") as handle:
+        json.dump(output, handle, indent=2, sort_keys=True)
+        handle.write("\n")
+    print(json.dumps({"status": output["status"], "selected": selected}, indent=2))
+
+
+if __name__ == "__main__":
+    main()
diff --git a/experiments/analyze_oral_a_bp.py b/experiments/analyze_oral_a_bp.py
new file mode 100755
index 0000000..d309273
--- /dev/null
+++ b/experiments/analyze_oral_a_bp.py
@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+"""Apply the frozen A1 exact-BP viability gate."""
+import argparse
+import json
+import math
+import os
+
+
+def read(path, variant):
+    with open(path) as handle:
+        record = json.load(handle)
+    args = record["args"]
+    expected = {
+        "mode": "bp", "depth": 20, "width": 16, "seed": 0,
+        "epochs": 200, "val_examples": 5000, "eval_split": "validation",
+        "normalization": "batchnorm",
+    }
+    for key, value in expected.items():
+        if args.get(key) != value:
+            raise ValueError(f"{path}: {key}={args.get(key)!r}, expected {value!r}")
+    if record["provenance"]["git_tracked_dirty"]:
+        raise ValueError(f"tracked-dirty result: {path}")
+    if record["split"]["validation_index_sha256"] != (
+            "8328b206a97c420e49e54e3eca4abe3274c4756b084355784ea3fb8059e4515b"):
+        raise ValueError(f"split drift: {path}")
+    accuracy = float(record["final"]["accuracy"])
+    loss = float(record["final"]["loss"])
+    return {
+        "variant": variant, "path": path,
+        "source_commit": record["provenance"]["git_commit"],
+        "accuracy": accuracy, "loss": loss,
+        "finite": record["final"]["finite"] and math.isfinite(accuracy + loss),
+        "test_evaluations": record["evaluation_protocol"]["test_evaluations"],
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--primary", default="results/oral_a_dev/bp_reference_primary.json")
+    parser.add_argument("--recovery", default="results/oral_a_dev/bp_reference_recovery.json")
+    parser.add_argument("--out", default="results/oral_a_bp_selection.json")
+    args = parser.parse_args()
+    primary = read(args.primary, "primary")
+    rows = [primary]
+    if primary["finite"] and primary["accuracy"] >= 0.90:
+        status = "passed_primary"
+        selected = primary
+    elif not os.path.exists(args.recovery):
+        status = "recovery_required"
+        selected = None
+    else:
+        recovery = read(args.recovery, "recovery")
+        rows.append(recovery)
+        selected = max(rows, key=lambda row: row["accuracy"] if row["finite"] else -1.0)
+        status = ("passed_recovery" if selected["finite"] and selected["accuracy"] >= 0.90
+                  else "failed_no_viable_reference")
+    if any(row["test_evaluations"] for row in rows):
+        raise ValueError("A1 must not evaluate test")
+    output = {
+        "protocol": "oral_a_A1_v1", "status": status,
+        "rows": rows, "selected": selected,
+        "confirmation_test_seeds_touched": False,
+    }
+    os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True)
+    with open(args.out, "w") as handle:
+        json.dump(output, handle, indent=2, sort_keys=True)
+        handle.write("\n")
+    print(json.dumps(output, indent=2))
+
+
+if __name__ == "__main__":
+    main()
diff --git a/experiments/analyze_oral_a_short.py b/experiments/analyze_oral_a_short.py
new file mode 100755
index 0000000..375718a
--- /dev/null
+++ b/experiments/analyze_oral_a_short.py
@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+"""Apply the frozen A2b selector and advancement gate."""
+import argparse
+import glob
+import json
+import math
+import os
+
+
+def read(path):
+    with open(path) as handle:
+        record = json.load(handle)
+    args = record["args"]
+    expected = {
+        "depth": 20, "width": 16, "seed": 0, "epochs": 20,
+        "train_limit": 10000, "val_examples": 5000,
+        "eval_split": "validation", "normalization": "batchnorm",
+    }
+    for key, value in expected.items():
+        if args.get(key) != value:
+            raise ValueError(f"{path}: {key} drift")
+    if record["provenance"]["git_tracked_dirty"]:
+        raise ValueError(f"tracked-dirty result: {path}")
+    if record["evaluation_protocol"]["test_evaluations"]:
+        raise ValueError(f"short screen touched test: {path}")
+    accuracy = float(record["final"]["accuracy"])
+    loss = float(record["final"]["loss"])
+    return {
+        "path": path, "mode": args["mode"], "lr": float(args["lr"]),
+        "vectorizer_mode": args.get("vectorizer_mode"),
+        "a_scale": float(args.get("a_scale", 0.0)),
+        "eta_A": float(args.get("eta_A", 0.0)),
+        "accuracy": accuracy, "loss": loss,
+        "finite": record["final"]["finite"] and math.isfinite(accuracy + loss),
+        "total_macs": record["work"]["total_macs_estimate"],
+        "source_commit": record["provenance"]["git_commit"],
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--input", default="results/oral_a_short")
+    parser.add_argument("--out", default="results/oral_a_short_selection.json")
+    args = parser.parse_args()
+    rows = [read(path) for path in sorted(glob.glob(os.path.join(args.input, "*.json")))]
+    if len(rows) != 10:
+        raise ValueError(f"expected 10 A2b runs, found {len(rows)}")
+    if len({row["source_commit"] for row in rows}) != 1:
+        raise ValueError("A2b source commits differ")
+    bp = [row for row in rows if row["mode"] == "bp"]
+    dfa = [row for row in rows if row["mode"] == "dfa"]
+    sdil = [row for row in rows if row["mode"] == "sdil"]
+    if len(bp) != 1 or len(dfa) != 3 or len(sdil) != 6:
+        raise ValueError("A2b method grid is incomplete")
+    selected_dfa = sorted(
+        dfa, key=lambda row: (-row["accuracy"], row["total_macs"], row["lr"]))[0]
+    best_accuracy = max(row["accuracy"] for row in sdil if row["finite"])
+    gated_near_best = [row for row in sdil if row["finite"]
+                       and row["vectorizer_mode"] == "channel_gated"
+                       and row["accuracy"] >= best_accuracy - 0.005]
+    pool = gated_near_best or [row for row in sdil if row["finite"]]
+    selected_sdil = sorted(
+        pool, key=lambda row: (-row["accuracy"], row["total_macs"], row["lr"]))[0]
+    passed = (bp[0]["finite"] and selected_dfa["finite"] and selected_sdil["finite"]
+              and bp[0]["accuracy"] >= 0.50
+              and selected_sdil["accuracy"] >= 0.35
+              and selected_sdil["accuracy"] >= selected_dfa["accuracy"] - 0.10)
+    output = {
+        "protocol": "oral_a_A2b_v1",
+        "status": "selected" if passed else "failed_advancement_gate",
+        "rows": rows, "selected_bp": bp[0],
+        "selected_dfa": selected_dfa, "selected_sdil": selected_sdil,
+        "confirmation_test_seeds_touched": False,
+    }
+    os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True)
+    with open(args.out, "w") as handle:
+        json.dump(output, handle, indent=2, sort_keys=True)
+        handle.write("\n")
+    print(json.dumps({key: output[key] for key in (
+        "status", "selected_bp", "selected_dfa", "selected_sdil")}, indent=2))
+
+
+if __name__ == "__main__":
+    main()
diff --git a/experiments/conv_run.py b/experiments/conv_run.py
index 9b41dd2..98c1f19 100644
--- a/experiments/conv_run.py
+++ b/experiments/conv_run.py
@@ -235,8 +235,14 @@ def run(args):
         "epochs": [],
     }
 
+    sync(args.device)
+    total_start = time.time()
+    predictor_warmup_wall = 0.0
+    apical_warmup_wall = 0.0
     loader_state = train.g.get_state().clone()
     if config is not None and config.learn_P and args.predictor_warmup_steps:
+        sync(args.device)
+        warmup_start = time.time()
         iterator = iter(train)
         for _ in range(args.predictor_warmup_steps):
             try:
@@ -249,8 +255,12 @@ def run(args):
                 forward["hiddens"], config.eta_P, config.nuisance_scale)
             counters["predictor_warmup_examples"] += x.shape[0]
         train.g.set_state(loader_state)
+        sync(args.device)
+        predictor_warmup_wall = time.time() - warmup_start
 
     if config is not None and args.a_warmup_steps:
+        sync(args.device)
+        warmup_start = time.time()
         iterator = iter(train)
         warmup_metrics = []
         for _ in range(args.a_warmup_steps):
@@ -277,14 +287,14 @@ def run(args):
             "steps": args.a_warmup_steps,
             "last": warmup_metrics[-1],
         }
+        sync(args.device)
+        apical_warmup_wall = time.time() - warmup_start
 
     step = 0
     train_wall = 0.0
     eval_wall = 0.0
     validation_evaluations = 0
     test_evaluations = 0
-    sync(args.device)
-    total_start = time.time()
     for epoch in range(args.epochs):
         lr = scheduled_lr(args.lr, epoch, args)
         output_lr = (scheduled_lr(args.output_lr, epoch, args)
@@ -393,6 +403,8 @@ def run(args):
         "work": work_report(net, args.mode, counters),
         "hardware": training_hardware,
         "timing": {
+            "predictor_warmup_wall_s": predictor_warmup_wall,
+            "apical_warmup_wall_s": apical_warmup_wall,
             "train_wall_s": train_wall,
             "evaluation_wall_s": eval_wall,
             "total_timed_wall_s": total_wall,
diff --git a/experiments/oral_a_apical_screen.sh b/experiments/oral_a_apical_screen.sh
new file mode 100755
index 0000000..7281def
--- /dev/null
+++ b/experiments/oral_a_apical_screen.sh
@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+device="${1:-cuda}"
+mkdir -p results/oral_a_apical_screen
+
+for vectorizer in spatial_template channel_gated; do
+  for a_scale in 0 1; do
+    for eta_a in 0.001 0.01 0.1; do
+      tag="${vectorizer}_a${a_scale}_etaA${eta_a}"
+      python3 experiments/conv_run.py \
+        --mode sdil --device "${device}" --depth 20 --width 16 \
+        --seed 0 --loader_seed 0 --batch_size 128 --epochs 0 \
+        --train_limit 10000 --val_examples 5000 --split_seed 2027 \
+        --eval_split validation --eval_every 0 --augment_train 1 \
+        --lr 0.03 --output_lr 0.1 --lr_schedule constant --warmup_epochs 0 \
+        --momentum 0.9 --weight_decay 1e-4 --normalization batchnorm \
+        --vectorizer_mode "${vectorizer}" --a_scale "${a_scale}" \
+        --eta_A "${eta_a}" --a_warmup_steps 100 \
+        --pert_sigma 0.01 --pert_directions 1 --pert_every 4 \
+        --alignment_probe 32 \
+        --out "results/oral_a_apical_screen/${tag}.json"
+    done
+  done
+done
diff --git a/experiments/oral_a_bp_reference.sh b/experiments/oral_a_bp_reference.sh
new file mode 100755
index 0000000..cf5669e
--- /dev/null
+++ b/experiments/oral_a_bp_reference.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+variant="${1:-primary}"
+device="${2:-cuda}"
+mkdir -p results/oral_a_dev
+
+common=(
+  --mode bp --device "${device}" --depth 20 --width 16 --seed 0 --loader_seed 0
+  --batch_size 128 --epochs 200 --val_examples 5000 --split_seed 2027
+  --eval_split validation --eval_every 20 --augment_train 1
+  --lr 0.1 --momentum 0.9 --normalization batchnorm
+)
+
+case "${variant}" in
+  primary)
+    python3 experiments/conv_run.py "${common[@]}" \
+      --lr_schedule step --lr_milestones 100,150 --lr_gamma 0.1 \
+      --warmup_epochs 0 --weight_decay 1e-4 \
+      --out results/oral_a_dev/bp_reference_primary.json
+    ;;
+  recovery)
+    python3 experiments/conv_run.py "${common[@]}" \
+      --lr_schedule cosine --warmup_epochs 5 --weight_decay 5e-4 \
+      --out results/oral_a_dev/bp_reference_recovery.json
+    ;;
+  *)
+    echo "variant must be primary or recovery" >&2
+    exit 2
+    ;;
+esac
diff --git a/experiments/oral_a_short_screen.py b/experiments/oral_a_short_screen.py
new file mode 100755
index 0000000..51df778
--- /dev/null
+++ b/experiments/oral_a_short_screen.py
@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+"""Run a deterministic shard of the frozen A2b short accuracy grid."""
+import argparse
+import json
+import os
+import subprocess
+import sys
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--selection", default="results/oral_a_apical_selection.json")
+    parser.add_argument("--device", default="cuda")
+    parser.add_argument("--shard_index", type=int, default=0)
+    parser.add_argument("--num_shards", type=int, default=1)
+    parser.add_argument("--dry_run", action="store_true")
+    args = parser.parse_args()
+    if not 0 <= args.shard_index < args.num_shards:
+        raise ValueError("invalid shard index")
+    with open(args.selection) as handle:
+        selection = json.load(handle)
+    if selection["status"] != "selected":
+        raise ValueError("A2a did not select both vectorizer families")
+
+    common = [
+        sys.executable, "experiments/conv_run.py", "--device", args.device,
+        "--depth", "20", "--width", "16", "--seed", "0", "--loader_seed", "0",
+        "--batch_size", "128", "--epochs", "20", "--train_limit", "10000",
+        "--val_examples", "5000", "--split_seed", "2027",
+        "--eval_split", "validation", "--eval_every", "0", "--augment_train", "1",
+        "--lr_schedule", "cosine", "--warmup_epochs", "0", "--momentum", "0.9",
+        "--weight_decay", "1e-4", "--normalization", "batchnorm",
+    ]
+    jobs = []
+    jobs.append(("bp_lr0.1", common + [
+        "--mode", "bp", "--lr", "0.1",
+        "--out", "results/oral_a_short/bp_lr0.1.json"]))
+    for rate in (0.01, 0.03, 0.1):
+        jobs.append((f"dfa_lr{rate}", common + [
+            "--mode", "dfa", "--lr", str(rate), "--output_lr", "0.1",
+            "--a_scale", "1", "--vectorizer_mode", "spatial_template",
+            "--out", f"results/oral_a_short/dfa_lr{rate}.json"]))
+    for mode in ("spatial_template", "channel_gated"):
+        chosen = selection["selected"][mode]
+        for rate in (0.01, 0.03, 0.1):
+            tag = f"sdil_{mode}_lr{rate}"
+            jobs.append((tag, common + [
+                "--mode", "sdil", "--lr", str(rate), "--output_lr", "0.1",
+                "--vectorizer_mode", mode,
+                "--a_scale", str(chosen["a_scale"]),
+                "--eta_A", str(chosen["eta_A"]), "--a_warmup_steps", "100",
+                "--pert_sigma", "0.01", "--pert_directions", "1",
+                "--pert_every", "4", "--alignment_probe", "32",
+                "--out", f"results/oral_a_short/{tag}.json"]))
+    os.makedirs("results/oral_a_short", exist_ok=True)
+    selected_jobs = [job for index, job in enumerate(jobs)
+                     if index % args.num_shards == args.shard_index]
+    for tag, command in selected_jobs:
+        print(tag, " ".join(command), flush=True)
+        if not args.dry_run:
+            subprocess.run(command, check=True)
+
+
+if __name__ == "__main__":
+    main()