From 931273085034d48db5d93b71669ba9bc30b8bc4d Mon Sep 17 00:00:00 2001 From: YurenHao0426 Date: Wed, 22 Jul 2026 06:22:25 -0500 Subject: experiments: freeze oral A development gates --- experiments/analyze_oral_a_short.py | 84 +++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100755 experiments/analyze_oral_a_short.py (limited to 'experiments/analyze_oral_a_short.py') diff --git a/experiments/analyze_oral_a_short.py b/experiments/analyze_oral_a_short.py new file mode 100755 index 0000000..375718a --- /dev/null +++ b/experiments/analyze_oral_a_short.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""Apply the frozen A2b selector and advancement gate.""" +import argparse +import glob +import json +import math +import os + + +def read(path): + with open(path) as handle: + record = json.load(handle) + args = record["args"] + expected = { + "depth": 20, "width": 16, "seed": 0, "epochs": 20, + "train_limit": 10000, "val_examples": 5000, + "eval_split": "validation", "normalization": "batchnorm", + } + for key, value in expected.items(): + if args.get(key) != value: + raise ValueError(f"{path}: {key} drift") + if record["provenance"]["git_tracked_dirty"]: + raise ValueError(f"tracked-dirty result: {path}") + if record["evaluation_protocol"]["test_evaluations"]: + raise ValueError(f"short screen touched test: {path}") + accuracy = float(record["final"]["accuracy"]) + loss = float(record["final"]["loss"]) + return { + "path": path, "mode": args["mode"], "lr": float(args["lr"]), + "vectorizer_mode": args.get("vectorizer_mode"), + "a_scale": float(args.get("a_scale", 0.0)), + "eta_A": float(args.get("eta_A", 0.0)), + "accuracy": accuracy, "loss": loss, + "finite": record["final"]["finite"] and math.isfinite(accuracy + loss), + "total_macs": record["work"]["total_macs_estimate"], + "source_commit": record["provenance"]["git_commit"], + } + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--input", default="results/oral_a_short") + parser.add_argument("--out", default="results/oral_a_short_selection.json") + args = parser.parse_args() + rows = [read(path) for path in sorted(glob.glob(os.path.join(args.input, "*.json")))] + if len(rows) != 10: + raise ValueError(f"expected 10 A2b runs, found {len(rows)}") + if len({row["source_commit"] for row in rows}) != 1: + raise ValueError("A2b source commits differ") + bp = [row for row in rows if row["mode"] == "bp"] + dfa = [row for row in rows if row["mode"] == "dfa"] + sdil = [row for row in rows if row["mode"] == "sdil"] + if len(bp) != 1 or len(dfa) != 3 or len(sdil) != 6: + raise ValueError("A2b method grid is incomplete") + selected_dfa = sorted( + dfa, key=lambda row: (-row["accuracy"], row["total_macs"], row["lr"]))[0] + best_accuracy = max(row["accuracy"] for row in sdil if row["finite"]) + gated_near_best = [row for row in sdil if row["finite"] + and row["vectorizer_mode"] == "channel_gated" + and row["accuracy"] >= best_accuracy - 0.005] + pool = gated_near_best or [row for row in sdil if row["finite"]] + selected_sdil = sorted( + pool, key=lambda row: (-row["accuracy"], row["total_macs"], row["lr"]))[0] + passed = (bp[0]["finite"] and selected_dfa["finite"] and selected_sdil["finite"] + and bp[0]["accuracy"] >= 0.50 + and selected_sdil["accuracy"] >= 0.35 + and selected_sdil["accuracy"] >= selected_dfa["accuracy"] - 0.10) + output = { + "protocol": "oral_a_A2b_v1", + "status": "selected" if passed else "failed_advancement_gate", + "rows": rows, "selected_bp": bp[0], + "selected_dfa": selected_dfa, "selected_sdil": selected_sdil, + "confirmation_test_seeds_touched": False, + } + os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) + with open(args.out, "w") as handle: + json.dump(output, handle, indent=2, sort_keys=True) + handle.write("\n") + print(json.dumps({key: output[key] for key in ( + "status", "selected_bp", "selected_dfa", "selected_sdil")}, indent=2)) + + +if __name__ == "__main__": + main() -- cgit v1.2.3