From 94fc8471c515fb0baacc0aa79d4a47f539858e57 Mon Sep 17 00:00:00 2001 From: YurenHao0426 Date: Wed, 22 Jul 2026 06:25:06 -0500 Subject: experiments: freeze oral A confirmation gates --- ORAL_A.md | 4 +- experiments/analyze_oral_a_confirmation.py | 146 +++++++++++++++++++++++++++++ experiments/analyze_oral_a_full.py | 83 ++++++++++++++++ experiments/oral_a_confirmation.py | 84 +++++++++++++++++ experiments/oral_a_full_development.py | 64 +++++++++++++ 5 files changed, 380 insertions(+), 1 deletion(-) create mode 100755 experiments/analyze_oral_a_confirmation.py create mode 100755 experiments/analyze_oral_a_full.py create mode 100755 experiments/oral_a_confirmation.py create mode 100755 experiments/oral_a_full_development.py diff --git a/ORAL_A.md b/ORAL_A.md index e3427c4..1a07708 100644 --- a/ORAL_A.md +++ b/ORAL_A.md @@ -118,7 +118,9 @@ No recovery branch follows an A3 failure. Only after A3 passes, run exact BP, selected DFA, and selected SDIL at depths 20, 32, and 56 for model/data-loader seeds 10--14. Hyperparameters and epoch -budgets are copied without depth-specific tuning. The CIFAR-10 test set is +budgets are copied without depth-specific tuning. Confirmation refits on all +50,000 original training examples (`val_examples=0`) after development has +closed. The CIFAR-10 test set is evaluated exactly once at the end of every run; no intermediate test metric is logged or used for selection. diff --git a/experiments/analyze_oral_a_confirmation.py b/experiments/analyze_oral_a_confirmation.py new file mode 100755 index 0000000..5de05cb --- /dev/null +++ b/experiments/analyze_oral_a_confirmation.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +"""Audit all 45 A4 trajectories and apply the frozen Oral-A gate.""" +import argparse +import glob +import json +import math +import os +import statistics + + +METHODS = ("bp", "dfa", "sdil") +DEPTHS = (20, 32, 56) +SEEDS = tuple(range(10, 15)) + + +def read(path): + with open(path) as handle: + record = json.load(handle) + args = record["args"] + key = (args["mode"], int(args["depth"]), int(args["seed"])) + if key[0] not in METHODS or key[1] not in DEPTHS or key[2] not in SEEDS: + raise ValueError(f"unexpected A4 key {key}: {path}") + expected = { + "width": 16, "epochs": 200, "val_examples": 0, + "eval_split": "test", "eval_every": 0, "normalization": "batchnorm", + } + for name, value in expected.items(): + if args.get(name) != value: + raise ValueError(f"{path}: {name} drift") + if record["provenance"]["git_tracked_dirty"]: + raise ValueError(f"tracked-dirty result: {path}") + protocol = record["evaluation_protocol"] + if protocol["test_evaluations"] != 1 or protocol["test_used_for_selection"]: + raise ValueError(f"invalid test protocol: {path}") + accuracy = float(record["final"]["accuracy"]) + loss = float(record["final"]["loss"]) + alignment = (None if record["diagnostics"] is None + else float(record["diagnostics"]["early_third_mean"])) + return key, { + "path": path, "accuracy": accuracy, "loss": loss, + "finite": record["final"]["finite"] and math.isfinite(accuracy + loss), + "early_third_alignment": alignment, + "total_macs": record["work"]["total_macs_estimate"], + "logical_queries": record["work"]["logical_batch_loss_queries"], + "peak_memory": record["hardware"]["peak_memory_allocated_bytes"], + "source_commit": record["provenance"]["git_commit"], + } + + +def mean_sd(values): + return {"mean": statistics.mean(values), "sample_sd": statistics.stdev(values)} + + +def frontier_members(points, cost): + members = [] + for candidate in points: + dominated = any( + other["accuracy"] >= candidate["accuracy"] + and other[cost] <= candidate[cost] + and (other["accuracy"] > candidate["accuracy"] + or other[cost] < candidate[cost]) + for other in points if other is not candidate) + if not dominated: + members.append(candidate["name"]) + return members + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--input", default="results/oral_a_confirm") + parser.add_argument("--out", default="results/oral_a_confirmation.json") + args = parser.parse_args() + rows = dict(read(path) for path in sorted(glob.glob(os.path.join(args.input, "*.json")))) + expected = {(method, depth, seed) for method in METHODS + for depth in DEPTHS for seed in SEEDS} + if set(rows) != expected or len(rows) != 45: + raise ValueError(f"incomplete A4 grid: missing={expected-set(rows)}, extra={set(rows)-expected}") + summaries = {} + for method in METHODS: + for depth in DEPTHS: + group = [rows[(method, depth, seed)] for seed in SEEDS] + summaries[f"{method}_d{depth}"] = { + "accuracy": mean_sd([row["accuracy"] for row in group]), + "total_macs": mean_sd([row["total_macs"] for row in group]), + "peak_memory": mean_sd([row["peak_memory"] for row in group]), + "logical_queries": mean_sd([row["logical_queries"] for row in group]), + "early_third_alignment": (None if method == "bp" else mean_sd( + [row["early_third_alignment"] for row in group])), + } + near_bp = all( + summaries[f"sdil_d{depth}"]["accuracy"]["mean"] + >= summaries[f"bp_d{depth}"]["accuracy"]["mean"] - 0.02 + for depth in DEPTHS) + d56_advantage = ( + summaries["sdil_d56"]["accuracy"]["mean"] + >= summaries["dfa_d56"]["accuracy"]["mean"] + 0.02) + paired_depth_changes = [ + rows[("sdil", 56, seed)]["accuracy"] - rows[("sdil", 20, seed)]["accuracy"] + for seed in SEEDS] + depth_preserved = statistics.mean(paired_depth_changes) >= -0.02 + align20 = summaries["sdil_d20"]["early_third_alignment"]["mean"] + align56 = summaries["sdil_d56"]["early_third_alignment"]["mean"] + alignment_gate = align56 >= 0.05 and align56 >= 0.30 * align20 + local_points = [] + for method in ("dfa", "sdil"): + for depth in DEPTHS: + summary = summaries[f"{method}_d{depth}"] + local_points.append({ + "name": f"{method}_d{depth}", + "accuracy": summary["accuracy"]["mean"], + "total_macs": summary["total_macs"]["mean"], + "logical_queries": summary["logical_queries"]["mean"], + "peak_memory": summary["peak_memory"]["mean"], + }) + frontiers = {cost: frontier_members(local_points, cost) for cost in ( + "total_macs", "logical_queries", "peak_memory")} + nondominated = all( + any(name.startswith("sdil_") for name in members) + for members in frontiers.values()) + checks = { + "all_finite": all(row["finite"] for row in rows.values()), + "within_2pt_bp_each_depth": near_bp, + "d56_beats_dfa_2pt": d56_advantage, + "paired_d20_to_d56_no_worse_than_minus_2pt": depth_preserved, + "d56_alignment_level_and_retention": alignment_gate, + "sdil_on_each_local_cost_frontier": nondominated, + } + output = { + "protocol": "oral_a_A4_v1", + "status": "passed" if all(checks.values()) else "failed", + "checks": checks, "summaries": summaries, + "local_frontiers": frontiers, + "paired_sdil_d20_to_d56_accuracy_change": mean_sd(paired_depth_changes), + "rows": {f"{method}_d{depth}_s{seed}": row + for (method, depth, seed), row in rows.items()}, + } + os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) + with open(args.out, "w") as handle: + json.dump(output, handle, indent=2, sort_keys=True) + handle.write("\n") + print(json.dumps({"status": output["status"], "checks": checks, + "summaries": summaries}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/experiments/analyze_oral_a_full.py b/experiments/analyze_oral_a_full.py new file mode 100755 index 0000000..f22d55c --- /dev/null +++ b/experiments/analyze_oral_a_full.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""Apply the frozen A3 full seed-0 ResNet-20 advancement gate.""" +import argparse +import json +import math +import os + + +SPLIT_HASH = "8328b206a97c420e49e54e3eca4abe3274c4756b084355784ea3fb8059e4515b" + + +def read_result(path, mode): + with open(path) as handle: + record = json.load(handle) + args = record["args"] + expected = { + "mode": mode, "depth": 20, "width": 16, "seed": 0, + "epochs": 200, "val_examples": 5000, + "eval_split": "validation", "normalization": "batchnorm", + } + for key, value in expected.items(): + if args.get(key) != value: + raise ValueError(f"{path}: {key} drift") + if record["provenance"]["git_tracked_dirty"]: + raise ValueError(f"tracked-dirty result: {path}") + if record["split"]["validation_index_sha256"] != SPLIT_HASH: + raise ValueError(f"split drift: {path}") + if record["evaluation_protocol"]["test_evaluations"]: + raise ValueError(f"A3 touched test: {path}") + alignment = None + if record["diagnostics"] is not None: + alignment = float(record["diagnostics"]["early_third_mean"]) + accuracy = float(record["final"]["accuracy"]) + loss = float(record["final"]["loss"]) + return { + "mode": mode, "path": path, "accuracy": accuracy, "loss": loss, + "finite": record["final"]["finite"] and math.isfinite(accuracy + loss), + "early_third_alignment": alignment, + "total_macs": record["work"]["total_macs_estimate"], + "peak_memory": record["hardware"]["peak_memory_allocated_bytes"], + "source_commit": record["provenance"]["git_commit"], + } + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--bp_selection", default="results/oral_a_bp_selection.json") + parser.add_argument("--dfa", default="results/oral_a_dev/dfa_full_r20_s0.json") + parser.add_argument("--sdil", default="results/oral_a_dev/sdil_full_r20_s0.json") + parser.add_argument("--out", default="results/oral_a_full_gate.json") + args = parser.parse_args() + with open(args.bp_selection) as handle: + bp_selection = json.load(handle) + if not bp_selection["status"].startswith("passed_"): + raise ValueError("A1 BP reference is not viable") + bp = read_result(bp_selection["selected"]["path"], "bp") + dfa = read_result(args.dfa, "dfa") + sdil = read_result(args.sdil, "sdil") + checks = { + "bp_at_least_90": bp["accuracy"] >= 0.90, + "sdil_within_5pt_bp": sdil["accuracy"] >= bp["accuracy"] - 0.05, + "sdil_beats_dfa_2pt": sdil["accuracy"] >= dfa["accuracy"] + 0.02, + "sdil_alignment_at_least_0p05": ( + sdil["early_third_alignment"] is not None + and sdil["early_third_alignment"] >= 0.05), + "all_finite": all(row["finite"] for row in (bp, dfa, sdil)), + "sdil_macs_no_more_than_bp": sdil["total_macs"] <= bp["total_macs"], + } + output = { + "protocol": "oral_a_A3_v1", + "status": "passed" if all(checks.values()) else "failed", + "checks": checks, "rows": [bp, dfa, sdil], + "confirmation_test_seeds_touched": False, + } + os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) + with open(args.out, "w") as handle: + json.dump(output, handle, indent=2, sort_keys=True) + handle.write("\n") + print(json.dumps(output, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/experiments/oral_a_confirmation.py b/experiments/oral_a_confirmation.py new file mode 100755 index 0000000..e8b0dba --- /dev/null +++ b/experiments/oral_a_confirmation.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""Run a deterministic shard of the frozen A4 independent test panel.""" +import argparse +import json +import os +import subprocess +import sys + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--gate", default="results/oral_a_full_gate.json") + parser.add_argument("--bp_selection", default="results/oral_a_bp_selection.json") + parser.add_argument("--short_selection", default="results/oral_a_short_selection.json") + parser.add_argument("--device", default="cuda") + parser.add_argument("--shard_index", type=int, default=0) + parser.add_argument("--num_shards", type=int, default=1) + parser.add_argument("--dry_run", action="store_true") + args = parser.parse_args() + if not 0 <= args.shard_index < args.num_shards: + raise ValueError("invalid shard index") + with open(args.gate) as handle: + gate = json.load(handle) + with open(args.bp_selection) as handle: + bp_selection = json.load(handle) + with open(args.short_selection) as handle: + short = json.load(handle) + if gate["status"] != "passed": + raise ValueError("A3 did not pass; A4 test confirmation is prohibited") + if short["status"] != "selected" or not bp_selection["status"].startswith("passed_"): + raise ValueError("development selections are incomplete") + dfa = short["selected_dfa"] + sdil = short["selected_sdil"] + bp_variant = bp_selection["selected"]["variant"] + jobs = [] + for depth in (20, 32, 56): + for seed in range(10, 15): + common = [ + sys.executable, "experiments/conv_run.py", "--device", args.device, + "--depth", str(depth), "--width", "16", "--seed", str(seed), + "--loader_seed", str(seed), "--perturb_seed", str(1000 + seed), + "--batch_size", "128", "--epochs", "200", "--val_examples", "0", + "--eval_split", "test", "--eval_every", "0", "--augment_train", "1", + "--momentum", "0.9", "--normalization", "batchnorm", + ] + if bp_variant == "primary": + bp_schedule = [ + "--lr", "0.1", "--lr_schedule", "step", + "--lr_milestones", "100,150", "--lr_gamma", "0.1", + "--warmup_epochs", "0", "--weight_decay", "1e-4"] + else: + bp_schedule = [ + "--lr", "0.1", "--lr_schedule", "cosine", + "--warmup_epochs", "5", "--weight_decay", "5e-4"] + jobs.append((f"bp_d{depth}_s{seed}", common + [ + "--mode", "bp", *bp_schedule, + "--out", f"results/oral_a_confirm/bp_d{depth}_s{seed}.json"])) + local_schedule = [ + "--output_lr", "0.1", "--lr_schedule", "step", + "--lr_milestones", "100,150", "--lr_gamma", "0.1", + "--warmup_epochs", "0", "--weight_decay", "1e-4", + "--alignment_probe", "32"] + jobs.append((f"dfa_d{depth}_s{seed}", common + local_schedule + [ + "--mode", "dfa", "--lr", str(dfa["lr"]), "--a_scale", "1", + "--vectorizer_mode", "spatial_template", + "--out", f"results/oral_a_confirm/dfa_d{depth}_s{seed}.json"])) + jobs.append((f"sdil_d{depth}_s{seed}", common + local_schedule + [ + "--mode", "sdil", "--lr", str(sdil["lr"]), + "--vectorizer_mode", sdil["vectorizer_mode"], + "--a_scale", str(sdil["a_scale"]), "--eta_A", str(sdil["eta_A"]), + "--a_warmup_steps", "100", "--pert_sigma", "0.01", + "--pert_directions", "1", "--pert_every", "4", + "--out", f"results/oral_a_confirm/sdil_d{depth}_s{seed}.json"])) + os.makedirs("results/oral_a_confirm", exist_ok=True) + for index, (tag, command) in enumerate(jobs): + if index % args.num_shards != args.shard_index: + continue + print(tag, " ".join(command), flush=True) + if not args.dry_run: + subprocess.run(command, check=True) + + +if __name__ == "__main__": + main() diff --git a/experiments/oral_a_full_development.py b/experiments/oral_a_full_development.py new file mode 100755 index 0000000..0d8b8eb --- /dev/null +++ b/experiments/oral_a_full_development.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +"""Run a shard of the frozen A3 full ResNet-20 local-method development gate.""" +import argparse +import json +import os +import subprocess +import sys + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--short_selection", default="results/oral_a_short_selection.json") + parser.add_argument("--bp_selection", default="results/oral_a_bp_selection.json") + parser.add_argument("--device", default="cuda") + parser.add_argument("--shard_index", type=int, default=0) + parser.add_argument("--num_shards", type=int, default=1) + parser.add_argument("--dry_run", action="store_true") + args = parser.parse_args() + if not 0 <= args.shard_index < args.num_shards: + raise ValueError("invalid shard index") + with open(args.short_selection) as handle: + short = json.load(handle) + with open(args.bp_selection) as handle: + bp = json.load(handle) + if short["status"] != "selected": + raise ValueError("A2b advancement gate did not pass") + if not bp["status"].startswith("passed_"): + raise ValueError("A1 BP reference gate did not pass") + dfa = short["selected_dfa"] + sdil = short["selected_sdil"] + common = [ + sys.executable, "experiments/conv_run.py", "--device", args.device, + "--depth", "20", "--width", "16", "--seed", "0", "--loader_seed", "0", + "--batch_size", "128", "--epochs", "200", "--val_examples", "5000", + "--split_seed", "2027", "--eval_split", "validation", "--eval_every", "20", + "--augment_train", "1", "--lr_schedule", "step", "--lr_milestones", "100,150", + "--lr_gamma", "0.1", "--warmup_epochs", "0", "--momentum", "0.9", + "--weight_decay", "1e-4", "--normalization", "batchnorm", + "--output_lr", "0.1", "--alignment_probe", "32", + ] + jobs = [ + ("dfa", common + [ + "--mode", "dfa", "--lr", str(dfa["lr"]), "--a_scale", "1", + "--vectorizer_mode", "spatial_template", + "--out", "results/oral_a_dev/dfa_full_r20_s0.json"]), + ("sdil", common + [ + "--mode", "sdil", "--lr", str(sdil["lr"]), + "--vectorizer_mode", sdil["vectorizer_mode"], + "--a_scale", str(sdil["a_scale"]), "--eta_A", str(sdil["eta_A"]), + "--a_warmup_steps", "100", "--pert_sigma", "0.01", + "--pert_directions", "1", "--pert_every", "4", + "--out", "results/oral_a_dev/sdil_full_r20_s0.json"]), + ] + os.makedirs("results/oral_a_dev", exist_ok=True) + for index, (tag, command) in enumerate(jobs): + if index % args.num_shards != args.shard_index: + continue + print(tag, " ".join(command), flush=True) + if not args.dry_run: + subprocess.run(command, check=True) + + +if __name__ == "__main__": + main() -- cgit v1.2.3