From 612196a53285b6efc44c0dd84699c7b57a6dc8e1 Mon Sep 17 00:00:00 2001 From: YurenHao0426 Date: Wed, 22 Jul 2026 12:53:53 -0500 Subject: protocol: freeze Oral-A v3 causal-capture funnel --- experiments/analyze_oral_a_v3_full.py | 113 ++++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 experiments/analyze_oral_a_v3_full.py (limited to 'experiments/analyze_oral_a_v3_full.py') diff --git a/experiments/analyze_oral_a_v3_full.py b/experiments/analyze_oral_a_v3_full.py new file mode 100644 index 0000000..f24e5d9 --- /dev/null +++ b/experiments/analyze_oral_a_v3_full.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""Apply the frozen Oral-A-v3 full-validation gate.""" +import argparse +import json +import math +import os + + +SPLIT_HASH = "8328b206a97c420e49e54e3eca4abe3274c4756b084355784ea3fb8059e4515b" + + +def finite_tree(value): + if isinstance(value, dict): + return all(finite_tree(item) for item in value.values()) + if isinstance(value, list): + return all(finite_tree(item) for item in value) + if isinstance(value, float): + return math.isfinite(value) + return True + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--run", default="results/oral_a_v3_dev/sdil_full_r20_s0.json") + parser.add_argument( + "--selection", default="results/oral_a_v3_calibration_gate.json") + parser.add_argument( + "--bp", default="results/oral_a_dev/bp_reference_primary.json") + parser.add_argument( + "--dfa", default="results/oral_a_dev/dfa_full_r20_s0.json") + parser.add_argument("--out", default="results/oral_a_v3_full_gate.json") + args = parser.parse_args() + with open(args.run) as handle: + run = json.load(handle) + with open(args.selection) as handle: + selection = json.load(handle) + with open(args.bp) as handle: + bp = json.load(handle) + with open(args.dfa) as handle: + dfa = json.load(handle) + if selection["status"] != "passed": + raise ValueError("v3 causal-capture gate did not pass") + chosen = selection["selected_v3"] + expected = { + "mode": "sdil", "depth": 20, "width": 16, "seed": 0, + "epochs": 200, "val_examples": 5000, "lr": 0.03, + "output_lr": 0.1, "lr_schedule": "step", + "lr_milestones": "100,150", "lr_gamma": 0.1, + "a_scale": 0.0, "eta_A": chosen["eta_A"], + "a_warmup_steps": 400, + "apical_calibration_mode": "vectorizer_subspace", + "pert_sigma": 0.01, "pert_directions": 1, "pert_every": 4, + "normalization": "batchnorm", "vectorizer_mode": "channel_gated", + } + for key, value in expected.items(): + if run["args"].get(key) != value: + raise ValueError( + f"v3 run {key}={run['args'].get(key)!r}, expected {value!r}") + if run["provenance"]["git_tracked_dirty"]: + raise ValueError("tracked-dirty v3 result") + if run["split"]["validation_index_sha256"] != SPLIT_HASH: + raise ValueError("v3 split drift") + if run["evaluation_protocol"]["test_evaluations"] != 0: + raise ValueError("test endpoint touched during v3 development") + values = run["diagnostics"]["teaching_negative_gradient_cosine"] + early_count = max(1, len(values) // 3) + early = sum(values[:early_count]) / early_count + sdil_accuracy = run["final"]["accuracy"] + bp_accuracy = bp["final"]["accuracy"] + dfa_accuracy = dfa["final"]["accuracy"] + checks = { + "all_metrics_finite": run["final"]["finite"] and finite_tree(run), + "bp_reference_at_least_90pct": bp_accuracy >= 0.90, + "sdil_within_5pt_of_bp": sdil_accuracy >= bp_accuracy - 0.05, + "sdil_at_least_2pt_above_dfa": sdil_accuracy >= dfa_accuracy + 0.02, + "early_third_alignment_at_least_0.05": early >= 0.05, + "sdil_macs_no_more_than_bp": ( + run["work"]["total_macs_estimate"] + <= bp["work"]["total_macs_estimate"]), + } + passed = all(checks.values()) + output = { + "protocol": "oral_a_v3_full_validation_v1", + "status": "passed" if passed else "failed", "checks": checks, + "metrics": { + "sdil_validation_accuracy": sdil_accuracy, + "bp_validation_accuracy": bp_accuracy, + "dfa_validation_accuracy": dfa_accuracy, + "early_third_alignment": early, + "sdil_total_macs": run["work"]["total_macs_estimate"], + "bp_total_macs": bp["work"]["total_macs_estimate"], + }, + "sources": {"sdil": args.run, "bp": args.bp, "dfa": args.dfa}, + "confirmation_test_seeds_touched": False, + "review_score_before": 5, "review_score_after": 6 if passed else 5, + "review_score_rationale": ( + "full standard-scale validation passed; independent depth panel remains" + if passed else "full standard-scale validation failed"), + } + os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) + with open(args.out, "w") as handle: + json.dump(output, handle, indent=2, sort_keys=True) + handle.write("\n") + print(json.dumps({ + "status": output["status"], "checks": checks, + "metrics": output["metrics"], + }, indent=2)) + + +if __name__ == "__main__": + main() + -- cgit v1.2.3