From 670c1399659f31ac79c4b454147417d4f8ddca7d Mon Sep 17 00:00:00 2001 From: YurenHao0426 Date: Thu, 23 Jul 2026 07:58:09 -0500 Subject: protocol: freeze independent oral-B-v2 gates --- experiments/bci_v2_confirmation.py | 149 +++++++++++++++++++++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100644 experiments/bci_v2_confirmation.py (limited to 'experiments/bci_v2_confirmation.py') diff --git a/experiments/bci_v2_confirmation.py b/experiments/bci_v2_confirmation.py new file mode 100644 index 0000000..6968ee7 --- /dev/null +++ b/experiments/bci_v2_confirmation.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""Run one untouched oral-B-v2 confirmation cell.""" +import argparse +import json +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from experiments.bci_v2_run import ( + D4_GATE_PATH, + OLD_R2_GATE_PATH, + PROTOCOL_PATH, + finite_tree, + provenance, + require_parent_gates, + run_cell, + sha256, + source_paths, +) + + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +R1_GATE_PATH = os.path.join(ROOT, "results", "bci_v2_dev_gate.json") +TASK_SEEDS = tuple(range(30, 36)) +MODEL_SEEDS = tuple(range(5)) + + +def require_r1_gate(r1): + development_digests = { + name: sha256(path) + for name, path in source_paths().items() + } + selected = r1.get("selected") or {} + if not ( + r1.get("protocol") == "oral_b_v2_development_v1" + and r1.get("status") == "passed" + and r1.get("complete_grid") is True + and r1.get("grid_size") == 24 + and r1.get("confirmation_seeds_touched") is False + and r1.get("oral_b_v2_confirmation_opened") is True + and r1.get("review_score_after") == 7 + and r1.get("input_sha256") == development_digests + and selected.get("forward_eta") in (0.03, 0.1) + and selected.get("gamma") in (0.8, 0.95) + and selected.get("critic_eta") in (0.01, 0.03) + ): + raise ValueError( + "oral-B-v2 R2 requires the complete eligible frozen R1 gate" + ) + return selected + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--task-seed", type=int, choices=TASK_SEEDS, required=True + ) + parser.add_argument( + "--model-seed", type=int, choices=MODEL_SEEDS, required=True + ) + parser.add_argument( + "--r1-gate", default="results/bci_v2_dev_gate.json" + ) + parser.add_argument( + "--outdir", default="results/bci_v2_confirmation" + ) + args = parser.parse_args() + require_parent_gates() + with open(args.r1_gate) as handle: + r1 = json.load(handle) + selected = require_r1_gate(r1) + extra_paths = { + "development_runner": os.path.join( + ROOT, "experiments", "bci_v2_run.py" + ), + "runner": os.path.abspath(__file__), + "r1_gate": os.path.abspath(args.r1_gate), + } + source = provenance(extra_paths) + if ( + source["git_tracked_dirty"] + or not all(source["tracked_inputs"].values()) + ): + raise RuntimeError( + "oral-B-v2 R2 requires clean, tracked, frozen inputs" + ) + cell = run_cell( + args.task_seed, + args.model_seed, + selected["forward_eta"], + selected["gamma"], + selected["critic_eta"], + split="untouched_confirmation", + performance_seed_offset=500_000, + challenge_seed_offset=510_000, + ) + result = { + "schema_version": 2, + "protocol": { + "name": "oral_b_v2_confirmation_v1", + "split": "untouched_confirmation", + "training_task_seed": args.task_seed, + "model_seed": args.model_seed, + "selected": { + "forward_eta": selected["forward_eta"], + "gamma": selected["gamma"], + "critic_eta": selected["critic_eta"], + }, + "no_further_selection": True, + "confirmation_grid_size": ( + len(TASK_SEEDS) * len(MODEL_SEEDS) + ), + "protocol_sha256": sha256(PROTOCOL_PATH), + "d4_gate_sha256": sha256(D4_GATE_PATH), + "old_r2_gate_sha256": sha256(OLD_R2_GATE_PATH), + "r1_gate_sha256": sha256(args.r1_gate), + }, + "args": vars(args), + "provenance": source, + **cell, + } + result["finite"] = finite_tree(result) + if not result["finite"]: + raise RuntimeError("non-finite oral-B-v2 confirmation record") + os.makedirs(args.outdir, exist_ok=True) + path = os.path.join( + args.outdir, + ( + f"bci_v2_confirm_t{args.task_seed}" + f"_m{args.model_seed}.json" + ), + ) + if os.path.exists(path): + raise FileExistsError(f"refusing to overwrite {path}") + with open(path, "w") as handle: + json.dump(result, handle, indent=2, sort_keys=True) + handle.write("\n") + print(json.dumps({ + "path": path, + "intact_final": result["conditions"]["intact"]["final_success"], + "terminal_outcome_accuracy": result["signatures"][ + "terminal_residual_outcome_balanced_acc" + ], + "finite": result["finite"], + }, indent=2)) + + +if __name__ == "__main__": + main() -- cgit v1.2.3