diff options
| -rwxr-xr-x | experiments/finalize_accept.sh | 2 | ||||
| -rw-r--r-- | experiments/plot_oral_a_scaling.py | 78 | ||||
| -rw-r--r-- | results/figs/figure6_standard_depth_scaling_manifest.json | 8 |
3 files changed, 83 insertions, 5 deletions
diff --git a/experiments/finalize_accept.sh b/experiments/finalize_accept.sh index c50545f..dc4d36e 100755 --- a/experiments/finalize_accept.sh +++ b/experiments/finalize_accept.sh @@ -112,8 +112,6 @@ if [ -f results/oral_a_dynamic_scaling_gate.json ]; then experiments/analyze_oral_a_dynamic_scaling.py >/dev/null fi if [ -f results/oral_a_dynamic_scaling_v2_gate.json ]; then - /home/yurenh2/miniconda3/envs/ep_pascal/bin/python3 \ - experiments/analyze_oral_a_dynamic_scaling_v2.py >/dev/null /home/yurenh2/miniconda3/bin/python \ experiments/plot_oral_a_scaling.py >/dev/null jq -e ' diff --git a/experiments/plot_oral_a_scaling.py b/experiments/plot_oral_a_scaling.py index a7ce6f5..9fd620a 100644 --- a/experiments/plot_oral_a_scaling.py +++ b/experiments/plot_oral_a_scaling.py @@ -12,6 +12,7 @@ import json import math import os import statistics +import subprocess import matplotlib @@ -60,6 +61,15 @@ def require(condition, message): raise RuntimeError(message) +def git_blob_sha256(commit, relative_path): + content = subprocess.run( + ["git", "show", f"{commit}:{relative_path}"], + check=True, + capture_output=True, + ).stdout + return hashlib.sha256(content).hexdigest() + + def expected_paths(d4_dir, new_dir): paths = {} for depth in DEPTHS: @@ -101,7 +111,7 @@ def require_exact_directories(d4_dir, new_dir): require(observed_new == expected_new, "standard-depth source directory drift") -def load_panel(d4_dir, new_dir, gate_path): +def load_panel(d4_dir, new_dir, gate_path, d4_gate_path, bci_v2_gate_path): gate = load_json(gate_path) require(gate.get("protocol") == PROTOCOL, "standard-depth protocol drift") require(gate.get("status") == "passed", "standard-depth gate is not passed") @@ -131,6 +141,52 @@ def load_panel(d4_dir, new_dir, gate_path): records[key] = record rows[key] = validate_record(record, path, method, depth, seed) + d4_commits = { + rows[(method, 20, seed)]["source_commit"] + for method in ("clean_kp", "dynamic") + for seed in SEEDS + } + new_commits = { + rows[(method, depth, seed)]["source_commit"] + for method in METHODS + for depth in DEPTHS + for seed in SEEDS + if not (depth == 20 and method in ("clean_kp", "dynamic")) + } + require(len(d4_commits) == 1, "D4 source revision drift") + require(len(new_commits) == 1, "standard-depth source revision drift") + d4_commit = next(iter(d4_commits)) + new_commit = next(iter(new_commits)) + require( + d4_commit == gate.get("d4_source_commit"), + "D4 source commit disagrees with gate", + ) + require( + new_commit == gate.get("new_source_commit"), + "standard-depth source commit disagrees with gate", + ) + historical_sources = { + "protocol_sha256": "ORAL_A_RECOVERY_V2.md", + "runner_sha256": "experiments/oral_a_dynamic_scaling_v2.py", + "analyzer_sha256": "experiments/analyze_oral_a_dynamic_scaling_v2.py", + "legacy_runner_sha256": "experiments/oral_a_dynamic_scaling.py", + "legacy_analyzer_sha256": "experiments/analyze_oral_a_dynamic_scaling.py", + "conv_run_sha256": "experiments/conv_run.py", + } + for gate_key, relative_path in historical_sources.items(): + require( + git_blob_sha256(new_commit, relative_path) == gate.get(gate_key), + f"historical source binding drift: {relative_path}", + ) + require( + sha256(d4_gate_path) == gate.get("d4_gate_sha256"), + "D4 prerequisite gate hash drift", + ) + require( + sha256(bci_v2_gate_path) == gate.get("bci_v2_gate_sha256"), + "BCI-v2 prerequisite gate hash drift", + ) + metrics = gate["metrics"] for method in METHODS: for depth in DEPTHS: @@ -516,11 +572,23 @@ def main(): parser.add_argument( "--gate", default="results/oral_a_dynamic_scaling_v2_gate.json" ) + parser.add_argument( + "--d4_gate", + default="results/kp_dynamic_projection_confirmation_gate.json", + ) + parser.add_argument( + "--bci_v2_gate", + default="results/bci_v2_calibrated_confirmation_gate.json", + ) parser.add_argument("--outdir", default="results/figs") args = parser.parse_args() gate, records, rows, paths = load_panel( - args.d4_dir, args.new_dir, args.gate + args.d4_dir, + args.new_dir, + args.gate, + args.d4_gate, + args.bci_v2_gate, ) del records os.makedirs(args.outdir, exist_ok=True) @@ -633,7 +701,11 @@ def main(): } ), }, - "sources": [{"path": args.gate, "sha256": sha256(args.gate)}] + "sources": [ + {"path": args.gate, "sha256": sha256(args.gate)}, + {"path": args.d4_gate, "sha256": sha256(args.d4_gate)}, + {"path": args.bci_v2_gate, "sha256": sha256(args.bci_v2_gate)}, + ] + [ {"path": path, "sha256": sha256(path)} for path in sorted(paths.values()) diff --git a/results/figs/figure6_standard_depth_scaling_manifest.json b/results/figs/figure6_standard_depth_scaling_manifest.json index b6c990f..49de04d 100644 --- a/results/figs/figure6_standard_depth_scaling_manifest.json +++ b/results/figs/figure6_standard_depth_scaling_manifest.json @@ -32,6 +32,14 @@ "sha256": "c0a4c0530dea26c0fffc7285e7893e6e321ba248d1e13abefbafad311a89b44f" }, { + "path": "results/kp_dynamic_projection_confirmation_gate.json", + "sha256": "636c587e47287338cdca7d9558dc3bfb99a2cec615badb23337025d85b8128ef" + }, + { + "path": "results/bci_v2_calibrated_confirmation_gate.json", + "sha256": "3b161c32727a2a7c4b50ca696dafff20a023d23e8cd008a0b8dddd662516b084" + }, + { "path": "results/kp_dynamic_projection_confirmation/seed10_clean_kp.json", "sha256": "7c34a0e561446ad6e46575915fb8bb9cddd7b3c24c323bac1e64fea42758eb1f" }, |