diff options
Diffstat (limited to 'experiments')
| -rw-r--r-- | experiments/analyze_oral_a_v2_calibration.py | 137 | ||||
| -rw-r--r-- | experiments/analyze_oral_a_v2_full.py | 116 | ||||
| -rw-r--r-- | experiments/conv_run.py | 12 | ||||
| -rw-r--r-- | experiments/oral_a_v2_calibration_screen.py | 56 | ||||
| -rw-r--r-- | experiments/oral_a_v2_full_development.py | 49 |
5 files changed, 370 insertions, 0 deletions
diff --git a/experiments/analyze_oral_a_v2_calibration.py b/experiments/analyze_oral_a_v2_calibration.py new file mode 100644 index 0000000..af2bd39 --- /dev/null +++ b/experiments/analyze_oral_a_v2_calibration.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +"""Apply the frozen Oral-A-v2 causal-capture gate.""" +import argparse +import glob +import json +import math +import os + + +MODES = ("unit_targets", "channel_subspace") +RATES = (0.01, 0.1, 1.0) +SPLIT_HASH = "8328b206a97c420e49e54e3eca4abe3274c4756b084355784ea3fb8059e4515b" + + +def load(path): + with open(path) as handle: + record = json.load(handle) + args = record["args"] + expected = { + "mode": "sdil", "depth": 20, "width": 16, "seed": 0, + "epochs": 0, "train_limit": 10000, "val_examples": 5000, + "a_warmup_steps": 400, "pert_directions": 1, "pert_every": 4, + "pert_sigma": 0.01, "perturb_seed": 1000, + "normalization": "batchnorm", "vectorizer_mode": "channel_gated", + "a_scale": 0.0, "alignment_probe": 64, + } + for key, value in expected.items(): + if args.get(key) != value: + raise ValueError( + f"{path}: {key}={args.get(key)!r}, expected {value!r}") + if record["provenance"]["git_tracked_dirty"]: + raise ValueError(f"tracked-dirty result: {path}") + if record["split"]["validation_index_sha256"] != SPLIT_HASH: + raise ValueError(f"split drift: {path}") + mode = args["apical_calibration_mode"] + expected_space = ("channel_basis_moments" if mode == "channel_subspace" + else "full_hidden_field") + if record.get("calibration_metric_space") != expected_space: + raise ValueError(f"calibration metric-space drift: {path}") + diagnostics = record.get("diagnostics") + warmup = record.get("apical_warmup", {}).get("mean") + if diagnostics is None or warmup is None: + raise ValueError(f"missing diagnostics/warmup aggregate: {path}") + values = diagnostics["teaching_negative_gradient_cosine"] + early_count = max(1, len(values) // 3) + metrics = { + "early_third_alignment": sum(values[:early_count]) / early_count, + "all_layer_alignment": sum(values) / len(values), + "mean_calibration_mse": warmup["calibration_mse"], + "mean_target_power": warmup["target_power"], + "mean_prediction_target_cosine": warmup["prediction_target_cosine"], + "mean_parameter_update_rms": warmup.get("parameter_update_rms", 0.0), + } + finite = (record["final"]["finite"] + and all(math.isfinite(value) for value in metrics.values())) + return { + "path": path, + "source_commit": record["provenance"]["git_commit"], + "calibration_mode": mode, + "eta_A": float(args["eta_A"]), + "metric_space": expected_space, + "metrics": metrics, + "finite": finite, + } + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--input", default="results/oral_a_v2_calibration") + parser.add_argument("--out", default="results/oral_a_v2_calibration_gate.json") + args = parser.parse_args() + rows = [load(path) for path in sorted(glob.glob( + os.path.join(args.input, "*.json")))] + observed = {(row["calibration_mode"], row["eta_A"]) for row in rows} + expected = {(mode, rate) for mode in MODES for rate in RATES} + if observed != expected or len(rows) != len(expected): + raise ValueError( + f"incomplete v2 grid: missing={expected-observed}, extra={observed-expected}") + if len({row["source_commit"] for row in rows}) != 1: + raise ValueError("v2 calibration source commits differ") + + selected = {} + for mode in MODES: + candidates = [row for row in rows + if row["calibration_mode"] == mode and row["finite"]] + if candidates: + candidates.sort(key=lambda row: ( + -row["metrics"]["early_third_alignment"], + -row["metrics"]["all_layer_alignment"], row["eta_A"])) + selected[mode] = candidates[0] + checks = { + "all_six_records_finite": all(row["finite"] for row in rows), + "both_modes_selected": len(selected) == len(MODES), + } + if checks["both_modes_selected"]: + structured = selected["channel_subspace"]["metrics"] + unit = selected["unit_targets"]["metrics"] + checks.update({ + "structured_early_third_at_least_0.01": ( + structured["early_third_alignment"] >= 0.01), + "structured_all_layer_at_least_0.01": ( + structured["all_layer_alignment"] >= 0.01), + "structured_early_gain_over_unit_at_least_0.01": ( + structured["early_third_alignment"] + - unit["early_third_alignment"] >= 0.01), + }) + else: + checks.update({ + "structured_early_third_at_least_0.01": False, + "structured_all_layer_at_least_0.01": False, + "structured_early_gain_over_unit_at_least_0.01": False, + }) + passed = all(checks.values()) + output = { + "protocol": "oral_a_v2_causal_capture_v1", + "status": "passed" if passed else "failed", + "checks": checks, + "rows": rows, + "selected": selected, + "confirmation_test_seeds_touched": False, + "review_score_before": 5, + "review_score_after": 5, + "score_change_rule": "mechanics/calibration alone cannot raise score", + } + os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) + with open(args.out, "w") as handle: + json.dump(output, handle, indent=2, sort_keys=True) + handle.write("\n") + print(json.dumps({ + "status": output["status"], "checks": checks, + "selected": selected, + }, indent=2)) + + +if __name__ == "__main__": + main() + diff --git a/experiments/analyze_oral_a_v2_full.py b/experiments/analyze_oral_a_v2_full.py new file mode 100644 index 0000000..16f7e78 --- /dev/null +++ b/experiments/analyze_oral_a_v2_full.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +"""Apply the frozen Oral-A-v2 full-validation gate.""" +import argparse +import json +import math +import os + + +SPLIT_HASH = "8328b206a97c420e49e54e3eca4abe3274c4756b084355784ea3fb8059e4515b" + + +def finite_tree(value): + if isinstance(value, dict): + return all(finite_tree(item) for item in value.values()) + if isinstance(value, list): + return all(finite_tree(item) for item in value) + if isinstance(value, float): + return math.isfinite(value) + return True + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--run", default="results/oral_a_v2_dev/sdil_full_r20_s0.json") + parser.add_argument( + "--selection", default="results/oral_a_v2_calibration_gate.json") + parser.add_argument( + "--bp", default="results/oral_a_dev/bp_reference_primary.json") + parser.add_argument( + "--dfa", default="results/oral_a_dev/dfa_full_r20_s0.json") + parser.add_argument("--out", default="results/oral_a_v2_full_gate.json") + args = parser.parse_args() + with open(args.run) as handle: + run = json.load(handle) + with open(args.selection) as handle: + selection = json.load(handle) + with open(args.bp) as handle: + bp = json.load(handle) + with open(args.dfa) as handle: + dfa = json.load(handle) + if selection["status"] != "passed": + raise ValueError("v2 causal-capture gate did not pass") + chosen = selection["selected"]["channel_subspace"] + expected = { + "mode": "sdil", "depth": 20, "width": 16, "seed": 0, + "epochs": 200, "val_examples": 5000, "lr": 0.03, + "output_lr": 0.1, "lr_schedule": "step", + "lr_milestones": "100,150", "lr_gamma": 0.1, + "a_scale": 0.0, "eta_A": chosen["eta_A"], + "a_warmup_steps": 400, "apical_calibration_mode": "channel_subspace", + "pert_sigma": 0.01, "pert_directions": 1, "pert_every": 4, + "normalization": "batchnorm", "vectorizer_mode": "channel_gated", + } + for key, value in expected.items(): + if run["args"].get(key) != value: + raise ValueError( + f"v2 run {key}={run['args'].get(key)!r}, expected {value!r}") + if run["provenance"]["git_tracked_dirty"]: + raise ValueError("tracked-dirty v2 result") + if run["split"]["validation_index_sha256"] != SPLIT_HASH: + raise ValueError("v2 split drift") + if run["evaluation_protocol"]["test_evaluations"] != 0: + raise ValueError("test endpoint touched during v2 development") + values = run["diagnostics"]["teaching_negative_gradient_cosine"] + early_count = max(1, len(values) // 3) + early = sum(values[:early_count]) / early_count + sdil_accuracy = run["final"]["accuracy"] + bp_accuracy = bp["final"]["accuracy"] + dfa_accuracy = dfa["final"]["accuracy"] + checks = { + "all_metrics_finite": run["final"]["finite"] and finite_tree(run), + "bp_reference_at_least_90pct": bp_accuracy >= 0.90, + "sdil_within_5pt_of_bp": sdil_accuracy >= bp_accuracy - 0.05, + "sdil_at_least_2pt_above_dfa": sdil_accuracy >= dfa_accuracy + 0.02, + "early_third_alignment_at_least_0.05": early >= 0.05, + "sdil_macs_no_more_than_bp": ( + run["work"]["total_macs_estimate"] + <= bp["work"]["total_macs_estimate"]), + } + passed = all(checks.values()) + output = { + "protocol": "oral_a_v2_full_validation_v1", + "status": "passed" if passed else "failed", + "checks": checks, + "metrics": { + "sdil_validation_accuracy": sdil_accuracy, + "bp_validation_accuracy": bp_accuracy, + "dfa_validation_accuracy": dfa_accuracy, + "early_third_alignment": early, + "sdil_total_macs": run["work"]["total_macs_estimate"], + "bp_total_macs": bp["work"]["total_macs_estimate"], + }, + "sources": {"sdil": args.run, "bp": args.bp, "dfa": args.dfa}, + "confirmation_test_seeds_touched": False, + "review_score_before": 5, + "review_score_after": 6 if passed else 5, + "review_score_rationale": ( + "full standard-scale validation gate passed; independent depth " + "confirmation still required" + if passed else + "full standard-scale validation gate failed; controlled evidence " + "remains unchanged"), + } + os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) + with open(args.out, "w") as handle: + json.dump(output, handle, indent=2, sort_keys=True) + handle.write("\n") + print(json.dumps({ + "status": output["status"], "checks": checks, + "metrics": output["metrics"], + }, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/experiments/conv_run.py b/experiments/conv_run.py index cb99545..a33005d 100644 --- a/experiments/conv_run.py +++ b/experiments/conv_run.py @@ -212,6 +212,11 @@ def run(args): log = { "schema_version": 1, "protocol_family": "oral_a_cifar_local_resnet_development", + "calibration_metric_space": ( + None if config is None else + ("channel_basis_moments" + if config.apical_calibration_mode == "channel_subspace" + else "full_hidden_field")), "args": vars(args), "provenance": provenance(), "split": split, @@ -284,8 +289,15 @@ def run(args): counters["per_example_loss_terms"] += 2 * config.pert_directions * batch counters["perturbation_events"] += 1 train.g.set_state(loader_state) + warmup_mean = { + key: sum(metric[key] for metric in warmup_metrics) + / len(warmup_metrics) + for key in warmup_metrics[0] + } log["apical_warmup"] = { "steps": args.a_warmup_steps, + "first": warmup_metrics[0], + "mean": warmup_mean, "last": warmup_metrics[-1], } sync(args.device) diff --git a/experiments/oral_a_v2_calibration_screen.py b/experiments/oral_a_v2_calibration_screen.py new file mode 100644 index 0000000..7666dd4 --- /dev/null +++ b/experiments/oral_a_v2_calibration_screen.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +"""Run a deterministic shard of the frozen Oral-A-v2 calibration screen.""" +import argparse +import os +import subprocess +import sys + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--device", default="cuda") + parser.add_argument("--shard_index", type=int, default=0) + parser.add_argument("--num_shards", type=int, default=1) + parser.add_argument("--dry_run", action="store_true") + args = parser.parse_args() + if not 0 <= args.shard_index < args.num_shards: + raise ValueError("invalid shard index") + + common = [ + sys.executable, "experiments/conv_run.py", + "--mode", "sdil", "--device", args.device, + "--depth", "20", "--width", "16", "--seed", "0", + "--loader_seed", "0", "--batch_size", "128", "--epochs", "0", + "--train_limit", "10000", "--val_examples", "5000", + "--split_seed", "2027", "--eval_split", "validation", + "--eval_every", "0", "--augment_train", "1", + "--lr", "0.03", "--output_lr", "0.1", + "--lr_schedule", "constant", "--warmup_epochs", "0", + "--momentum", "0.9", "--weight_decay", "1e-4", + "--normalization", "batchnorm", "--vectorizer_mode", "channel_gated", + "--a_scale", "0", "--a_warmup_steps", "400", + "--pert_sigma", "0.01", "--pert_directions", "1", + "--pert_every", "4", "--perturb_seed", "1000", + "--alignment_probe", "64", + ] + jobs = [] + for calibration in ("unit_targets", "channel_subspace"): + for rate in (0.01, 0.1, 1.0): + tag = f"{calibration}_etaA{rate}" + jobs.append((tag, common + [ + "--apical_calibration_mode", calibration, + "--eta_A", str(rate), + "--out", f"results/oral_a_v2_calibration/{tag}.json", + ])) + os.makedirs("results/oral_a_v2_calibration", exist_ok=True) + for index, (tag, command) in enumerate(jobs): + if index % args.num_shards != args.shard_index: + continue + print(tag, " ".join(command), flush=True) + if not args.dry_run: + subprocess.run(command, check=True) + + +if __name__ == "__main__": + main() + diff --git a/experiments/oral_a_v2_full_development.py b/experiments/oral_a_v2_full_development.py new file mode 100644 index 0000000..457f7b2 --- /dev/null +++ b/experiments/oral_a_v2_full_development.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +"""Run the single frozen Oral-A-v2 full ResNet-20 validation job.""" +import argparse +import json +import os +import subprocess +import sys + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--selection", default="results/oral_a_v2_calibration_gate.json") + parser.add_argument("--device", default="cuda") + parser.add_argument("--dry_run", action="store_true") + args = parser.parse_args() + with open(args.selection) as handle: + selection = json.load(handle) + if selection["status"] != "passed": + raise ValueError("Oral-A-v2 causal-capture gate did not pass") + chosen = selection["selected"]["channel_subspace"] + command = [ + sys.executable, "experiments/conv_run.py", + "--mode", "sdil", "--device", args.device, + "--depth", "20", "--width", "16", "--seed", "0", + "--loader_seed", "0", "--batch_size", "128", "--epochs", "200", + "--val_examples", "5000", "--split_seed", "2027", + "--eval_split", "validation", "--eval_every", "20", + "--augment_train", "1", "--lr", "0.03", "--output_lr", "0.1", + "--lr_schedule", "step", "--lr_milestones", "100,150", + "--lr_gamma", "0.1", "--warmup_epochs", "0", "--momentum", "0.9", + "--weight_decay", "1e-4", "--normalization", "batchnorm", + "--vectorizer_mode", "channel_gated", "--a_scale", "0", + "--eta_A", str(chosen["eta_A"]), "--a_warmup_steps", "400", + "--apical_calibration_mode", "channel_subspace", + "--pert_sigma", "0.01", "--pert_directions", "1", + "--pert_every", "4", "--perturb_seed", "1000", + "--alignment_probe", "32", + "--out", "results/oral_a_v2_dev/sdil_full_r20_s0.json", + ] + os.makedirs("results/oral_a_v2_dev", exist_ok=True) + print(" ".join(command), flush=True) + if not args.dry_run: + subprocess.run(command, check=True) + + +if __name__ == "__main__": + main() + |