From fc8fe99504fe86a3636721031a7dc3d41a7909a6 Mon Sep 17 00:00:00 2001 From: YurenHao0426 Date: Wed, 22 Jul 2026 12:36:26 -0500 Subject: protocol: freeze post-failure Oral-A v2 funnel --- ORAL_A_V2.md | 105 ++++++++++++++++++++ README.md | 1 + experiments/analyze_oral_a_v2_calibration.py | 137 +++++++++++++++++++++++++++ experiments/analyze_oral_a_v2_full.py | 116 +++++++++++++++++++++++ experiments/conv_run.py | 12 +++ experiments/oral_a_v2_calibration_screen.py | 56 +++++++++++ experiments/oral_a_v2_full_development.py | 49 ++++++++++ 7 files changed, 476 insertions(+) create mode 100644 ORAL_A_V2.md create mode 100644 experiments/analyze_oral_a_v2_calibration.py create mode 100644 experiments/analyze_oral_a_v2_full.py create mode 100644 experiments/oral_a_v2_calibration_screen.py create mode 100644 experiments/oral_a_v2_full_development.py diff --git a/ORAL_A_V2.md b/ORAL_A_V2.md new file mode 100644 index 0000000..d4586f6 --- /dev/null +++ b/ORAL_A_V2.md @@ -0,0 +1,105 @@ +# Oral-A-v2: representable-subspace development protocol + +## Status and claim boundary + +This is a transparent post-failure development branch. Frozen Oral-A-v1 +failed when its SDIL run became nonfinite at epoch 89; it remains failed and is +never relabeled. The executable diagnosis found that its unitwise K1 causal +target was not captured by the translation-shared feedback model. The v2 +change is restricted to that diagnosed interface: perturb the two basis fields +that the channel-gated vectorizer can actually express, then apply the same +expected full-field delta rule. Architecture, task update, causal-query count, +cadence, sigma, data split, and v1 optimization schedule do not change. + +No v2 GPU endpoint was observed before this protocol and its selectors were +committed. The CIFAR-10 test set and confirmation seeds 10--14 remain +untouched. A v2 result cannot be pooled with v1 or described as preregistered +before the v1 failure. + +## V2-0: mechanics and mathematical identity + +`experiments/conv_local_smoke.py` must establish all of the following on CPU: + +- structured moment-estimator cosine above 0.985 and norm ratio in [0.90,1.10]; +- antithetic directional derivative relative error below `2e-6` at a + differentiable fixed ResNet point; +- absolute error below `1e-14` between the implemented A/G update and the + analytically expected full-field delta-rule update; +- all legacy convolutional, BatchNorm, predictor, and local-eligibility checks + remain green. + +This stage passed before the GPU funnel was opened: cosine `0.998217`, norm +ratio `1.006989`, JVP relative error `1.69e-10`, and delta-rule absolute error +`2.71e-20`. + +## V2-1: frozen-forward causal-capture screen + +Use seed 0 ResNet-20, the frozen 45,000/5,000 split, the first 10,000 training +examples, batch 128, and 400 feedback-only minibatches. Forward weights, BN +state and affine parameters, and the readout remain bitwise fixed. The +vectorizer is `channel_gated`, starts at exactly zero (`a_scale=0`), and uses +K1 antithetic calibration with `sigma=0.01`. Cross: + +- estimator: legacy `unit_targets`, v2 `channel_subspace`; +- apical rate: `0.01`, `0.1`, `1.0`. + +All candidates use identical data, perturbation seed 1000, and a fixed +64-example training-prefix exact-gradient audit. Autograd is used only after +calibration to audit teaching alignment; it never updates a parameter. + +Within each estimator select maximum early-third teaching/negative-gradient +cosine, then maximum all-layer cosine, then lower apical rate. V2 advances +only if all records are finite and the selected structured estimator has: + +1. early-third cosine at least `0.01`; +2. all-layer cosine at least `0.01`; +3. early-third cosine at least `0.01` above the best matched unit-target run. + +The estimator-specific prediction/target cosine is reported, but it is not +compared across modes because one record is measured in the full hidden field +and the other in its two channel-basis moments. No rate or threshold is added +after results are observed. + +## V2-2: full ResNet-20 validation gate + +Only if V2-1 passes, run one seed-0 SDIL model for 200 epochs on all 45,000 +development-training examples. Copy Oral-A-v1 exactly: ResNet-20 width 16, +batch 128, hidden LR `0.03`, output LR `0.1`, momentum `0.9`, weight decay +`1e-4`, and 10x drops at epochs 100 and 150. Use the V2-1-selected apical +rate, zero-initialized channel-gated feedback, 400 feedback-only warmup steps, +and structured K1/every-4 calibration at `sigma=0.01`. + +The already frozen BP (`91.62%`) and DFA (`33.06%`) records remain the matched +references. V2 passes only if: + +1. every loss and metric is finite; +2. final validation accuracy is within 5 points of BP and at least 2 points + above DFA; +3. early-third teaching alignment is at least `0.05`; +4. estimated total training MACs do not exceed BP. + +Failure closes v2; there is no LR, warmup, cadence, sigma, direction-count, or +vectorizer recovery branch. + +## V2-3: untouched depth confirmation + +Only after V2-2 passes, copy its complete hyperparameters without depth tuning +to ResNet-20/32/56 and model/data-loader seeds 10--14. Refit on all 50,000 +training examples. Each run evaluates CIFAR-10 test exactly once at the end; +no intermediate test endpoint is logged. Exact BP and the v1-selected DFA are +run on the identical panel. + +The confirmation gate is unchanged from Oral-A-v1: all 45 trajectories finite; +SDIL within 2 points of BP at every depth; SDIL at least 2 points above DFA at +depth 56; paired SDIL depth-20 to depth-56 change no worse than -2 points; +depth-56 early-third alignment at least `0.05` and at least 30% of depth-20; +and SDIL nondominated in accuracy versus estimated MACs, causal queries, and +peak memory. Wall time is descriptive only. + +## Reviewer-score rule + +V2-0 and V2-1 cannot raise the strict ICLR score because they establish +mechanics and causal capture rather than standard-scale task success. A V2-2 +pass can move the score from 5 only after its full audit is committed. A V2-3 +multi-depth, multi-seed pass is required for an oral-level scaling claim. + diff --git a/README.md b/README.md index d13f8f4..f8615e7 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,7 @@ somatic statistic rather than arbitrary top-down context. `ROADMAP.md` and - `BASELINES.md`: matched and native-author baseline ledger; - `ROADMAP.md`: accept/oral evidence gates and their state; - `ORAL_A.md`: frozen standard CIFAR ResNet funnel; +- `ORAL_A_V2.md`: frozen post-failure representable-subspace funnel; - `REVIEW_SCORECARD.md`: adversarial ICLR-style score trajectory; - `results/figs/`: deterministic PDF/PNG main figures, captions, and a source hash manifest. diff --git a/experiments/analyze_oral_a_v2_calibration.py b/experiments/analyze_oral_a_v2_calibration.py new file mode 100644 index 0000000..af2bd39 --- /dev/null +++ b/experiments/analyze_oral_a_v2_calibration.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +"""Apply the frozen Oral-A-v2 causal-capture gate.""" +import argparse +import glob +import json +import math +import os + + +MODES = ("unit_targets", "channel_subspace") +RATES = (0.01, 0.1, 1.0) +SPLIT_HASH = "8328b206a97c420e49e54e3eca4abe3274c4756b084355784ea3fb8059e4515b" + + +def load(path): + with open(path) as handle: + record = json.load(handle) + args = record["args"] + expected = { + "mode": "sdil", "depth": 20, "width": 16, "seed": 0, + "epochs": 0, "train_limit": 10000, "val_examples": 5000, + "a_warmup_steps": 400, "pert_directions": 1, "pert_every": 4, + "pert_sigma": 0.01, "perturb_seed": 1000, + "normalization": "batchnorm", "vectorizer_mode": "channel_gated", + "a_scale": 0.0, "alignment_probe": 64, + } + for key, value in expected.items(): + if args.get(key) != value: + raise ValueError( + f"{path}: {key}={args.get(key)!r}, expected {value!r}") + if record["provenance"]["git_tracked_dirty"]: + raise ValueError(f"tracked-dirty result: {path}") + if record["split"]["validation_index_sha256"] != SPLIT_HASH: + raise ValueError(f"split drift: {path}") + mode = args["apical_calibration_mode"] + expected_space = ("channel_basis_moments" if mode == "channel_subspace" + else "full_hidden_field") + if record.get("calibration_metric_space") != expected_space: + raise ValueError(f"calibration metric-space drift: {path}") + diagnostics = record.get("diagnostics") + warmup = record.get("apical_warmup", {}).get("mean") + if diagnostics is None or warmup is None: + raise ValueError(f"missing diagnostics/warmup aggregate: {path}") + values = diagnostics["teaching_negative_gradient_cosine"] + early_count = max(1, len(values) // 3) + metrics = { + "early_third_alignment": sum(values[:early_count]) / early_count, + "all_layer_alignment": sum(values) / len(values), + "mean_calibration_mse": warmup["calibration_mse"], + "mean_target_power": warmup["target_power"], + "mean_prediction_target_cosine": warmup["prediction_target_cosine"], + "mean_parameter_update_rms": warmup.get("parameter_update_rms", 0.0), + } + finite = (record["final"]["finite"] + and all(math.isfinite(value) for value in metrics.values())) + return { + "path": path, + "source_commit": record["provenance"]["git_commit"], + "calibration_mode": mode, + "eta_A": float(args["eta_A"]), + "metric_space": expected_space, + "metrics": metrics, + "finite": finite, + } + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--input", default="results/oral_a_v2_calibration") + parser.add_argument("--out", default="results/oral_a_v2_calibration_gate.json") + args = parser.parse_args() + rows = [load(path) for path in sorted(glob.glob( + os.path.join(args.input, "*.json")))] + observed = {(row["calibration_mode"], row["eta_A"]) for row in rows} + expected = {(mode, rate) for mode in MODES for rate in RATES} + if observed != expected or len(rows) != len(expected): + raise ValueError( + f"incomplete v2 grid: missing={expected-observed}, extra={observed-expected}") + if len({row["source_commit"] for row in rows}) != 1: + raise ValueError("v2 calibration source commits differ") + + selected = {} + for mode in MODES: + candidates = [row for row in rows + if row["calibration_mode"] == mode and row["finite"]] + if candidates: + candidates.sort(key=lambda row: ( + -row["metrics"]["early_third_alignment"], + -row["metrics"]["all_layer_alignment"], row["eta_A"])) + selected[mode] = candidates[0] + checks = { + "all_six_records_finite": all(row["finite"] for row in rows), + "both_modes_selected": len(selected) == len(MODES), + } + if checks["both_modes_selected"]: + structured = selected["channel_subspace"]["metrics"] + unit = selected["unit_targets"]["metrics"] + checks.update({ + "structured_early_third_at_least_0.01": ( + structured["early_third_alignment"] >= 0.01), + "structured_all_layer_at_least_0.01": ( + structured["all_layer_alignment"] >= 0.01), + "structured_early_gain_over_unit_at_least_0.01": ( + structured["early_third_alignment"] + - unit["early_third_alignment"] >= 0.01), + }) + else: + checks.update({ + "structured_early_third_at_least_0.01": False, + "structured_all_layer_at_least_0.01": False, + "structured_early_gain_over_unit_at_least_0.01": False, + }) + passed = all(checks.values()) + output = { + "protocol": "oral_a_v2_causal_capture_v1", + "status": "passed" if passed else "failed", + "checks": checks, + "rows": rows, + "selected": selected, + "confirmation_test_seeds_touched": False, + "review_score_before": 5, + "review_score_after": 5, + "score_change_rule": "mechanics/calibration alone cannot raise score", + } + os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) + with open(args.out, "w") as handle: + json.dump(output, handle, indent=2, sort_keys=True) + handle.write("\n") + print(json.dumps({ + "status": output["status"], "checks": checks, + "selected": selected, + }, indent=2)) + + +if __name__ == "__main__": + main() + diff --git a/experiments/analyze_oral_a_v2_full.py b/experiments/analyze_oral_a_v2_full.py new file mode 100644 index 0000000..16f7e78 --- /dev/null +++ b/experiments/analyze_oral_a_v2_full.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +"""Apply the frozen Oral-A-v2 full-validation gate.""" +import argparse +import json +import math +import os + + +SPLIT_HASH = "8328b206a97c420e49e54e3eca4abe3274c4756b084355784ea3fb8059e4515b" + + +def finite_tree(value): + if isinstance(value, dict): + return all(finite_tree(item) for item in value.values()) + if isinstance(value, list): + return all(finite_tree(item) for item in value) + if isinstance(value, float): + return math.isfinite(value) + return True + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--run", default="results/oral_a_v2_dev/sdil_full_r20_s0.json") + parser.add_argument( + "--selection", default="results/oral_a_v2_calibration_gate.json") + parser.add_argument( + "--bp", default="results/oral_a_dev/bp_reference_primary.json") + parser.add_argument( + "--dfa", default="results/oral_a_dev/dfa_full_r20_s0.json") + parser.add_argument("--out", default="results/oral_a_v2_full_gate.json") + args = parser.parse_args() + with open(args.run) as handle: + run = json.load(handle) + with open(args.selection) as handle: + selection = json.load(handle) + with open(args.bp) as handle: + bp = json.load(handle) + with open(args.dfa) as handle: + dfa = json.load(handle) + if selection["status"] != "passed": + raise ValueError("v2 causal-capture gate did not pass") + chosen = selection["selected"]["channel_subspace"] + expected = { + "mode": "sdil", "depth": 20, "width": 16, "seed": 0, + "epochs": 200, "val_examples": 5000, "lr": 0.03, + "output_lr": 0.1, "lr_schedule": "step", + "lr_milestones": "100,150", "lr_gamma": 0.1, + "a_scale": 0.0, "eta_A": chosen["eta_A"], + "a_warmup_steps": 400, "apical_calibration_mode": "channel_subspace", + "pert_sigma": 0.01, "pert_directions": 1, "pert_every": 4, + "normalization": "batchnorm", "vectorizer_mode": "channel_gated", + } + for key, value in expected.items(): + if run["args"].get(key) != value: + raise ValueError( + f"v2 run {key}={run['args'].get(key)!r}, expected {value!r}") + if run["provenance"]["git_tracked_dirty"]: + raise ValueError("tracked-dirty v2 result") + if run["split"]["validation_index_sha256"] != SPLIT_HASH: + raise ValueError("v2 split drift") + if run["evaluation_protocol"]["test_evaluations"] != 0: + raise ValueError("test endpoint touched during v2 development") + values = run["diagnostics"]["teaching_negative_gradient_cosine"] + early_count = max(1, len(values) // 3) + early = sum(values[:early_count]) / early_count + sdil_accuracy = run["final"]["accuracy"] + bp_accuracy = bp["final"]["accuracy"] + dfa_accuracy = dfa["final"]["accuracy"] + checks = { + "all_metrics_finite": run["final"]["finite"] and finite_tree(run), + "bp_reference_at_least_90pct": bp_accuracy >= 0.90, + "sdil_within_5pt_of_bp": sdil_accuracy >= bp_accuracy - 0.05, + "sdil_at_least_2pt_above_dfa": sdil_accuracy >= dfa_accuracy + 0.02, + "early_third_alignment_at_least_0.05": early >= 0.05, + "sdil_macs_no_more_than_bp": ( + run["work"]["total_macs_estimate"] + <= bp["work"]["total_macs_estimate"]), + } + passed = all(checks.values()) + output = { + "protocol": "oral_a_v2_full_validation_v1", + "status": "passed" if passed else "failed", + "checks": checks, + "metrics": { + "sdil_validation_accuracy": sdil_accuracy, + "bp_validation_accuracy": bp_accuracy, + "dfa_validation_accuracy": dfa_accuracy, + "early_third_alignment": early, + "sdil_total_macs": run["work"]["total_macs_estimate"], + "bp_total_macs": bp["work"]["total_macs_estimate"], + }, + "sources": {"sdil": args.run, "bp": args.bp, "dfa": args.dfa}, + "confirmation_test_seeds_touched": False, + "review_score_before": 5, + "review_score_after": 6 if passed else 5, + "review_score_rationale": ( + "full standard-scale validation gate passed; independent depth " + "confirmation still required" + if passed else + "full standard-scale validation gate failed; controlled evidence " + "remains unchanged"), + } + os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) + with open(args.out, "w") as handle: + json.dump(output, handle, indent=2, sort_keys=True) + handle.write("\n") + print(json.dumps({ + "status": output["status"], "checks": checks, + "metrics": output["metrics"], + }, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/experiments/conv_run.py b/experiments/conv_run.py index cb99545..a33005d 100644 --- a/experiments/conv_run.py +++ b/experiments/conv_run.py @@ -212,6 +212,11 @@ def run(args): log = { "schema_version": 1, "protocol_family": "oral_a_cifar_local_resnet_development", + "calibration_metric_space": ( + None if config is None else + ("channel_basis_moments" + if config.apical_calibration_mode == "channel_subspace" + else "full_hidden_field")), "args": vars(args), "provenance": provenance(), "split": split, @@ -284,8 +289,15 @@ def run(args): counters["per_example_loss_terms"] += 2 * config.pert_directions * batch counters["perturbation_events"] += 1 train.g.set_state(loader_state) + warmup_mean = { + key: sum(metric[key] for metric in warmup_metrics) + / len(warmup_metrics) + for key in warmup_metrics[0] + } log["apical_warmup"] = { "steps": args.a_warmup_steps, + "first": warmup_metrics[0], + "mean": warmup_mean, "last": warmup_metrics[-1], } sync(args.device) diff --git a/experiments/oral_a_v2_calibration_screen.py b/experiments/oral_a_v2_calibration_screen.py new file mode 100644 index 0000000..7666dd4 --- /dev/null +++ b/experiments/oral_a_v2_calibration_screen.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +"""Run a deterministic shard of the frozen Oral-A-v2 calibration screen.""" +import argparse +import os +import subprocess +import sys + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--device", default="cuda") + parser.add_argument("--shard_index", type=int, default=0) + parser.add_argument("--num_shards", type=int, default=1) + parser.add_argument("--dry_run", action="store_true") + args = parser.parse_args() + if not 0 <= args.shard_index < args.num_shards: + raise ValueError("invalid shard index") + + common = [ + sys.executable, "experiments/conv_run.py", + "--mode", "sdil", "--device", args.device, + "--depth", "20", "--width", "16", "--seed", "0", + "--loader_seed", "0", "--batch_size", "128", "--epochs", "0", + "--train_limit", "10000", "--val_examples", "5000", + "--split_seed", "2027", "--eval_split", "validation", + "--eval_every", "0", "--augment_train", "1", + "--lr", "0.03", "--output_lr", "0.1", + "--lr_schedule", "constant", "--warmup_epochs", "0", + "--momentum", "0.9", "--weight_decay", "1e-4", + "--normalization", "batchnorm", "--vectorizer_mode", "channel_gated", + "--a_scale", "0", "--a_warmup_steps", "400", + "--pert_sigma", "0.01", "--pert_directions", "1", + "--pert_every", "4", "--perturb_seed", "1000", + "--alignment_probe", "64", + ] + jobs = [] + for calibration in ("unit_targets", "channel_subspace"): + for rate in (0.01, 0.1, 1.0): + tag = f"{calibration}_etaA{rate}" + jobs.append((tag, common + [ + "--apical_calibration_mode", calibration, + "--eta_A", str(rate), + "--out", f"results/oral_a_v2_calibration/{tag}.json", + ])) + os.makedirs("results/oral_a_v2_calibration", exist_ok=True) + for index, (tag, command) in enumerate(jobs): + if index % args.num_shards != args.shard_index: + continue + print(tag, " ".join(command), flush=True) + if not args.dry_run: + subprocess.run(command, check=True) + + +if __name__ == "__main__": + main() + diff --git a/experiments/oral_a_v2_full_development.py b/experiments/oral_a_v2_full_development.py new file mode 100644 index 0000000..457f7b2 --- /dev/null +++ b/experiments/oral_a_v2_full_development.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +"""Run the single frozen Oral-A-v2 full ResNet-20 validation job.""" +import argparse +import json +import os +import subprocess +import sys + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--selection", default="results/oral_a_v2_calibration_gate.json") + parser.add_argument("--device", default="cuda") + parser.add_argument("--dry_run", action="store_true") + args = parser.parse_args() + with open(args.selection) as handle: + selection = json.load(handle) + if selection["status"] != "passed": + raise ValueError("Oral-A-v2 causal-capture gate did not pass") + chosen = selection["selected"]["channel_subspace"] + command = [ + sys.executable, "experiments/conv_run.py", + "--mode", "sdil", "--device", args.device, + "--depth", "20", "--width", "16", "--seed", "0", + "--loader_seed", "0", "--batch_size", "128", "--epochs", "200", + "--val_examples", "5000", "--split_seed", "2027", + "--eval_split", "validation", "--eval_every", "20", + "--augment_train", "1", "--lr", "0.03", "--output_lr", "0.1", + "--lr_schedule", "step", "--lr_milestones", "100,150", + "--lr_gamma", "0.1", "--warmup_epochs", "0", "--momentum", "0.9", + "--weight_decay", "1e-4", "--normalization", "batchnorm", + "--vectorizer_mode", "channel_gated", "--a_scale", "0", + "--eta_A", str(chosen["eta_A"]), "--a_warmup_steps", "400", + "--apical_calibration_mode", "channel_subspace", + "--pert_sigma", "0.01", "--pert_directions", "1", + "--pert_every", "4", "--perturb_seed", "1000", + "--alignment_probe", "32", + "--out", "results/oral_a_v2_dev/sdil_full_r20_s0.json", + ] + os.makedirs("results/oral_a_v2_dev", exist_ok=True) + print(" ".join(command), flush=True) + if not args.dry_run: + subprocess.run(command, check=True) + + +if __name__ == "__main__": + main() + -- cgit v1.2.3