From 2fb10ac8e5a45e1b401640193427b7ac56f2b36b Mon Sep 17 00:00:00 2001 From: YurenHao0426 Date: Wed, 22 Jul 2026 12:26:03 -0500 Subject: analysis: diagnose Oral-A calibration failure --- ORAL_A.md | 2 +- PAPER_PLAN.md | 2 +- README.md | 2 +- RESULTS.md | 13 +++- REVIEW_SCORECARD.md | 2 +- ROADMAP.md | 10 ++- experiments/analyze_oral_a_failure.py | 137 ++++++++++++++++++++++++++++++++++ results/oral_a_failure_diagnosis.json | 90 ++++++++++++++++++++++ 8 files changed, 252 insertions(+), 6 deletions(-) create mode 100644 experiments/analyze_oral_a_failure.py create mode 100644 results/oral_a_failure_diagnosis.json diff --git a/ORAL_A.md b/ORAL_A.md index be0b32f..03e4c2f 100644 --- a/ORAL_A.md +++ b/ORAL_A.md @@ -156,7 +156,7 @@ published multi-seed values are not pooled with this matched panel. - A2a selected both vectorizer families. A2b selected channel-gated SDIL at hidden LR `0.03`: `41.98%` versus tuned DFA's `37.16%` and BP's `74.94%`. - In A3, BP ended at `91.62%` and DFA ended finite at `33.06%`. SDIL became - nonfinite at epoch 90 and ended at `10.00%`; its early-third alignment was + nonfinite at epoch 89 and ended at `10.00%`; its early-third alignment was nonfinite. It passed the BP and MAC checks but failed the four remaining checks. - A3 status is `failed`. A4 was not opened, no confirmation JSON was produced, diff --git a/PAPER_PLAN.md b/PAPER_PLAN.md index 1055738..82b7c08 100644 --- a/PAPER_PLAN.md +++ b/PAPER_PLAN.md @@ -179,6 +179,6 @@ causal-feedback backbone preserves depth on the controlled task, but standard useful scaling remains unresolved. The correct response is a narrower title, claim, and score—not another post-hoc ResNet tuning branch. -This is the realized branch: A3 SDIL became nonfinite at epoch 90 and ended at +This is the realized branch: A3 SDIL became nonfinite at epoch 89 and ended at chance. A4 remained untouched. The working title therefore stays mechanism-focused, and the standard-ResNet result is a disclosed limitation. diff --git a/README.md b/README.md index 2c9dc7a..a690477 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ and scaling behavior. See `NOVELTY.md` for the exact prior-art boundary. - The frozen standard-ResNet funnel passed its BP reference and short screen: BP reached `91.62%`, and short-run channel-gated SDIL reached `41.98%` versus tuned DFA's `37.16%`. The full A3 run then failed decisively: SDIL - became nonfinite at epoch 90 and ended at chance, while DFA remained finite + became nonfinite at epoch 89 and ended at chance, while DFA remained finite at `33.06%`. A4 was therefore not opened and no confirmation test seed was touched. - Native author-code fidelity is complete. BurstCCN reaches `80.10%` at its diff --git a/RESULTS.md b/RESULTS.md index 6b11b34..e14cec3 100644 --- a/RESULTS.md +++ b/RESULTS.md @@ -623,7 +623,7 @@ The full 200-epoch A3 gate failed: | SDIL | 10.00 | no | NaN | 9.165e14 | SDIL reached `35.26%` at epoch 20, fell to `22.82%` at epoch 80, and became -nonfinite at epoch 90. It passed only the BP-reference and MAC checks; it failed +nonfinite at epoch 89. It passed only the BP-reference and MAC checks; it failed the within-5-point, DFA-advantage, alignment, and all-finite checks. The failure is specifically for the frozen recipe selected under the short cosine screen and transferred to the long step schedule. It is not evidence that every @@ -631,6 +631,17 @@ possible SDIL optimizer must diverge, but it does falsify the preregistered standard-ResNet scaling claim. A4 was not opened and confirmation seeds 10--14 were not touched. +The executable post-failure diagnosis finds a specific precursor rather than +only a bad endpoint. Before nonfiniteness, the apical prediction--target cosine +never exceeds `8.51e-5` in magnitude, while calibration MSE differs from target +power by at most `6.96e-8` relatively. Target power grows `2560.7x` from epoch +1 to 80 and `2.13e9x` by epoch 87. Thus the positive short-run teaching +alignment cannot be attributed to successful causal calibration of `A`; the +full-spatial K1 estimator is effectively all noise after projection into the +small channel-gated vectorizer. This is a post-failure diagnosis, not a passed +preregistered result, and motivates a separately labelled representable- +subspace estimator rather than a threshold change to Oral-A v1. + ## How to run `experiments/run.py --mode {bp,fa,dfa,sdil} --dataset {mnist,fmnist,cifar10} --depth D --residual {0,1} --act {tanh,gelu,silu,relu}` Batteries: `experiments/run_v2.sh "" "" `. diff --git a/REVIEW_SCORECARD.md b/REVIEW_SCORECARD.md index 861ba0e..df15681 100644 --- a/REVIEW_SCORECARD.md +++ b/REVIEW_SCORECARD.md @@ -57,7 +57,7 @@ Every formal result report records: | Current audited package | 5 | Strong depth-preservation and residual-necessity evidence; negative gates retained | Standard useful scale is absent | | Native baselines complete | 5 | BurstCCN is below its published endpoint; Dual Prop reproduces 92.46% versus 92.41%, with strict provenance and cost semantics | Fairness objection narrows, but SDIL gains no standard-scale evidence | | Oral-A A1/A2 | 5 | BP reached 91.62%; short channel-gated SDIL reached 41.98% versus tuned DFA at 37.16% | Development screening alone cannot raise the score | -| Oral-A A3 fails | 5 | Full ResNet-20 SDIL became nonfinite at epoch 90 and ended at 10%; DFA ended finite at 33.06% | Standard-scale and oral-A claims are closed; A4 remains untouched | +| Oral-A A3 fails | 5 | Full ResNet-20 SDIL became nonfinite at epoch 89 and ended at 10%; DFA ended finite at 33.06% | Standard-scale and oral-A claims are closed; A4 remains untouched | | Oral-A A4 | not opened | The prerequisite A3 gate failed | No oral-A confirmation claim is available | These are conditional reviewer forecasts, not promised scores. A failed stage leaves its negative diff --git a/ROADMAP.md b/ROADMAP.md index e2dd375..3203e04 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -341,12 +341,20 @@ claim is explicitly computational rather than a fit to cortical data. `ORAL_A.md` froze a validation-only ResNet-20 development funnel. A1 passed with `91.62%` BP validation accuracy. A2b selected channel-gated SDIL at `41.98%`, ahead of tuned DFA at `37.16%`, on the 10k/20-epoch screen. In full A3, DFA -ended finite at `33.06%`; SDIL became nonfinite at epoch 90 and ended at +ended finite at `33.06%`; SDIL became nonfinite at epoch 89 and ended at `10.00%`, failing alignment, accuracy, and finiteness gates. The MAC gate alone passed. Per the stop rule, the five-seed ResNet-20/32/56 test panel was never run. The exact BatchNorm/local-gradient mechanics remain verified, but the current training recipe does not establish standard-network scaling. +Post-failure diagnosis is recorded separately in +`results/oral_a_failure_diagnosis.json`: prediction--target cosine remains +below `8.51e-5` in magnitude, calibration MSE is indistinguishable from target +power, and target power grows `2.13e9x` before nonfiniteness. The next +development branch must therefore reduce causal-estimator variance in the +representable channel-gated subspace; changing only the v1 threshold is not an +allowed response. + Prepare convolutional local-update primitives and ResNet-20/32/56 protocols early. Queue frozen runs opportunistically on authorized idle GPUs. Because BurstCCN already reports CIFAR-10 and ImageNet scaling, dataset scale alone is not novel. The oral-level target is a memorable joint diff --git a/experiments/analyze_oral_a_failure.py b/experiments/analyze_oral_a_failure.py new file mode 100644 index 0000000..9f84bbf --- /dev/null +++ b/experiments/analyze_oral_a_failure.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +"""Audit the frozen Oral-A failure and quantify its calibration precursor.""" +import argparse +import json +import math +import os + + +def load(path): + with open(path) as handle: + return json.load(handle) + + +def finite(value): + return isinstance(value, (int, float)) and math.isfinite(value) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--full", default="results/oral_a_dev/sdil_full_r20_s0.json") + parser.add_argument( + "--short", default="results/oral_a_short/sdil_channel_gated_lr0.03.json") + parser.add_argument( + "--gate", default="results/oral_a_full_gate.json") + parser.add_argument( + "--out", default="results/oral_a_failure_diagnosis.json") + args = parser.parse_args() + + full = load(args.full) + short = load(args.short) + gate = load(args.gate) + expected = { + "mode": "sdil", "depth": 20, "epochs": 200, + "vectorizer_mode": "channel_gated", "a_scale": 1.0, + "eta_A": 0.001, "pert_directions": 1, "pert_every": 4, + } + for key, value in expected.items(): + if full["args"].get(key) != value: + raise ValueError(f"full A3 drift: {key}={full['args'].get(key)!r}") + if full["provenance"]["git_tracked_dirty"]: + raise ValueError("full A3 result has tracked source edits") + if gate["status"] != "failed" or gate["confirmation_test_seeds_touched"]: + raise ValueError("expected a failed A3 gate with untouched confirmation") + + calibration_rows = [] + first_nonfinite_epoch = None + eval_rows = [] + for row in full["epochs"]: + loss = row["train_loss"] + if first_nonfinite_epoch is None and not finite(loss): + first_nonfinite_epoch = row["epoch"] + if finite(row.get("eval_accuracy")): + eval_rows.append({ + "epoch": row["epoch"], + "accuracy": row["eval_accuracy"], + "loss": row["eval_loss"] if finite(row["eval_loss"]) else None, + "finite": finite(row["eval_loss"]), + }) + calibration = row.get("calibration") + if calibration and all(finite(calibration.get(key)) for key in ( + "calibration_mse", "target_power", "prediction_target_cosine")): + power = calibration["target_power"] + calibration_rows.append({ + "epoch": row["epoch"], + "target_power": power, + "mse_to_target_power": calibration["calibration_mse"] / power, + "prediction_target_cosine": calibration["prediction_target_cosine"], + }) + if first_nonfinite_epoch is None or not calibration_rows: + raise ValueError("A3 failure trajectory is incomplete") + finite_eval_rows = [row for row in eval_rows if row["finite"]] + + row_by_epoch = {row["epoch"]: row for row in calibration_rows} + checkpoints = [row_by_epoch[epoch] for epoch in (1, 20, 40, 60, 80, 87)] + target_growth_1_to_80 = ( + row_by_epoch[80]["target_power"] / row_by_epoch[1]["target_power"]) + target_growth_1_to_87 = ( + row_by_epoch[87]["target_power"] / row_by_epoch[1]["target_power"]) + max_abs_cosine = max(abs(row["prediction_target_cosine"]) + for row in calibration_rows) + max_mse_power_gap = max(abs(row["mse_to_target_power"] - 1.0) + for row in calibration_rows) + short_calibration = short["apical_warmup"]["last"] + output = { + "protocol": "oral_a_A3_failure_diagnosis_v1", + "source_paths": {"full": args.full, "short": args.short, "gate": args.gate}, + "source_commits": { + "full": full["provenance"]["git_commit"], + "short": short["provenance"]["git_commit"], + }, + "frozen_outcome": { + "first_nonfinite_epoch": first_nonfinite_epoch, + "final_accuracy": full["final"]["accuracy"], + "final_finite": full["final"]["finite"], + "last_finite_validation": finite_eval_rows[-1], + "last_reported_validation": eval_rows[-1], + "confirmation_test_seeds_touched": False, + }, + "short_screen_contrast": { + "accuracy": short["final"]["accuracy"], + "early_third_alignment": short["diagnostics"]["early_third_mean"], + "warmup_prediction_target_cosine": ( + short_calibration["prediction_target_cosine"]), + "warmup_mse_to_target_power": ( + short_calibration["calibration_mse"] + / short_calibration["target_power"]), + }, + "calibration_precursor": { + "checkpoints": checkpoints, + "target_power_growth_epoch1_to_80": target_growth_1_to_80, + "target_power_growth_epoch1_to_87": target_growth_1_to_87, + "max_abs_prediction_target_cosine_before_nonfinite": max_abs_cosine, + "max_abs_mse_to_target_power_minus_one": max_mse_power_gap, + }, + "diagnosis": { + "classification": "causal_target_not_captured_before_runaway", + "evidence": [ + "prediction-target cosine remains effectively zero", + "calibration MSE remains indistinguishable from target power", + "target power grows by orders of magnitude before nonfiniteness", + "short-run teaching alignment therefore cannot be attributed to learned A", + ], + "next_test": ( + "reduce estimator variance by perturbing the representable " + "channel-gated feedback subspace, not every spatial unit"), + }, + } + os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) + with open(args.out, "w") as handle: + json.dump(output, handle, indent=2, sort_keys=True) + handle.write("\n") + print(json.dumps(output, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/results/oral_a_failure_diagnosis.json b/results/oral_a_failure_diagnosis.json new file mode 100644 index 0000000..7c6b31b --- /dev/null +++ b/results/oral_a_failure_diagnosis.json @@ -0,0 +1,90 @@ +{ + "calibration_precursor": { + "checkpoints": [ + { + "epoch": 1, + "mse_to_target_power": 1.0000000242141691, + "prediction_target_cosine": -5.120151685872971e-06, + "target_power": 142.2609250246797 + }, + { + "epoch": 20, + "mse_to_target_power": 1.0000000055445346, + "prediction_target_cosine": -9.188050003749003e-06, + "target_power": 492.22621874637883 + }, + { + "epoch": 40, + "mse_to_target_power": 1.0000000207505706, + "prediction_target_cosine": -1.6028913891638828e-05, + "target_power": 3347.859035403952 + }, + { + "epoch": 60, + "mse_to_target_power": 0.9999999997640454, + "prediction_target_cosine": 4.038185739212611e-05, + "target_power": 12104.714606234214 + }, + { + "epoch": 80, + "mse_to_target_power": 0.9999999967773018, + "prediction_target_cosine": 6.468792391264821e-05, + "target_power": 364280.45838307025 + }, + { + "epoch": 87, + "mse_to_target_power": 1.0000000346545865, + "prediction_target_cosine": 1.4983485078330897e-05, + "target_power": 302564383414.18854 + } + ], + "max_abs_mse_to_target_power_minus_one": 6.963728138575931e-08, + "max_abs_prediction_target_cosine_before_nonfinite": 8.508675264290486e-05, + "target_power_growth_epoch1_to_80": 2560.650145638194, + "target_power_growth_epoch1_to_87": 2126827049.3932126 + }, + "diagnosis": { + "classification": "causal_target_not_captured_before_runaway", + "evidence": [ + "prediction-target cosine remains effectively zero", + "calibration MSE remains indistinguishable from target power", + "target power grows by orders of magnitude before nonfiniteness", + "short-run teaching alignment therefore cannot be attributed to learned A" + ], + "next_test": "reduce estimator variance by perturbing the representable channel-gated feedback subspace, not every spatial unit" + }, + "frozen_outcome": { + "confirmation_test_seeds_touched": false, + "final_accuracy": 0.1, + "final_finite": false, + "first_nonfinite_epoch": 89, + "last_finite_validation": { + "accuracy": 0.2282, + "epoch": 80, + "finite": true, + "loss": 1080.0792 + }, + "last_reported_validation": { + "accuracy": 0.1, + "epoch": 200, + "finite": false, + "loss": null + } + }, + "protocol": "oral_a_A3_failure_diagnosis_v1", + "short_screen_contrast": { + "accuracy": 0.4198, + "early_third_alignment": 0.08889565182228883, + "warmup_mse_to_target_power": 1.0000000128001325, + "warmup_prediction_target_cosine": 0.00016553175127441687 + }, + "source_commits": { + "full": "a1d060e6d96d4ab973d16cd75149dd61899c7c95", + "short": "a1d060e6d96d4ab973d16cd75149dd61899c7c95" + }, + "source_paths": { + "full": "results/oral_a_dev/sdil_full_r20_s0.json", + "gate": "results/oral_a_full_gate.json", + "short": "results/oral_a_short/sdil_channel_gated_lr0.03.json" + } +} -- cgit v1.2.3