diff options
| -rw-r--r-- | RESULTS.md | 28 | ||||
| -rw-r--r-- | ROADMAP.md | 8 | ||||
| -rw-r--r-- | experiments/analyze_c2_context_validation.py | 8 |
3 files changed, 41 insertions, 3 deletions
@@ -326,6 +326,34 @@ normalized local deltas do not remove the basin variability. This points to a re limitation of the fixed linear `A_l c` vectorizer: on a compositional target, the correct hidden direction depends on neural state as well as the two-dimensional output error. +## C2 bounded context-vectorizer recovery (validation only) + +To test the identified representation bottleneck, the apical vectorizer was extended to +`A_l c + G_l vec(c outer tanh(h_top))`. The gated path is zero-initialized, trained by the same +local perturbation regression, and vanishes in neutral periods. After a task-seed-0 development +screen selected `eta_A=0.01`, K1/e4, an independent frozen panel used new task seeds 3--5 and five +student seeds, again crossing BP, FA, DFA, and SDIL at d1/d4. All 120 runs came from clean revision +`b18227f`; independent test sets were never evaluated. + +| method | depth | validation acc (%) | d1-to-d4 gain | d4 lesion drop | +|:-------|------:|-------------------:|--------------:|---------------:| +| BP | 1 | 68.633 ± 5.351 | — | — | +| BP | 4 | 97.187 ± 1.532 | +28.553 | +34.170 | +| FA | 1 | 52.640 ± 4.528 | — | — | +| FA | 4 | 80.763 ± 20.592 | +28.123 | +20.313 | +| DFA | 1 | 56.313 ± 8.374 | — | — | +| DFA | 4 | 77.097 ± 17.667 | +20.783 | +25.393 | +| context-SDIL | 1 | 63.977 ± 8.636 | — | — | +| context-SDIL | 4 | 79.120 ± 22.765 | +15.143 | +27.150 | + +Context conditioning raises SDIL's recovery of BP depth gain from the linear panel's `21.0%` to +`53.0%`, confirming that state dependence was a real bottleneck. It is still below the frozen +70% requirement, while FA recovers `98.5%`; context-SDIL's d4 mean is `1.643` points below FA. +Exactly 10/15 SDIL pairs gain with depth and 10/15 are hurt by the lesion, but one d4 run ends with +nonfinite loss and chance accuracy. **The independent recovery gate fails and C2 test remains +prohibited.** A richer vectorizer can improve mean credit without curing coupled A/W basin +instability; this is now a documented limitation rather than an oral-level scaling result. + ## Paper-faithful non-backprop baselines `PETITE` in the initial request is treated as **PEPITA**. Implementations and hyperparameters were @@ -140,6 +140,14 @@ be positive; its d4 lesion must lose at least 2 points on average and harm at le Failure closes this recovery. Passing freezes a single test confirmation over task seeds 0--5, student seeds 0--4, all four methods, and d1/d4, retrained on the unchanged 8,000-example subsets. +**Context-vectorizer recovery status: failed validation on 2026-07-22; test remains unevaluated.** +Context-SDIL recovered `53.0%` of BP's depth gain (`+15.143` versus `+28.553` points), below 70%, +and its d4 endpoint trailed FA by `1.643` points. One d4 trajectory had nonfinite final loss. +Positive depth gains and positive lesion effects each occurred in exactly 10/15 runs, so those +secondary conditions passed at the boundary. Context conditioning improves substantially over +linear SDIL's 21.0% recovery but does not make causal-feedback learning reliable enough. The +predeclared mechanism-recovery branch is closed and no C2 test panel will run. + ### C3. Causal calibration is genuinely amortized **Status: passed on 2026-07-22.** The frozen five-seed confirmation retained 112.9% of the K16/e4 diff --git a/experiments/analyze_c2_context_validation.py b/experiments/analyze_c2_context_validation.py index c1ba9b3..f9ed7f6 100644 --- a/experiments/analyze_c2_context_validation.py +++ b/experiments/analyze_c2_context_validation.py @@ -45,8 +45,6 @@ def audit_row(path, row): raise RuntimeError(f"protocol mismatch {path}: {mismatches}") if row["final"].get("eval_split") != "validation": raise RuntimeError(f"test-contaminated validation row: {path}") - if not math.isfinite(row["final"].get("val_loss", math.nan)): - raise RuntimeError(f"nonfinite final validation loss: {path}") if any("eval_acc" in step or "cos_r_negg" in step for step in row.get("steps", [])): raise RuntimeError(f"intermediate held-out metric/diagnostic: {path}") split = row.get("split", {}) @@ -93,6 +91,9 @@ def main(): if any(len(hashes) != 1 for hashes in split_hashes.values()): raise RuntimeError(f"methods did not share splits within tasks: {split_hashes}") + nonfinite = [key for key, row in rows.items() + if not math.isfinite(row["final"].get("val_loss", math.nan))] + print(f"commit={next(iter(commits))} rows={len(rows)} task_seeds={list(TASK_SEEDS)}") print("| method | depth | validation (%) | depth gain | lesion drop |") print("|:---|---:|---:|---:|---:|") @@ -134,7 +135,7 @@ def main(): lesion_mean = statistics.mean(lesion_drops) lesion_positive = sum(value > 0 for value in lesion_drops) comparator_ok = strongest_recovery <= 0.5 or deep_advantage >= 2.0 - passed = (bp_gain >= 5.0 and recovery >= 0.7 and positive_gains >= 10 + passed = (not nonfinite and bp_gain >= 5.0 and recovery >= 0.7 and positive_gains >= 10 and comparator_ok and lesion_mean >= 2.0 and lesion_positive >= 10) print(f"BP gain={bp_gain:+.3f}; context-SDIL gain={sdil_gain:+.3f}; " @@ -142,6 +143,7 @@ def main(): print(f"competitor recoveries={competitor_recoveries}; strongest={100 * strongest_recovery:.1f}%") print(f"context-SDIL d4 advantage over strongest endpoint={deep_advantage:+.3f}") print(f"context-SDIL lesion mean={lesion_mean:+.3f}; positive={lesion_positive}/15") + print(f"nonfinite final losses={nonfinite}") print(f"C2 context validation gate: {'PASS' if passed else 'FAIL'}") if not passed: raise SystemExit(1) |