diff options
| -rw-r--r-- | docs/campaign/CASCADE_ABLATION_PLAN.md | 24 | ||||
| -rw-r--r-- | ep_run/casc_eq_train.py | 20 |
2 files changed, 40 insertions, 4 deletions
diff --git a/docs/campaign/CASCADE_ABLATION_PLAN.md b/docs/campaign/CASCADE_ABLATION_PLAN.md index 55df056..2cc9f05 100644 --- a/docs/campaign/CASCADE_ABLATION_PLAN.md +++ b/docs/campaign/CASCADE_ABLATION_PLAN.md @@ -342,3 +342,27 @@ val); the D1a "no-death" correction; Delta cancellation scope. (lr, sharpening-rate driver) and D (BP, EP-specificity). 8. Process fixes: watcher was not harness-tracked (user caught it — now all watchers via tracked bg tasks); zsh $VAR word-splitting cost two launch retries (all launches now via bash scripts). + +### RESULT 6 (2026-07-10 09:35): WALL-2 DIAGNOSED — marginal under-convergence, EP-specific; kretry fix shipped; OLMo2 matrix launched. +A/B/C/D verdict (resume from pre-bifurcation ckpt-10000, beta floored): +| arm | skips @ window | note | +|---|---|---| +| A ctl (K3, lr1e-3) | **16, accelerating** (val wobble 2.00@12400) | leading indicator REPRODUCES | +| B K8 | **2** | rejections nearly eliminated | +| C lr3e-4 | **1**, best 1.5163 (best of all) | never touches the edge | +| D BP (same ckpt/config/lr) | clean through 12750 | **EP-specific confirmed** | +**Mechanism (two walls, two levers — revises "K refuted"):** +- Wall-1 (~2-4k): cos erosion = finite-beta SNR -> beta-floor (K genuinely irrelevant there). +- Wall-2 (~11k+): operator sharpens -> a growing fraction of batches sit at the CONTRACTIVITY EDGE of + the nudged fb relaxation and under-converge at K3 -> drift-guard rejections climb -> gradient bias + + occasional marginal escapes -> blowup. K8 CONVERGES those batches (16 -> 2 rejections) => marginal + under-convergence, NOT hard divergence. lr modulates when the edge arrives (C: skips~1 and better CE). + BP has no relaxation -> no wall-2 (D clean). Original 12100 didn't literally replay in A (fresh Adam + + different data order — the recorded confounds) but the leading indicator did. +**FIX SHIPPED: `--kretry N`** — on drift-reject, RETRY the batch once with N fb rounds (B proved K8 +converges them) instead of dropping it. Converts biased skips into converged gradients; costs extra +rounds ONLY on marginal batches (~0.1-1% of steps). Telemetry: skips=(d/g/r). +**OLMo2 4k matrix LAUNCHED** (ol_bp_s1-3 + ol_ep_s1-3, wd 0.1, EP: beta_floor 3e-4 + kretry 8; twin +step-0 losses bitwise-identical per seed). Watcher auto-computes parity and — if EP mean within 0.05 +of BP — AUTO-LAUNCHES the Stage-1 OLMo2 TinyStories epoch (stage1_ol_ep, 58.8k steps, kretry armed). +OLMo2's bounded-per-branch signals may also shift wall-2 later; kretry is the belt-and-suspenders. diff --git a/ep_run/casc_eq_train.py b/ep_run/casc_eq_train.py index f3b8736..5a2677c 100644 --- a/ep_run/casc_eq_train.py +++ b/ep_run/casc_eq_train.py @@ -37,6 +37,7 @@ ap.add_argument('--sig0', type=float, default=-1.0) # override SIG0 (beta-s ap.add_argument('--olmo2', action='store_true') # OLMo2-standard block: norm-AFTER-sublayer RMSNorm, full-width QK-norm, RoPE(500k), SwiGLU, no-bias, untied head, final RMSNorm, 0.02 init ap.add_argument('--wd', type=float, default=-1.0) # >=0: grouped weight decay (linear weights+head decay; embeddings/norm-gains none). <0 = legacy uniform 1e-4 ap.add_argument('--zloss', type=float, default=0.0) # z-loss coefficient on train objective (OLMo2-style logit regularizer); 0 = off +ap.add_argument('--kretry', type=int, default=0) # >0: on drift-reject, RETRY the batch once with this many fb rounds (diag B: K8 converges the marginal batches) instead of dropping it ap.add_argument('--dtop_every', type=int, default=1) # 1 = exact (DEFAULT, BP-parity); 2 = fast mode (~20% cheaper, ~4% CE tax at high lr) ap.add_argument('--gate_every', type=int, default=200) # in-training cos(EP,BP) telemetry; <=0 = fully BP-free (no bp_gate at all) ap.add_argument('--gate_govern', action='store_true') # let gate cos adjust K/bscale (default: observe-only => training control is BP-free) @@ -290,9 +291,20 @@ def ep_step(x, y): drift = sum(float((a - b).norm()) for a, b in zip(zp, zs_free)) / max( sum(float(b.norm()) for b in zs_free), 1e-9) if (not math.isfinite(drift)) or (drift > 0.5 and not args.noguard): - GOV['skd'] = GOV.get('skd', 0) + 1 # drift-guard reject (relaxation non-convergence) - for p in all_params: p.grad = None - return free_ce, beta_t, GOV['K'], False + ok_retry = False + if args.kretry > 0 and math.isfinite(drift) and not args.noguard: + GOV['skr'] = GOV.get('skr', 0) + 1 # marginal batch: retry once with deeper relaxation + z0, zs, ins, outs = free_states_graphed(x) + zs_free = [z.clone() for z in zs] + zp, last_outs = relax(z0, zs, ins, outs, y, +beta_t, args.kretry, x) + with torch.no_grad(): + drift = sum(float((a - b).norm()) for a, b in zip(zp, zs_free)) / max( + sum(float(b.norm()) for b in zs_free), 1e-9) + ok_retry = math.isfinite(drift) and drift <= 0.5 + if not ok_retry: + GOV['skd'] = GOV.get('skd', 0) + 1 # drift-guard reject (relaxation non-convergence) + for p in all_params: p.grad = None + return free_ce, beta_t, GOV['K'], False GOV['drift'] = drift E = 0.0 for z, o in zip(zp, last_outs): E = E + 0.5 * ((z.detach() - o) ** 2).sum() @@ -368,7 +380,7 @@ for step in range(start_step, args.steps + 1): val = evaluate(); best = min(best, val) gtag = '' if math.isnan(gcos) else f' cos={gcos:.4f}' print(f'step {step:5d}/{args.steps} | train {ce:.4f} val {val:.4f} (best {best:.4f}) ' - f'| beta={beta_t:.2e} K={rounds} skips={skips}(d{GOV.get("skd",0)}/g{GOV.get("skg",0)}){gtag} ' + f'| beta={beta_t:.2e} K={rounds} skips={skips}(d{GOV.get("skd",0)}/g{GOV.get("skg",0)}/r{GOV.get("skr",0)}){gtag} ' f'drift={GOV["drift"]:.3f} gn={GOV["gn"]:.2e} sig={GOV["sig"]:.1f} | {step/max(time.time()-t0,1e-9):.3f} it/s', flush=True) if wb is not None: try: wb.log({'train_ce': ce, 'val_ce': val, 'best': best, 'beta_t': beta_t, |