diff options
| -rw-r--r-- | docs/campaign/CASCADE_ABLATION_PLAN.md | 6 | ||||
| -rw-r--r-- | ep_run/casc_eq_train.py | 6 |
2 files changed, 11 insertions, 1 deletions
diff --git a/docs/campaign/CASCADE_ABLATION_PLAN.md b/docs/campaign/CASCADE_ABLATION_PLAN.md index 2314e8e..66a9501 100644 --- a/docs/campaign/CASCADE_ABLATION_PLAN.md +++ b/docs/campaign/CASCADE_ABLATION_PLAN.md @@ -452,3 +452,9 @@ and zero skips at 1e-3 (larger nudge does NOT destabilize the OLMo2 relaxation). harvesting (auto-kill at 27k). RECIPE UPDATE for Stage-2 (and the next epoch): late beta_floor schedule — floor 3e-4 early, ramp to ~1e-3 in the back half (or floor ∝ 1/grad-norm). This likely also closes the +0.02-0.03 gap. + +### Dose-response SUSTAINED (25.6k-27k, 2k-step parallel traces): floor 3e-4 ~0.975 (accelerating +down, -0.0025/1k) | 5e-4 ~0.983 | 1e-3 ~0.990 flat, zero instability. Late-SNR mechanism + fix both +confirmed in-training. `--bf_late/--bf_late_at` flags shipped. NEXT-RUN RECIPE (post-epoch): OLMo2 + +Muon + beta_floor 3e-4 + bf_late 1e-3 @ ~20k + kretry 8 — expected to hold cos>=0.99 end-to-end and +likely close the +0.02-0.03 column gap. diff --git a/ep_run/casc_eq_train.py b/ep_run/casc_eq_train.py index 5a2677c..6bd83f8 100644 --- a/ep_run/casc_eq_train.py +++ b/ep_run/casc_eq_train.py @@ -38,6 +38,8 @@ ap.add_argument('--olmo2', action='store_true') # OLMo2-standard block: ap.add_argument('--wd', type=float, default=-1.0) # >=0: grouped weight decay (linear weights+head decay; embeddings/norm-gains none). <0 = legacy uniform 1e-4 ap.add_argument('--zloss', type=float, default=0.0) # z-loss coefficient on train objective (OLMo2-style logit regularizer); 0 = off ap.add_argument('--kretry', type=int, default=0) # >0: on drift-reject, RETRY the batch once with this many fb rounds (diag B: K8 converges the marginal batches) instead of dropping it +ap.add_argument('--bf_late', type=float, default=0.0) # >0: raise beta_floor to this value from step --bf_late_at (late-training SNR fix; dose-response 2026-07-10) +ap.add_argument('--bf_late_at', type=int, default=25000) ap.add_argument('--dtop_every', type=int, default=1) # 1 = exact (DEFAULT, BP-parity); 2 = fast mode (~20% cheaper, ~4% CE tax at high lr) ap.add_argument('--gate_every', type=int, default=200) # in-training cos(EP,BP) telemetry; <=0 = fully BP-free (no bp_gate at all) ap.add_argument('--gate_govern', action='store_true') # let gate cos adjust K/bscale (default: observe-only => training control is BP-free) @@ -282,7 +284,9 @@ def ep_step(x, y): if SIG0 is None: SIG0 = args.sig0 if args.sig0 > 0 else sig beta_t = args.beta * GOV['bscale'] * (SIG0 * SIG0) / max(sig * sig, 1e-9) if args.beta_fixed: beta_t = args.beta * GOV['bscale'] - if args.beta_floor > 0.0: beta_t = max(beta_t, args.beta_floor) + fl = args.beta_floor + if args.bf_late > 0.0 and GOV.get('step', 0) >= args.bf_late_at: fl = args.bf_late + if fl > 0.0: beta_t = max(beta_t, fl) z0, zs, ins, outs = free_states_graphed(x) zs_free = [z.clone() for z in zs] free_ce = F.cross_entropy(readout(zs_free[-1]).reshape(-1, vocab), y.reshape(-1)).item() |