From 98d5ea09d1498f802f2ad06f5f67b385e228f063 Mon Sep 17 00:00:00 2001 From: Yuren Hao Date: Tue, 28 Jul 2026 13:20:34 -0500 Subject: =?UTF-8?q?RESULT=2066:=20head=E8=A2=AB=E8=8A=82=E6=B5=81=E5=88=B0?= =?UTF-8?q?60%(=E4=BD=8D=E7=A7=BB=E5=BE=AE=E6=8E=A2=E9=92=88:=E5=85=A8?= =?UTF-8?q?=E6=97=8F1.00-1.02=E5=94=AFW=5Fout=200.60)=20=E2=80=94=20?= =?UTF-8?q?=E7=94=A8=E6=88=B7=E7=AD=89=E4=BB=B7LR=E5=81=87=E8=AF=B4?= =?UTF-8?q?=E5=91=BD=E4=B8=AD=E5=B9=B6=E5=AE=9A=E4=BD=8D=E5=88=B0=E5=8D=95?= =?UTF-8?q?=E7=9F=A9=E9=98=B5;=20Adam=E5=B2=9Bm/=E2=88=9Av=E8=8A=82?= =?UTF-8?q?=E6=B5=81,=E5=AF=B9centered/fp32/K8=E5=85=A8=E4=B8=8D=E6=95=8F?= =?UTF-8?q?=E6=84=9F=3DEP=E5=A4=B4=E8=AF=BB=E5=85=B1=E4=BA=AB=E7=BB=93?= =?UTF-8?q?=E6=9E=84;=20--head=5Flr=5Fmult=E5=AE=9E=E8=A3=85;=20=E7=94=B5?= =?UTF-8?q?=E6=B1=A02(headmix=E8=AF=8A=E6=96=AD+hlr1.67/2.5=E8=A1=A5?= =?UTF-8?q?=E5=81=BF)=E5=9C=A8=E9=A3=9E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014FAPDWQ49M5Ye3NpTndTpn --- docs/campaign/CASCADE_ABLATION_PLAN.md | 17 +++++++++++++++++ ep_run/casc_eq_train.py | 6 +++++- ep_run/muon.py | 10 ++++++++-- ep_run/probe_stepdisp.py | 24 ++++++++++++++++++++++++ 4 files changed, 54 insertions(+), 3 deletions(-) create mode 100644 ep_run/probe_stepdisp.py diff --git a/docs/campaign/CASCADE_ABLATION_PLAN.md b/docs/campaign/CASCADE_ABLATION_PLAN.md index 277ebf1..5c4b25c 100644 --- a/docs/campaign/CASCADE_ABLATION_PLAN.md +++ b/docs/campaign/CASCADE_ABLATION_PLAN.md @@ -568,6 +568,23 @@ direction), not training-under-fault; wave-2 = co-training with faults injected +### RESULT 66 (2026-07-28): THE HEAD IS THROTTLED TO 60% — user's equivalent-LR hypothesis +CONFIRMED and localized to ONE matrix; symmetrization/precision/depth all irrelevant to it. +Displacement micro-probe (600 steps from s250000, ||dW||/||W|| per family): every family +IDENTICAL EP-vs-BP (ratio 1.00-1.02) EXCEPT W_out: EP 0.0068 vs BP 0.0114 = 60%. Muon-managed +blocks are magnitude-immune (msign); W_out lives on the Adam island where step ~ m/sqrt(v) — +the EP head-read's per-coordinate temporal coherence is lower, v inflates ~2.8x, Adam throttles +the loss-facing matrix by 40%. Explains R44 (gap lives head-side), the anneal freeze (head +can't track features), and width-scaling (louder states -> stronger throttle). +BATTERY 1 (all closed ~0% of the 0.0685 leak): centered 0%, fp32 +3%, K8 (arm died, rerun +pending) — the throttle source is INVARIANT to estimator symmetry, precision, and relax depth: +it is the shared structure of the EP W_out read, not removable noise. (cos flat 0.986-0.988 +through the losing segment; EP gn 0.15, clip never fires — user's checklist all measured.) +FA-argument (user): static direction quality exonerated a priori — FA learns at cos 0.3. +BATTERY 2 IN FLIGHT: leak_headmix (--bpmix head = the head's share of the leak, diagnostic +ceiling), leak_hlr17 / leak_hlr25 (--head_lr_mult, the zero-cost compensation candidate). +If hlr claws back what headmix shows, the width fix is ONE hyperparameter. + ### RESULT 65 (2026-07-28): THE LEAK INSTRUMENT WORKS — first reading +0.0685/6k at the SAME state (20x the trajectory-averaged rate: front-loaded/self-adapting), screening battery 1 in flight. Paired 6k segments from fw135m_bsign_s250000: BP tail val_mean 3.4982 vs EP-plain diff --git a/ep_run/casc_eq_train.py b/ep_run/casc_eq_train.py index 0cd5521..07e2ad4 100644 --- a/ep_run/casc_eq_train.py +++ b/ep_run/casc_eq_train.py @@ -49,6 +49,8 @@ ap.add_argument('--bsign_rand', action='store_true') # random-sign beta per ste ap.add_argument('--bf16', action='store_true') # cast model to bf16 (E-accumulation + tok_sigma stay fp32) — the x0.5 cost lever, GATE before production ap.add_argument('--amp', action='store_true') # PROPER mixed precision: autocast(bf16) matmuls, fp32 params/states/d/E — amp_gate.py PASSED 2026-07-12 (cos 0.9682 vs fp32 0.9687); --bf16 naive-cast stays DEAD (state quantization, RESULT 11) ap.add_argument('--dtop_every', type=int, default=1) # 1 = exact (DEFAULT, BP-parity); 2 = fast mode (~20% cheaper, ~4% CE tax at high lr) +ap.add_argument('--head_lr_mult', type=float, default=1.0) # W_out Adam-group LR multiplier (C768 + # head-throttle compensation, RESULT 66) ap.add_argument('--res_gate', type=float, default=0.02) # legality residual threshold; 0.02 was calibrated # at C512 — C768 ran half a schedule semi-converged # UNDER it (R59b). Per-width rule: ~100x the healthy @@ -313,7 +315,9 @@ if args.opt == 'muon': from muon import build_hybrid opt, sched = build_hybrid(blocks, all_params, args.lr, args.muon_lr, args.warmup, muon_mom=args.muon_mom, adam_b1=args.adam_b1, - total_steps=(args.steps if args.cosine else 0), lr_min_ratio=args.lr_min_ratio) + total_steps=(args.steps if args.cosine else 0), lr_min_ratio=args.lr_min_ratio, + head_param=(W_out if args.untie or args.olmo2 else None), + head_lr_mult=args.head_lr_mult) else: if args.wd >= 0: # OLMo2-style grouped decay: linear weights + head decay; embeddings/norm-gains none nodecay = {id(p) for p in tok.parameters()} | {id(p) for p in pos.parameters()} | \ diff --git a/ep_run/muon.py b/ep_run/muon.py index d65a231..0806243 100644 --- a/ep_run/muon.py +++ b/ep_run/muon.py @@ -57,7 +57,7 @@ class MultiSched: def build_hybrid(blocks, other_params, lr_adamw, lr_muon, warmup, total_steps=0, lr_min_ratio=0.1, - muon_mom=0.95, adam_b1=0.9): + muon_mom=0.95, adam_b1=0.9, head_param=None, head_lr_mult=1.0): """Muon(2D block matrices) + AdamW(everything else). Scheds: linear warmup, then cosine decay to lr_min_ratio*peak if total_steps>0 (long runs), else constant after warmup (legacy). muon_mom/adam_b1: momentum knobs (late-SNR noise-averaging arms, 2026-07-13).""" @@ -66,7 +66,13 @@ def build_hybrid(blocks, other_params, lr_adamw, lr_muon, warmup, total_steps=0, mat_ids = {id(p) for p in mats} rest = [p for p in other_params if id(p) not in mat_ids] om = Muon(mats, lr=lr_muon, momentum=muon_mom) - oa = torch.optim.AdamW(rest, lr=lr_adamw, weight_decay=1e-4, betas=(adam_b1, 0.999)) + if head_param is not None and head_lr_mult != 1.0: + hid = id(head_param) + groups = [{'params': [p for p in rest if id(p) != hid], 'lr': lr_adamw}, + {'params': [p for p in rest if id(p) == hid], 'lr': lr_adamw * head_lr_mult}] + oa = torch.optim.AdamW(groups, lr=lr_adamw, weight_decay=1e-4, betas=(adam_b1, 0.999)) + else: + oa = torch.optim.AdamW(rest, lr=lr_adamw, weight_decay=1e-4, betas=(adam_b1, 0.999)) if total_steps > 0: def fn(s): if s < warmup: return (s + 1) / max(warmup, 1) diff --git a/ep_run/probe_stepdisp.py b/ep_run/probe_stepdisp.py new file mode 100644 index 0000000..2d86867 --- /dev/null +++ b/ep_run/probe_stepdisp.py @@ -0,0 +1,24 @@ +"""Direct effective-step-size measurement (user hypothesis: high cos but shrunken equivalent +LR -> slow learning masquerading as plateau). Load two ckpts of the same tag 600 steps apart +(saved by the micro-runs), report per-family ||dW||/||W|| — the actual distance moved in +weight space. Compare EP vs BP micro-runs from the same start.""" +import sys, torch +a, b, label = sys.argv[1], sys.argv[2], sys.argv[3] +ca, cb = (torch.load(p, map_location='cpu', weights_only=False) for p in (a, b)) +fams = {'tok': None, 'wout': None} +out = {} +ta, tb = ca['tok']['weight'].float(), cb['tok']['weight'].float() +out['tok'] = float((tb-ta).norm()/ta.norm()) +wa, wb = ca['wout'].float(), cb['wout'].float() +out['wout'] = float((wb-wa).norm()/wa.norm()) +Ba, Bb = ca['blocks'], cb['blocks'] +import re +groups = {'attn_bot': [], 'attn_top': [], 'ffn_bot': [], 'ffn_top': []} +for k in Ba: + m = re.match(r'(\d+)\.(attn|ff)\.', k) + if not m or Ba[k].dim() < 2: continue + l = int(m.group(1)); fam = ('attn' if m.group(2)=='attn' else 'ffn') + ('_bot' if l < 6 else '_top') + d = float((Bb[k].float()-Ba[k].float()).norm()/Ba[k].float().norm()) + groups[fam].append(d) +for k, v in groups.items(): out[k] = sum(v)/len(v) +print(label + ': ' + ' '.join(f'{k} {v:.5f}' for k, v in out.items())) -- cgit v1.2.3