From 98d5ea09d1498f802f2ad06f5f67b385e228f063 Mon Sep 17 00:00:00 2001 From: Yuren Hao Date: Tue, 28 Jul 2026 13:20:34 -0500 Subject: =?UTF-8?q?RESULT=2066:=20head=E8=A2=AB=E8=8A=82=E6=B5=81=E5=88=B0?= =?UTF-8?q?60%(=E4=BD=8D=E7=A7=BB=E5=BE=AE=E6=8E=A2=E9=92=88:=E5=85=A8?= =?UTF-8?q?=E6=97=8F1.00-1.02=E5=94=AFW=5Fout=200.60)=20=E2=80=94=20?= =?UTF-8?q?=E7=94=A8=E6=88=B7=E7=AD=89=E4=BB=B7LR=E5=81=87=E8=AF=B4?= =?UTF-8?q?=E5=91=BD=E4=B8=AD=E5=B9=B6=E5=AE=9A=E4=BD=8D=E5=88=B0=E5=8D=95?= =?UTF-8?q?=E7=9F=A9=E9=98=B5;=20Adam=E5=B2=9Bm/=E2=88=9Av=E8=8A=82?= =?UTF-8?q?=E6=B5=81,=E5=AF=B9centered/fp32/K8=E5=85=A8=E4=B8=8D=E6=95=8F?= =?UTF-8?q?=E6=84=9F=3DEP=E5=A4=B4=E8=AF=BB=E5=85=B1=E4=BA=AB=E7=BB=93?= =?UTF-8?q?=E6=9E=84;=20--head=5Flr=5Fmult=E5=AE=9E=E8=A3=85;=20=E7=94=B5?= =?UTF-8?q?=E6=B1=A02(headmix=E8=AF=8A=E6=96=AD+hlr1.67/2.5=E8=A1=A5?= =?UTF-8?q?=E5=81=BF)=E5=9C=A8=E9=A3=9E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014FAPDWQ49M5Ye3NpTndTpn --- ep_run/casc_eq_train.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'ep_run/casc_eq_train.py') diff --git a/ep_run/casc_eq_train.py b/ep_run/casc_eq_train.py index 0cd5521..07e2ad4 100644 --- a/ep_run/casc_eq_train.py +++ b/ep_run/casc_eq_train.py @@ -49,6 +49,8 @@ ap.add_argument('--bsign_rand', action='store_true') # random-sign beta per ste ap.add_argument('--bf16', action='store_true') # cast model to bf16 (E-accumulation + tok_sigma stay fp32) — the x0.5 cost lever, GATE before production ap.add_argument('--amp', action='store_true') # PROPER mixed precision: autocast(bf16) matmuls, fp32 params/states/d/E — amp_gate.py PASSED 2026-07-12 (cos 0.9682 vs fp32 0.9687); --bf16 naive-cast stays DEAD (state quantization, RESULT 11) ap.add_argument('--dtop_every', type=int, default=1) # 1 = exact (DEFAULT, BP-parity); 2 = fast mode (~20% cheaper, ~4% CE tax at high lr) +ap.add_argument('--head_lr_mult', type=float, default=1.0) # W_out Adam-group LR multiplier (C768 + # head-throttle compensation, RESULT 66) ap.add_argument('--res_gate', type=float, default=0.02) # legality residual threshold; 0.02 was calibrated # at C512 — C768 ran half a schedule semi-converged # UNDER it (R59b). Per-width rule: ~100x the healthy @@ -313,7 +315,9 @@ if args.opt == 'muon': from muon import build_hybrid opt, sched = build_hybrid(blocks, all_params, args.lr, args.muon_lr, args.warmup, muon_mom=args.muon_mom, adam_b1=args.adam_b1, - total_steps=(args.steps if args.cosine else 0), lr_min_ratio=args.lr_min_ratio) + total_steps=(args.steps if args.cosine else 0), lr_min_ratio=args.lr_min_ratio, + head_param=(W_out if args.untie or args.olmo2 else None), + head_lr_mult=args.head_lr_mult) else: if args.wd >= 0: # OLMo2-style grouped decay: linear weights + head decay; embeddings/norm-gains none nodecay = {id(p) for p in tok.parameters()} | {id(p) for p in pos.parameters()} | \ -- cgit v1.2.3