From 807c6ba3bab8d9afe18ca7d8fb67038bc2cfaf73 Mon Sep 17 00:00:00 2001 From: Yuren Hao Date: Wed, 5 Aug 2026 09:09:14 -0500 Subject: =?UTF-8?q?RESULT=2090:=20=E4=BC=98=E5=8C=96=E5=99=A8=E7=AD=9B?= =?UTF-8?q?=E9=80=89=E6=94=B6=E6=95=9B=20=E2=80=94=20Muon=204.7227=20?= =?UTF-8?q?=E9=A2=86=E8=B7=91,=20NS=E6=B6=88=E8=9E=8D=E5=88=A4=E5=86=B3(?= =?UTF-8?q?=E6=91=98NS=E4=BB=A3=E4=BB=B70.45,=20nanoGPT=E7=89=88=E6=9C=AA?= =?UTF-8?q?=E8=B0=83=E7=94=A8NS=E4=B9=8B=E8=B0=9C=E5=BD=92=E4=BB=96?= =?UTF-8?q?=E4=BB=AC),=20sign=E5=AE=B6=E6=97=8F=E8=81=9A=E5=9C=A85.13-5.25?= =?UTF-8?q?;=20k=E4=BB=B7=E5=80=BC=E6=9B=B2=E7=BA=BF=E5=9C=A8=E8=B7=91;=20?= =?UTF-8?q?=E6=9B=B2=E7=8E=87=E9=80=9A=E9=81=93=E6=AD=BB=E5=88=91+?= =?UTF-8?q?=E5=81=8F=E7=BD=AE=E5=85=AC=E7=90=86=E5=85=A5=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014FAPDWQ49M5Ye3NpTndTpn --- docs/campaign/CASCADE_ABLATION_PLAN.md | 20 ++++++++++++++++++++ ep_run/casc_eq_train.py | 4 ++-- ep_run/muon.py | 6 +++++- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/docs/campaign/CASCADE_ABLATION_PLAN.md b/docs/campaign/CASCADE_ABLATION_PLAN.md index 2298ab4..142d5ed 100644 --- a/docs/campaign/CASCADE_ABLATION_PLAN.md +++ b/docs/campaign/CASCADE_ABLATION_PLAN.md @@ -2177,3 +2177,23 @@ BP seed2 在跑(定散布尺度)。 - 旧读出终值 +45.2% ppl = 精度伪影在 135M 的完整体量。 - 交付物已出终版: assets/figs/fig_135m_curves.{png,pdf}(四曲线, 横轴 token) + assets/EPT_135M_samples.ipynb(EP/BP 同 prompt 同种子并排, 输出已烤入, 代码可跑)。 + +## RESULT 90 (2026-08-06): 优化器筛选定案(进行中收敛) + NS 消融判决 +协议: C128/11M, fp32, 2500 步, 各优化器 LR 扫到最优被包住(muon 尚差一档), seed 1, read_lin+bsign。 +| 优化器 | 最优 LR | best val | 家族 | +|---|---|---|---| +| Muon | 8e-3(边缘,16e-3 在跑) | **4.7227** | 谱归一 | +| OLion(NS+sign) | 1e-2(包住) | 4.7542 | 谱归一+sign | +| **OLion 无 NS** | 1e-2 | **5.2045** | ← **摘 NS 代价 0.45** | +| C-Lion / ConsensusLion | 3e-3 | 5.1270 / 5.1297 | 干净梯度下打平(判决树"折入已知"分支) | +| SignLine / Lion / DitherLion | 3e-3 | 5.20 / 5.20 / 5.25 | sign 家族 | +| SGDM | 0.3 | 5.3499 | | +| AdamW / Adafactor | farm2 重跑中 | | | +- **NS 消融**: 官方仓库 nanoGPT 版(跑 GPT-2 实验的那版)定义 NS 但从未调用; 我方同字节消融实测 + 摘 NS 代价 0.45 CE ⟹ 在 EP 设置下 sign-after-orthogonalization ≫ sign-of-momentum, 正交化是 + 承重墙, "冠军天然全模拟原生"证伪。 +- **离线 k 曲线**: sign(NS_k) vs sign(NS_5) 一致率 k=0/1/2/3 = 0.72/0.72/0.78/0.86 — sign 不提前 + 收敛; 质量-k 曲线(optb5, k=1/2/3 训练臂)= 模拟正交化器的精度规格表, 在跑。 +- 事故记录: NFS 同名 tag 跨机互毁(→机器前缀铁律入 fleet 记忆); 发射+等待同调用被超时杀树(→分调用铁律)。 +- 曲率通道(EP 原生 Adam)已判死: E[s·ĝ] 奇通道 z=-0.2 埋于噪声, 需 4 万步 EMA 无法当活预条件器; + 偏置公理(用户): 逐坐标归一化把持久偏置放大成满幅漂移, sign 全家同罪, 评分表增第三轴。 diff --git a/ep_run/casc_eq_train.py b/ep_run/casc_eq_train.py index e29048c..d19504d 100644 --- a/ep_run/casc_eq_train.py +++ b/ep_run/casc_eq_train.py @@ -22,7 +22,7 @@ ap.add_argument('--wandb_run', default='') ap.add_argument('--kmax', type=int, default=8) # adaptive fb rounds cap ap.add_argument('--noguard', action='store_true') # diagnosis: skip only non-finite grads ap.add_argument('--untie', action='store_true') # separate readout matrix (untied from tok) -ap.add_argument('--opt', choices=['adamw', 'muon', 'sgdm', 'lion', 'olion', 'adafactor', 'signline', 'conslion', 'ditherlion', 'cautlion'], default='adamw') +ap.add_argument('--opt', choices=['adamw', 'muon', 'sgdm', 'lion', 'olion', 'adafactor', 'signline', 'conslion', 'ditherlion', 'cautlion', 'olionns', 'olionk1', 'olionk2', 'olionk3'], default='adamw') ap.add_argument('--muon_lr', type=float, default=0.02) ap.add_argument('--tok_init', type=float, default=0.0) # >0: init tok/pos with this std (GPT-standard 0.02) ap.add_argument('--compile', action='store_true') # torch.compile each block (free speed where supported) @@ -345,7 +345,7 @@ if args.bf16: if args.untie: with torch.no_grad(): W_out.data = W_out.data.to(torch.bfloat16) print('[bf16] model cast to bfloat16 (E-accum + sigma stay fp32)', flush=True) -if args.opt in ('sgdm', 'lion', 'olion', 'adafactor', 'signline', 'conslion', 'ditherlion', 'cautlion'): +if args.opt in ('sgdm', 'lion', 'olion', 'adafactor', 'signline', 'conslion', 'ditherlion', 'cautlion', 'olionns', 'olionk1', 'olionk2', 'olionk3'): from muon import build_alt opt, sched = build_alt(args.opt, blocks, all_params, args.lr, args.warmup, total_steps=(args.steps if args.cosine else 0), lr_min_ratio=args.lr_min_ratio, diff --git a/ep_run/muon.py b/ep_run/muon.py index 1289620..7bad679 100644 --- a/ep_run/muon.py +++ b/ep_run/muon.py @@ -177,7 +177,11 @@ def build_alt(opt_name, blocks, other_params, lr, warmup, total_steps=0, lr_min_ 'signline': lambda: SignLine(mats, lr=lm, wd=wd), 'conslion': lambda: ConsensusLion(mats, lr=lm, wd=wd), 'ditherlion': lambda: DitherLion(mats, lr=lm, wd=wd), - 'cautlion': lambda: CautiousLion(mats, lr=lm, wd=wd)}[opt_name]() + 'cautlion': lambda: CautiousLion(mats, lr=lm, wd=wd), + 'olionns': lambda: OLion(mats, lr=lm, wd=wd, ns_steps=0), + 'olionk1': lambda: OLion(mats, lr=lm, wd=wd, ns_steps=1), + 'olionk2': lambda: OLion(mats, lr=lm, wd=wd, ns_steps=2), + 'olionk3': lambda: OLion(mats, lr=lm, wd=wd, ns_steps=3)}[opt_name]() oa = torch.optim.AdamW(rest, lr=lr, weight_decay=1e-4) opts = [om, oa] if total_steps > 0: -- cgit v1.2.3