diff options
| -rw-r--r-- | docs/hardware/EQOLION_DESIGN.md | 12 | ||||
| -rw-r--r-- | ep_run/casc_eq_train.py | 19 | ||||
| -rw-r--r-- | ep_run/muon.py | 26 |
3 files changed, 54 insertions, 3 deletions
diff --git a/docs/hardware/EQOLION_DESIGN.md b/docs/hardware/EQOLION_DESIGN.md index a44fccb..7be8a58 100644 --- a/docs/hardware/EQOLION_DESIGN.md +++ b/docs/hardware/EQOLION_DESIGN.md @@ -88,3 +88,15 @@ sign(M) vs sign(polar(M)) 一致率**中位 0.700（0.647-0.816）**，谱偏斜 | **DitherLion** | sign(m + τ·noise·rms(m)) | 基底噪声（免费）+ 比较器 | 硬 sign 变期望意义的软 sign，小分量跨步携带比例信息 | | KFAC-lite（下一轮） | u = m·Q，Q 用反 Hebbian 局部规则追踪**输入活动协方差**的白化 | 搭车探针 + 外积写 + 漏衰减 | **数据统计是平稳的**——R4 杀掉的是追极因子（高翻腾），活动协方差不翻腾，追踪前提在这里成立 | 测试协议 = 与 stage A 完全同网格（C128/2500 步/3 档 LR），本地双卡 ~45 分钟出结果，与农场标准组同表对比。 + +--- +## 偏置放大公理（2026-08-06，用户指出，账已算） +**逐坐标归一化在持久偏置下把小偏置放大成全速漂移**：真梯度≈0 而偏置 b 持久的坐标，Adam 的 +v→b² ⟹ 更新→lr·sign(b) 满幅；SGD 只走 lr·b。sign 全家同罪（Lion/OLion/ConsensusLion—— +持久偏置下快慢 EMA 同号，否决门大开；**否决只挡方差不挡偏置**）。DitherLion 例外候选：软化 +sign 在噪声尺度下恢复比例响应，小偏置不被推到满幅。 +**曲率通道 = 偏置磁铁**：β 无关加性读出偏移 δ 经 1/β_t 归一化后随硬币翻号 ⟹ 偶通道(梯度)被 +bsign 清除，**奇通道 E[s·ĝ] = β·c + δ/β**——污染不翻号全额沉积且被 1/β 放大(333×@3e-3)。 +存活路径=锁相分离：c∝β 而污染∝1/β，两档|β|×随机符号四点调制按档积分可解出双通道，零额外相位。 +**评分表新增第三轴：偏置放大系数**（质量/硬件成本之外）。器件电池预注册预测：持久偏置下排序 +重排——per-matrix(Muon/Hyperball)优雅退化, per-coordinate(Adam/sign族)满幅漂移, Dither 翻身。 diff --git a/ep_run/casc_eq_train.py b/ep_run/casc_eq_train.py index 6a6865d..e29048c 100644 --- a/ep_run/casc_eq_train.py +++ b/ep_run/casc_eq_train.py @@ -22,7 +22,7 @@ ap.add_argument('--wandb_run', default='') ap.add_argument('--kmax', type=int, default=8) # adaptive fb rounds cap ap.add_argument('--noguard', action='store_true') # diagnosis: skip only non-finite grads ap.add_argument('--untie', action='store_true') # separate readout matrix (untied from tok) -ap.add_argument('--opt', choices=['adamw', 'muon', 'sgdm', 'lion', 'olion', 'adafactor', 'signline', 'conslion', 'ditherlion'], default='adamw') +ap.add_argument('--opt', choices=['adamw', 'muon', 'sgdm', 'lion', 'olion', 'adafactor', 'signline', 'conslion', 'ditherlion', 'cautlion'], default='adamw') ap.add_argument('--muon_lr', type=float, default=0.02) ap.add_argument('--tok_init', type=float, default=0.0) # >0: init tok/pos with this std (GPT-standard 0.02) ap.add_argument('--compile', action='store_true') # torch.compile each block (free speed where supported) @@ -61,6 +61,7 @@ ap.add_argument('--gen_new', type=int, default=120) ap.add_argument('--probe_dgspec', type=int, default=0) # >0: M1 spectroscopy, value = n batches; exits before training ap.add_argument('--probe_gains', default='1,2,4,8,16,32,64,128,256') ap.add_argument('--probe_f64', action='store_true') # fp64 states+model in the probe: the fp-floor decisive arm # >1: per-STEP log-uniform dgain_top in +ap.add_argument('--dump_grad', type=int, default=0) # >0: record per-step grad estimates + beta sign ap.add_argument('--dump_mom', type=int, default=0) # >0: record 3 Muon momentum matrices every step # for N steps -> runs/momdump_{tag}.pt, then exit # (EqOLion tracking-falsification input) @@ -344,7 +345,7 @@ if args.bf16: if args.untie: with torch.no_grad(): W_out.data = W_out.data.to(torch.bfloat16) print('[bf16] model cast to bfloat16 (E-accum + sigma stay fp32)', flush=True) -if args.opt in ('sgdm', 'lion', 'olion', 'adafactor', 'signline', 'conslion', 'ditherlion'): +if args.opt in ('sgdm', 'lion', 'olion', 'adafactor', 'signline', 'conslion', 'ditherlion', 'cautlion'): from muon import build_alt opt, sched = build_alt(args.opt, blocks, all_params, args.lr, args.warmup, total_steps=(args.steps if args.cosine else 0), lr_min_ratio=args.lr_min_ratio, @@ -1050,6 +1051,20 @@ for step in range(start_step, args.steps + 1): with torch.no_grad(): WSNAP['p'] = [p.detach().clone() for p in all_params] WSNAP['o'] = _clone_state(opt.state_dict()) + if args.dump_grad > 0 and args.opt == 'muon': + # per-step gradient ESTIMATES + the bsign coin, for the odd/even (curvature-channel) + # decomposition test: E[s*ghat] = beta*c. Captured BEFORE opt.step consumes grads. + om_ = opt.optimizers[0] + _m2 = om_.param_groups[0]['params'] + _s2 = [_m2[1], _m2[len(_m2) // 2], _m2[-1]] + if 'GDUMP' not in globals(): + GDUMP = {'shapes': [tuple(p.shape) for p in _s2], 'sign': [], 'traj': []} + GDUMP['sign'].append(1.0 if beta_t > 0 else -1.0) + GDUMP['traj'].append([p.grad.detach().float().cpu().clone() for p in _s2]) + if step >= args.dump_grad: + torch.save(GDUMP, f'runs/graddump_{args.tag}.pt') + print(f'[graddump] DONE {len(GDUMP["traj"])} steps -> runs/graddump_{args.tag}.pt', flush=True) + import sys; sys.exit(0) opt.step(); sched.step(); opt.zero_grad(set_to_none=True) if args.dump_mom > 0 and args.opt == 'muon': # per-step momentum trajectory for the EqOLion tracking falsification: three diff --git a/ep_run/muon.py b/ep_run/muon.py index e6c2692..1289620 100644 --- a/ep_run/muon.py +++ b/ep_run/muon.py @@ -176,7 +176,8 @@ def build_alt(opt_name, blocks, other_params, lr, warmup, total_steps=0, lr_min_ 'adafactor': lambda: Adafactor2D(mats, lr=lm, wd=wd), 'signline': lambda: SignLine(mats, lr=lm, wd=wd), 'conslion': lambda: ConsensusLion(mats, lr=lm, wd=wd), - 'ditherlion': lambda: DitherLion(mats, lr=lm, wd=wd)}[opt_name]() + 'ditherlion': lambda: DitherLion(mats, lr=lm, wd=wd), + 'cautlion': lambda: CautiousLion(mats, lr=lm, wd=wd)}[opt_name]() oa = torch.optim.AdamW(rest, lr=lr, weight_decay=1e-4) opts = [om, oa] if total_steps > 0: @@ -247,6 +248,29 @@ class ConsensusLion(torch.optim.Optimizer): p.add_(u, alpha=-g_['lr']) +class CautiousLion(torch.optim.Optimizer): + """C-Lion (Liang et al., arXiv:2411.16085, ICLR'26): Lion masked where the update sign + disagrees with the CURRENT gradient sign. The mandatory prior-art baseline for ConsensusLion; + the difference under test is instantaneous-gradient gating (this) vs filtered two-EMA gating.""" + def __init__(self, params, lr=3e-4, betas=(0.9, 0.99), wd=0.0): + super().__init__(params, dict(lr=lr, betas=betas, wd=wd)) + + @torch.no_grad() + def step(self, closure=None): + for g_ in self.param_groups: + b1, b2 = g_['betas'] + for p in g_['params']: + if p.grad is None: continue + st = self.state[p] + if 'm' not in st: st['m'] = torch.zeros_like(p) + m = st['m'] + u = (b1 * m + (1 - b1) * p.grad).sign_() + u = u * ((u * p.grad) > 0) + if g_['wd'] > 0: p.mul_(1 - g_['lr'] * g_['wd']) + p.add_(u, alpha=-g_['lr']) + m.mul_(b2).add_(p.grad, alpha=1 - b2) + + class DitherLion(torch.optim.Optimizer): """Lion with dithered sign: sign(m + tau*noise*rms(m)). Free substrate noise turns the hard sign into an unbiased soft-sign in expectation, letting small entries carry proportional |