From d0d1849e16127d4248bf5ef6c0bb1d6686aa9924 Mon Sep 17 00:00:00 2001 From: Yuren Hao Date: Wed, 5 Aug 2026 06:48:06 -0500 Subject: =?UTF-8?q?=E7=AC=AC=E4=BA=8C=E4=BB=A3=E5=8E=9F=E8=AF=AD=E4=BC=98?= =?UTF-8?q?=E5=8C=96=E5=99=A8:=20SignLine/ConsensusLion/DitherLion=20?= =?UTF-8?q?=E5=AE=9E=E8=A3=85+=E6=9C=AC=E5=9C=B0=E7=AD=9B=E9=80=89?= =?UTF-8?q?=E7=94=B5=E6=B1=A0=E5=8F=91=E5=B0=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit R4后设计原则=只追不翻腾的目标。SignLine核心洞察: 正行/列缩放在sign内是无操作, 作为脉冲幅度 在sign外才有效=因子化自适应的模拟原生形态; ConsensusLion=双漏率电容+符合门的噪声否决; DitherLion=免费噪声做期望软sign。KFAC-lite(追平稳的活动协方差)列为下一轮。 9臂与stage A同协议本地筛选中; 发射纪律教训追加: 发射与等待必须分Bash调用(超时杀树会扫掉setsid)。 Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014FAPDWQ49M5Ye3NpTndTpn --- ep_run/muon.py | 85 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 84 insertions(+), 1 deletion(-) (limited to 'ep_run/muon.py') diff --git a/ep_run/muon.py b/ep_run/muon.py index 1466ee6..e6c2692 100644 --- a/ep_run/muon.py +++ b/ep_run/muon.py @@ -173,7 +173,10 @@ def build_alt(opt_name, blocks, other_params, lr, warmup, total_steps=0, lr_min_ else: om = {'lion': lambda: Lion(mats, lr=lm, wd=wd), 'olion': lambda: OLion(mats, lr=lm, wd=wd), - 'adafactor': lambda: Adafactor2D(mats, lr=lm, wd=wd)}[opt_name]() + 'adafactor': lambda: Adafactor2D(mats, lr=lm, wd=wd), + 'signline': lambda: SignLine(mats, lr=lm, wd=wd), + 'conslion': lambda: ConsensusLion(mats, lr=lm, wd=wd), + 'ditherlion': lambda: DitherLion(mats, lr=lm, wd=wd)}[opt_name]() oa = torch.optim.AdamW(rest, lr=lr, weight_decay=1e-4) opts = [om, oa] if total_steps > 0: @@ -185,3 +188,83 @@ def build_alt(opt_name, blocks, other_params, lr, warmup, total_steps=0, lr_min_ fn = lambda s: min(1.0, (s + 1) / max(warmup, 1)) scheds = [torch.optim.lr_scheduler.LambdaLR(o, fn) for o in opts] return MultiOpt(opts), MultiSched(scheds) + + +class SignLine(torch.optim.Optimizer): + """sign(momentum) with per-row/per-column pulse amplitudes from leaky RMS line statistics. + Insight: positive row/col scaling INSIDE a sign is a no-op; applied OUTSIDE as amplitudes + (a_i b_j sign(m_ij)) it is the analog-native form of factored adaptivity: per-line DACs set + drive amplitude, comparators give the sign. Amplitudes normalized to unit mean so lr keeps + its scale; alpha in [0,1] interpolates flat-Lion (0) -> full line-adaptive (1).""" + def __init__(self, params, lr=3e-4, betas=(0.9, 0.99), beta_line=0.999, alpha=1.0, wd=0.0): + super().__init__(params, dict(lr=lr, betas=betas, bl=beta_line, alpha=alpha, wd=wd)) + + @torch.no_grad() + def step(self, closure=None): + for g_ in self.param_groups: + b1, b2 = g_['betas'] + for p in g_['params']: + if p.grad is None: continue + st = self.state[p] + if 'm' not in st: + st['m'] = torch.zeros_like(p) + st['r'] = torch.ones(p.shape[0], device=p.device) + st['c'] = torch.ones(p.shape[1], device=p.device) + m, r, c = st['m'], st['r'], st['c'] + u = b1 * m + (1 - b1) * p.grad + r.mul_(g_['bl']).add_(u.float().pow(2).mean(1), alpha=1 - g_['bl']) + c.mul_(g_['bl']).add_(u.float().pow(2).mean(0), alpha=1 - g_['bl']) + a = (r / r.mean()).clamp_min(1e-12).pow(-0.25 * g_['alpha']) + b = (c / c.mean()).clamp_min(1e-12).pow(-0.25 * g_['alpha']) + amp = torch.outer(a, b).to(p.dtype) + amp = amp / amp.mean() + if g_['wd'] > 0: p.mul_(1 - g_['lr'] * g_['wd']) + p.add_(u.sign_() * amp, alpha=-g_['lr']) + m.mul_(b2).add_(p.grad, alpha=1 - b2) + + +class ConsensusLion(torch.optim.Optimizer): + """two-timescale sign consensus: update only where sign(fast momentum) == sign(slow momentum). + Two leaky integrators with different leaks + one comparator + coincidence gate — the entire + optimizer is capacitors and logic. Where they disagree, write nothing (noise veto).""" + def __init__(self, params, lr=3e-4, beta_fast=0.9, beta_slow=0.99, wd=0.0): + super().__init__(params, dict(lr=lr, bf=beta_fast, bs=beta_slow, wd=wd)) + + @torch.no_grad() + def step(self, closure=None): + for g_ in self.param_groups: + for p in g_['params']: + if p.grad is None: continue + st = self.state[p] + if 'mf' not in st: + st['mf'] = torch.zeros_like(p); st['ms'] = torch.zeros_like(p) + mf, ms = st['mf'], st['ms'] + mf.mul_(g_['bf']).add_(p.grad, alpha=1 - g_['bf']) + ms.mul_(g_['bs']).add_(p.grad, alpha=1 - g_['bs']) + sf, ss = mf.sign(), ms.sign() + u = sf * (sf == ss) + if g_['wd'] > 0: p.mul_(1 - g_['lr'] * g_['wd']) + p.add_(u, alpha=-g_['lr']) + + +class DitherLion(torch.optim.Optimizer): + """Lion with dithered sign: sign(m + tau*noise*rms(m)). Free substrate noise turns the hard + sign into an unbiased soft-sign in expectation, letting small entries carry proportional + information across steps. tau=0 recovers Lion.""" + def __init__(self, params, lr=3e-4, betas=(0.9, 0.99), tau=0.5, wd=0.0): + super().__init__(params, dict(lr=lr, betas=betas, tau=tau, wd=wd)) + + @torch.no_grad() + def step(self, closure=None): + for g_ in self.param_groups: + b1, b2 = g_['betas'] + for p in g_['params']: + if p.grad is None: continue + st = self.state[p] + if 'm' not in st: st['m'] = torch.zeros_like(p) + m = st['m'] + u = b1 * m + (1 - b1) * p.grad + d = torch.randn_like(u) * (g_['tau'] * u.float().pow(2).mean().sqrt().to(u.dtype)) + if g_['wd'] > 0: p.mul_(1 - g_['lr'] * g_['wd']) + p.add_((u + d).sign_(), alpha=-g_['lr']) + m.mul_(b2).add_(p.grad, alpha=1 - b2) -- cgit v1.2.3