From 77889401c95c75a591f69e51caf7617bc848f0cb Mon Sep 17 00:00:00 2001 From: Yuren Hao Date: Tue, 28 Jul 2026 09:27:18 -0500 Subject: =?UTF-8?q?RESULT=2064:=20=E9=9D=9Eplateau=E8=80=8C=E6=98=AF?= =?UTF-8?q?=E6=81=92=E9=80=9F=E6=B3=84=E6=BC=8F(=E5=AE=BD=E5=BA=A6=C3=974-?= =?UTF-8?q?5,=E9=80=80=E7=81=AB=E8=BF=BD=E4=B8=8D=E4=B8=8A);=20=E5=85=A8?= =?UTF-8?q?=E5=AB=8C=E7=96=91=E8=87=B3=E5=88=86=E8=BE=A8=E7=8E=87=E6=B4=97?= =?UTF-8?q?=E6=B8=85(bias=E2=89=A4sem,msign=E6=97=A0=E6=94=BE=E5=A4=A7,?= =?UTF-8?q?=CF=83=E4=B8=A4=E8=BE=B9=E5=90=8C=E4=B8=80/BP=E6=9B=B4=E5=93=8D?= =?UTF-8?q?);=2072M=E8=B4=A6=E6=9C=ACn=3D3=E5=90=88=E5=8D=B7(BP=203.2882?= =?UTF-8?q?=C2=B10.006,bsign=20gap+0.076);=20=E7=A3=81=E7=9B=98=E6=BB=A1?= =?UTF-8?q?=E4=BA=8B=E6=95=85+458GB=E4=BF=AE=E5=89=AA(=E6=B8=85=E5=8D=95?= =?UTF-8?q?=E8=90=BD=E6=A1=A3);=20amp=E9=99=8D=E7=BA=A7=E6=AC=A1=E8=A6=81(?= =?UTF-8?q?~12%);=20=E6=B3=84=E6=BC=8F=E7=8E=87=E9=85=8D=E5=AF=B9=E6=8E=A2?= =?UTF-8?q?=E9=92=88=E5=9C=A8=E9=A3=9E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014FAPDWQ49M5Ye3NpTndTpn --- ep_run/probe_msign.py | 173 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 173 insertions(+) create mode 100644 ep_run/probe_msign.py (limited to 'ep_run/probe_msign.py') diff --git a/ep_run/probe_msign.py b/ep_run/probe_msign.py new file mode 100644 index 0000000..e6e8824 --- /dev/null +++ b/ep_run/probe_msign.py @@ -0,0 +1,173 @@ +"""Suspect-2 probe (C768 plateau): does Muon msign AMPLIFY the tiny EP-vs-BP discrepancy? +Per weight matrix: cos(gEP,gBP) raw vs cos(msign(gEP),msign(gBP)). Derived from probe_bias. +Old docstring: Direct measurement of the systematic single-sided EP bias (the audit gap: BBP probes +measured NOISE thoroughly, discarded the mean/bias component unreported) + verification of the +user's claim: PER-SAMPLE random nudge sign makes the batch-averaged estimator unbiased at O(beta). +Three estimators at fixed ckpt, N batches, two betas: + plain : all samples nudged +beta -> bias = beta*E[B] (systematic, batch-indep) + persample : sample i nudged s_i*beta, read re-flipped by s_i -> odd-order bias cancels + WITHIN each batch (sqrt(B) suppression per step, 0 in expectation) + BP : exact reference. +Self-check: persample with all s=+1 must equal plain exactly. +Report per layer family: |mean(ghat-gbp)| / |mean(gbp)| (systematic), across-batch sem (noise +floor), for beta in {3e-3, 1e-2} — plain's bias should scale ~x3.3, persample's should sit at +the noise floor. Read-only, coexists on GPU0.""" +import argparse, pickle +import numpy as np, torch, torch.nn as nn, torch.nn.functional as F +from pathlib import Path + +ap = argparse.ArgumentParser() +ap.add_argument('--ckpt', default='runs/fw72m_plain2_s35000.pt') +ap.add_argument('--K', type=int, default=3) +ap.add_argument('--betas', default='3e-3,1e-2') +ap.add_argument('--nb', type=int, default=16) +a = ap.parse_args() +dev = 'cuda' +torch.manual_seed(11) +B, T = 8, 256 + +class RMSNorm(nn.Module): + def __init__(self, C, eps=1e-6): + super().__init__(); self.g = nn.Parameter(torch.ones(C)); self.eps = eps + def forward(self, x): + return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) * self.g + +class SwiGLU(nn.Module): + def __init__(self, C): + super().__init__() + h = ((8 * C // 3) + 63) // 64 * 64 + self.w1 = nn.Linear(C, h, bias=False); self.w3 = nn.Linear(C, h, bias=False) + self.w2 = nn.Linear(h, C, bias=False) + def forward(self, x): + return self.w2(F.silu(self.w1(x)) * self.w3(x)) + +class Olmo2Attn(nn.Module): + def __init__(self, C, H, T): + super().__init__() + self.H, self.hd = H, C // H + self.qkv = nn.Linear(C, 3 * C, bias=False); self.proj = nn.Linear(C, C, bias=False) + self.qn, self.kn = RMSNorm(C), RMSNorm(C) + inv = 1.0 / (500000.0 ** (torch.arange(0, self.hd, 2).float() / self.hd)) + fr = torch.outer(torch.arange(T).float(), inv) + self.register_buffer('rc', fr.cos(), persistent=False) + self.register_buffer('rs', fr.sin(), persistent=False) + def rope(self, x): + Tn = x.shape[2] + x1, x2 = x[..., ::2], x[..., 1::2] + c, s = self.rc[None, None, :Tn].to(x.dtype), self.rs[None, None, :Tn].to(x.dtype) + return torch.stack((x1 * c - x2 * s, x1 * s + x2 * c), dim=-1).flatten(-2) + def forward(self, x): + Bn, Tn, C = x.shape + q, k, v = self.qkv(x).split(C, dim=2) + q, k = self.qn(q), self.kn(k) + q = self.rope(q.view(Bn, Tn, self.H, self.hd).transpose(1, 2)) + k = self.rope(k.view(Bn, Tn, self.H, self.hd).transpose(1, 2)) + v = v.view(Bn, Tn, self.H, self.hd).transpose(1, 2) + y = F.scaled_dot_product_attention(q, k, v, is_causal=True) + return self.proj(y.transpose(1, 2).contiguous().view(Bn, Tn, C)) + +class Olmo2Block(nn.Module): + def __init__(self, C, H, T): + super().__init__() + self.attn = Olmo2Attn(C, H, T); self.ff = SwiGLU(C) + self.na, self.nf = RMSNorm(C), RMSNorm(C) + def forward(self, z): + z = z + self.na(self.attn(z)) + return z + self.nf(self.ff(z)) + +ck = torch.load(a.ckpt, map_location=dev, weights_only=False) +cfg = ck['config']; C, H, L = cfg['C'], cfg['H'], cfg['L'] +DD = Path('/home/yurenh2/ept/ep_run/data') / cfg.get('data', 'fineweb_edu') +vocab = pickle.load(open(DD / 'meta.pkl', 'rb'))['vocab_size'] +data = np.memmap(DD / 'val.bin', dtype=np.uint16, mode='r') +tok = nn.Embedding(vocab, C).to(dev); tok.load_state_dict(ck['tok']) +blocks = nn.ModuleList([Olmo2Block(C, H, T) for _ in range(L)]).to(dev) +blocks.load_state_dict(ck['blocks'], strict=False) +W_out = ck['wout'].to(dev); ln_f = RMSNorm(C).to(dev); ln_f.load_state_dict(ck['lnf']) +NBT = B * T +params = list(blocks.parameters()) +names = [n for n, _ in blocks.named_parameters()] + +def readout(z): return ln_f(z) @ W_out.t() + +def get_batch(g): + ix = torch.randint(len(data) - T - 1, (B,), generator=g) + x = torch.stack([torch.from_numpy(data[i:i + T].astype(np.int64)) for i in ix]) + y = torch.stack([torch.from_numpy(data[i + 1:i + 1 + T].astype(np.int64)) for i in ix]) + return x.to(dev), y.to(dev) + +def grads_from(outs_list, cots): + obj = sum((o * c.detach()).sum() for o, c in zip(outs_list, cots)) + gs = torch.autograd.grad(obj, params, allow_unused=True, retain_graph=True) + return [g.float() if g is not None else torch.zeros_like(p) for g, p in zip(gs, params)] + +def estimators(x, y, beta, signs): + """signs: (B,) tensor of +-1. Returns (g_bp, g_ep) where g_ep uses per-sample sign nudges.""" + z = tok(x) + zs_bp = [] + for b in blocks: + z = b(z); zs_bp.append(z) + ce = F.cross_entropy(readout(zs_bp[-1]).reshape(-1, vocab), y.reshape(-1)) + g_bp = [g.float() for g in torch.autograd.grad(ce, params, retain_graph=True)] + f_ins, f_outs = [], [] + prev = tok(x).detach() + for b in blocks: + i = prev.detach().requires_grad_(True) + o = b(i) + f_ins.append(i); f_outs.append(o) + prev = o.detach() + ins, outs = f_ins, f_outs + zs = [o.detach().float() for o in f_outs] + d = [None] * L + sB = signs.view(B, 1, 1).float() + for k in range(a.K): + zc = zs[L - 1].detach().requires_grad_(True) + # per-sample signed top objective: sum_i s_i * (sum_t loss_it) (token-sum, matches beta*NBT*mean scaling) + ce_tok = F.cross_entropy(readout(zc).reshape(-1, vocab), y.reshape(-1), reduction='none').view(B, T) + obj_top = (signs.float()[:, None] * ce_tok).sum() + d[L - 1] = (-beta * torch.autograd.grad(obj_top, zc)[0]).detach().float() + for l in range(L - 2, -1, -1): + d[l] = torch.autograd.grad(outs[l + 1], ins[l + 1], grad_outputs=d[l + 1], + retain_graph=True)[0].detach().float() + prev = tok(x).detach() + n_ins, n_outs = [], [] + for l in range(L): + i = prev.detach().requires_grad_(True) + o = blocks[l](i) + n_ins.append(i); n_outs.append(o) + zs[l] = o.detach().float() + d[l] + prev = zs[l] + ins, outs = n_ins, n_outs + md = [-(sB * di) / (beta * NBT) for di in d] # re-flip per sample; scale matches trainer + g_ep = grads_from(outs, md) + return g_bp, g_ep + + +from muon import newton_schulz +FAMS = {'qkv_bot': [f'{l}.attn.qkv.weight' for l in range(0,4)], + 'qkv_top': [f'{l}.attn.qkv.weight' for l in range(8,12)], + 'w2_bot': [f'{l}.ff.w2.weight' for l in range(0,4)], + 'w2_top': [f'{l}.ff.w2.weight' for l in range(8,12)], + 'proj_top':[f'{l}.attn.proj.weight' for l in range(8,12)]} +IDX = {k: [names.index(n) for n in v] for k, v in FAMS.items()} +g0 = torch.Generator().manual_seed(1234) +def cosf(a, b): + return float((a*b).sum() / (a.norm()*b.norm() + 1e-30)) +acc = {k: {'raw': [], 'ms': []} for k in IDX} +for bi in range(6): + x, y = get_batch(g0) + ones = torch.ones(B, dtype=torch.long, device=dev) + g_bp, g_ep = estimators(x, y, 3e-3, ones) + for k, idxs in IDX.items(): + for i in idxs: + acc[k]['raw'].append(cosf(g_ep[i], g_bp[i])) + me = newton_schulz(g_ep[i].bfloat16()).float() + mb = newton_schulz(g_bp[i].bfloat16()).float() + acc[k]['ms'].append(cosf(me, mb)) + del g_bp, g_ep + torch.cuda.empty_cache() +import numpy as np +print(f'{"family":>9} {"cos_raw":>9} {"cos_msign":>10} (mean over 6 batches x 4 mats)') +for k in IDX: + print(f'{k:>9} {np.mean(acc[k]["raw"]):>9.4f} {np.mean(acc[k]["ms"]):>10.4f}') +print('MSIGN_DONE', flush=True) -- cgit v1.2.3