RESULT 49: 回路解剖=23段乘积近临界(下行×72/上行×13k/顶读1e-6),plain只热b8一段(h6 logit 94 vs cent 75)边际越线,灾变=硬底响应打伤b1(入口vjp 1.3→14.9); batch噪声对照0.39/0.45=Q1封口; --logit_knee 80实装+knee臂排c190后

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014FAPDWQ49M5Ye3NpTndTpn

1 files changed, 198 insertions, 0 deletions

diff --git a/ep_run/probe_cycle.py b/ep_run/probe_cycle.py
new file mode 100644
index 0000000..fb06069
--- /dev/null
+++ b/ep_run/probe_cycle.py
@@ -0,0 +1,198 @@
+"""Cycle-gain decomposition of the nudge loop at the DOMINANT MODE (user directive: measure
+the explosion's cause). Run the trainer-faithful nudged relax, let the residual converge to the
+dominant mode, then difference consecutive sweeps to get per-stage amplification factors ALONG
+THE ACTUAL MODE (norms audits failed; alignment is everything):
+  gH        = |dd_11| / |dz_11|          top stage: beta*NBT*CE-curvature read
+  gV(l)     = |dd_l| / |dd_{l+1}|        down-chain vjp through block l+1
+  gF(l)     = |do_l| / |dz_{l-1}|        up-chain rebuild through block l
+  closure:  gH * prod(gV) ~ dd-chain, rebuild adds dd_l -> next dz; rho_meas from residuals.
+Plus per-head carrier analysis for blocks 8-11 (share of the mode through each head, its
+max|logit| and attention entropy) -> names the physical carrier (switching-regime heads?).
+Control column for the batch-noise question: per-batch BP u1 vs 8-batch-mean BP u1 overlap."""
+import argparse, pickle
+import numpy as np, torch, torch.nn as nn, torch.nn.functional as F
+from pathlib import Path
+
+ap = argparse.ArgumentParser()
+ap.add_argument('--ckpts', default='plain:195000,cent:195000,plain:210000')
+ap.add_argument('--K', type=int, default=16)
+ap.add_argument('--beta', type=float, default=3e-3)
+a = ap.parse_args()
+dev = 'cuda'
+torch.manual_seed(7)
+B, T = 8, 256
+
+class RMSNorm(nn.Module):
+    def __init__(self, C, eps=1e-6):
+        super().__init__(); self.g = nn.Parameter(torch.ones(C)); self.eps = eps
+    def forward(self, x):
+        return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) * self.g
+
+class SwiGLU(nn.Module):
+    def __init__(self, C):
+        super().__init__()
+        h = ((8 * C // 3) + 63) // 64 * 64
+        self.w1 = nn.Linear(C, h, bias=False); self.w3 = nn.Linear(C, h, bias=False)
+        self.w2 = nn.Linear(h, C, bias=False)
+    def forward(self, x):
+        return self.w2(F.silu(self.w1(x)) * self.w3(x))
+
+class Olmo2Attn(nn.Module):
+    def __init__(self, C, H, T):
+        super().__init__()
+        self.H, self.hd = H, C // H
+        self.qkv = nn.Linear(C, 3 * C, bias=False); self.proj = nn.Linear(C, C, bias=False)
+        self.qn, self.kn = RMSNorm(C), RMSNorm(C)
+        inv = 1.0 / (500000.0 ** (torch.arange(0, self.hd, 2).float() / self.hd))
+        fr = torch.outer(torch.arange(T).float(), inv)
+        self.register_buffer('rc', fr.cos(), persistent=False)
+        self.register_buffer('rs', fr.sin(), persistent=False)
+    def rope(self, x):
+        Tn = x.shape[2]
+        x1, x2 = x[..., ::2], x[..., 1::2]
+        c, s = self.rc[None, None, :Tn].to(x.dtype), self.rs[None, None, :Tn].to(x.dtype)
+        return torch.stack((x1 * c - x2 * s, x1 * s + x2 * c), dim=-1).flatten(-2)
+    def heads(self, x):
+        """per-head attention outputs (pre-proj) + logit stats + entropy"""
+        Bn, Tn, C = x.shape
+        q, k, v = self.qkv(x).split(C, dim=2)
+        q, k = self.qn(q), self.kn(k)
+        q = self.rope(q.view(Bn, Tn, self.H, self.hd).transpose(1, 2))
+        k = self.rope(k.view(Bn, Tn, self.H, self.hd).transpose(1, 2))
+        v = v.view(Bn, Tn, self.H, self.hd).transpose(1, 2)
+        lg = (q @ k.transpose(-2, -1)) / (self.hd ** 0.5)
+        mask = torch.ones(Tn, Tn, dtype=torch.bool, device=x.device).tril()
+        lgm = lg.masked_fill(~mask, float('-inf'))
+        p = lgm.softmax(-1)
+        y = p @ v                                    # (B,H,T,hd)
+        ent = -(p.clamp_min(1e-12).log() * p).sum(-1).mean(dim=(0, 2))    # (H,)
+        mx = lg.masked_fill(~mask, 0).abs().amax(dim=(0, 2, 3))           # (H,)
+        return y, mx, ent
+    def forward(self, x):
+        Bn, Tn, C = x.shape
+        q, k, v = self.qkv(x).split(C, dim=2)
+        q, k = self.qn(q), self.kn(k)
+        q = self.rope(q.view(Bn, Tn, self.H, self.hd).transpose(1, 2))
+        k = self.rope(k.view(Bn, Tn, self.H, self.hd).transpose(1, 2))
+        v = v.view(Bn, Tn, self.H, self.hd).transpose(1, 2)
+        y = F.scaled_dot_product_attention(q, k, v, is_causal=True)
+        return self.proj(y.transpose(1, 2).contiguous().view(Bn, Tn, C))
+
+class Olmo2Block(nn.Module):
+    def __init__(self, C, H, T):
+        super().__init__()
+        self.attn = Olmo2Attn(C, H, T); self.ff = SwiGLU(C)
+        self.na, self.nf = RMSNorm(C), RMSNorm(C)
+    def forward(self, z):
+        z = z + self.na(self.attn(z))
+        return z + self.nf(self.ff(z))
+
+first = True
+for spec in a.ckpts.split(','):
+    lineage, step = spec.split(':'); step = int(step)
+    ck = torch.load(f'runs/fw72m_{lineage}_s{step}.pt', map_location='cpu', weights_only=False)
+    cfg = ck['config']; C, H, L = cfg['C'], cfg['H'], cfg['L']
+    if first:
+        DD = Path('/home/yurenh2/ept/ep_run/data') / cfg.get('data', 'fineweb_edu')
+        vocab = pickle.load(open(DD / 'meta.pkl', 'rb'))['vocab_size']
+        data = np.memmap(DD / 'val.bin', dtype=np.uint16, mode='r')
+        ix = torch.randint(len(data) - T - 1, (B,))
+        x = torch.stack([torch.from_numpy(data[i:i + T].astype(np.int64)) for i in ix]).to(dev)
+        y = torch.stack([torch.from_numpy(data[i + 1:i + 1 + T].astype(np.int64)) for i in ix]).to(dev)
+        first = False
+    tok = nn.Embedding(vocab, C).to(dev); tok.load_state_dict(ck['tok'])
+    blocks = nn.ModuleList([Olmo2Block(C, H, T) for _ in range(L)]).to(dev)
+    blocks.load_state_dict(ck['blocks'], strict=False)
+    W_out = ck['wout'].to(dev)
+    ln_f = RMSNorm(C).to(dev); ln_f.load_state_dict(ck['lnf'])
+    NBT = B * T
+
+    def readout(z): return ln_f(z) @ W_out.t()
+
+    f_ins, f_outs = [], []
+    prev = tok(x).detach()
+    for b in blocks:
+        i = prev.detach().requires_grad_(True)
+        o = b(i)
+        f_ins.append(i); f_outs.append(o)
+        prev = o.detach()
+    ins, outs = f_ins, f_outs
+    zs = [o.detach().float() for o in f_outs]
+    d = [None] * L
+    hist = []                                  # (zs_copy, d_copy, os_copy) per sweep
+    res_seq = []
+    for k in range(a.K):
+        zc = zs[L - 1].detach().requires_grad_(True)
+        ce_k = F.cross_entropy(readout(zc).reshape(-1, vocab), y.reshape(-1))
+        d[L - 1] = (-a.beta * NBT * torch.autograd.grad(ce_k, zc)[0]).detach().float()
+        for l in range(L - 2, -1, -1):
+            d[l] = torch.autograd.grad(outs[l + 1], ins[l + 1], grad_outputs=d[l + 1],
+                                       retain_graph=True)[0].detach().float()
+        prev = tok(x).detach()
+        n_ins, n_outs, os_now, rnum, rden = [], [], [], 0.0, 0.0
+        for l in range(L):
+            i = prev.detach().requires_grad_(True)
+            o = blocks[l](i)
+            n_ins.append(i); n_outs.append(o)
+            znew = o.detach().float() + d[l]
+            os_now.append(o.detach().float())
+            rnum += float((znew - zs[l]).norm()); rden += float(zs[l].norm())
+            zs[l] = znew
+            prev = zs[l]
+        ins, outs = n_ins, n_outs
+        res_seq.append(rnum / max(rden, 1e-9))
+        hist.append(([z.clone() for z in zs], [di.clone() for di in d], os_now))
+    rho = res_seq[-1] / max(res_seq[-2], 1e-12)
+    zA, dA, oA = hist[-2]; zB, dB, oB = hist[-1]
+    dz = [zB[l] - zA[l] for l in range(L)]
+    dd = [dB[l] - dA[l] for l in range(L)]
+    do = [oB[l] - oA[l] for l in range(L)]
+    gH = float(dd[L - 1].norm() / max(dz[L - 1].norm(), 1e-12))
+    gV = [float(dd[l].norm() / max(dd[l + 1].norm(), 1e-12)) for l in range(L - 1)]
+    gF = [float(do[l].norm() / max(dz[l - 1].norm(), 1e-12)) for l in range(1, L)]
+    print(f'{lineage} s{step//1000}k | rho {rho:.4f} | gH(top read) {gH:.4f}', flush=True)
+    print('  gV vjp  l<-l+1 (11<-top .. 0<-1): ' + ' '.join(f'{v:.2f}' for v in reversed(gV)), flush=True)
+    print('  gF fwd  l-1->l (1 .. 11):         ' + ' '.join(f'{v:.2f}' for v in gF), flush=True)
+    # per-head carriers, blocks 8-11: mode share through each head + logit/entropy state
+    for l in range(8, 12):
+        at = blocks[l].attn
+        xin = zA[l - 1]
+        with torch.no_grad():
+            y0, mx, ent = at.heads(xin)
+            y1, _, _ = at.heads(xin + dz[l - 1])
+            share = torch.linalg.vector_norm(y1 - y0, dim=(0, 2, 3))
+            share = (share / share.sum()).cpu().numpy()
+        order = np.argsort(-share)[:3]
+        cells = ' '.join(f'h{h}:share {share[h]:.2f} maxlg {float(mx[h]):.0f} ent {float(ent[h]):.2f}'
+                         for h in order)
+        print(f'  blk{l} carriers: {cells}', flush=True)
+    del tok, blocks, W_out, ln_f, ins, outs, f_ins, f_outs, zs, d, hist
+    torch.cuda.empty_cache()
+
+# batch-noise control (the "shared with BP" question): per-batch BP u1 vs mean-BP u1
+ck = torch.load('runs/fw72m_plain_s195000.pt', map_location='cpu', weights_only=False)
+cfg = ck['config']; C, H, L = cfg['C'], cfg['H'], cfg['L']
+tok = nn.Embedding(vocab, C).to(dev); tok.load_state_dict(ck['tok'])
+blocks = nn.ModuleList([Olmo2Block(C, H, T) for _ in range(L)]).to(dev)
+blocks.load_state_dict(ck['blocks'], strict=False)
+W_out = ck['wout'].to(dev)
+ln_f = RMSNorm(C).to(dev); ln_f.load_state_dict(ck['lnf'])
+params = list(blocks.parameters())
+names = [n for n, _ in blocks.named_parameters()]
+SEL = [i for i, n in enumerate(names) if n in ('0.attn.qkv.weight', '8.attn.qkv.weight')]
+gs = {i: [] for i in SEL}
+for _ in range(8):
+    ixb = torch.randint(len(data) - T - 1, (B,))
+    xb = torch.stack([torch.from_numpy(data[i:i + T].astype(np.int64)) for i in ixb]).to(dev)
+    yb = torch.stack([torch.from_numpy(data[i + 1:i + 1 + T].astype(np.int64)) for i in ixb]).to(dev)
+    z = tok(xb)
+    for b in blocks: z = b(z)
+    ce = F.cross_entropy((ln_f(z) @ W_out.t()).reshape(-1, vocab), yb.reshape(-1))
+    g = torch.autograd.grad(ce, params, allow_unused=True)
+    for i in SEL: gs[i].append(g[i].detach().cpu())
+for i in SEL:
+    gbar = torch.stack(gs[i]).mean(0)
+    u = torch.linalg.svd(gbar, full_matrices=False).U[:, 0]
+    ovs = [abs(float(u @ torch.linalg.svd(g, full_matrices=False).U[:, 0])) for g in gs[i]]
+    print(f'BP-batch-noise control {names[i]}: per-batch u1 vs mean u1 overlap = {np.mean(ovs):.3f}', flush=True)
+print('CYCLE_DONE', flush=True)