From d0268c2f2ba7a1ceb5125f19b5f3b7dd61eb99eb Mon Sep 17 00:00:00 2001 From: Yuren Hao Date: Wed, 15 Jul 2026 04:27:01 -0500 Subject: =?UTF-8?q?RESULT=2025:=20bias=20decomposition=20probe=20=E2=80=94?= =?UTF-8?q?=20100%=20between-block=20transmission=20(TRANS=3DEP=200.9987,?= =?UTF-8?q?=20ANCHOR=3D1.0000,=20K1=3DBP=20exactly,=20K3=3DK8=20saturated)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014FAPDWQ49M5Ye3NpTndTpn --- ep_run/probe_blockcos.py | 190 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 190 insertions(+) create mode 100644 ep_run/probe_blockcos.py (limited to 'ep_run') diff --git a/ep_run/probe_blockcos.py b/ep_run/probe_blockcos.py new file mode 100644 index 0000000..b55107a --- /dev/null +++ b/ep_run/probe_blockcos.py @@ -0,0 +1,190 @@ +"""Error-source decomposition for cascade-EP: WITHIN-block vs BETWEEN-block. + +Key code fact: the theta-read is exact autograd of the block-local energy, so ALL +finite-beta error enters through STATE DISPLACEMENT. That factorizes exactly: + grad(anchor, cot) = sum_l + (free, c) = BP (exact cotangents at free states) + (nudged, d) = EP (EP-transmitted d at beta-displaced anchors) + (free, d) = TRANS-only -> isolates BETWEEN-block transmission error (d quality) + (nudged, c) = ANCHOR-only -> isolates WITHIN-block anchor displacement error +cos(each, BP) tells which factor dominates. Plus per-block cos-by-depth profile: +transmission compounding must show as decay toward block 0 (furthest from loss). +Self-check: (free, c) rebuilt through the same code path must give cos==1 vs BP. +Usage: probe_blockcos.py [--ckpt runs/stage1b_ep_muon_s45000.pt] [--K 3] [--nb 4] +""" +import argparse, pickle +import numpy as np, torch, torch.nn as nn, torch.nn.functional as F +from pathlib import Path + +ap = argparse.ArgumentParser() +ap.add_argument('--ckpt', default='runs/stage1b_ep_muon_s45000.pt') +ap.add_argument('--K', type=int, default=3) +ap.add_argument('--nb', type=int, default=4) +ap.add_argument('--betas', default='1e-3,3e-3') +a = ap.parse_args() +dev = 'cuda' +torch.manual_seed(7) + +ck = torch.load(a.ckpt, map_location='cpu', weights_only=False) +DD = Path('/home/yurenh2/ept/ep_run/data') / ck.get('config', {}).get('data', 'tinystories_bpe') +vocab = pickle.load(open(DD / 'meta.pkl', 'rb'))['vocab_size'] +B, T = 8, 256 + +def get_batch(): + data = np.memmap(DD / 'val.bin', dtype=np.uint16, mode='r') + ix = torch.randint(len(data) - T - 1, (B,)) + x = torch.stack([torch.from_numpy(data[i:i + T].astype(np.int64)) for i in ix]) + y = torch.stack([torch.from_numpy(data[i + 1:i + 1 + T].astype(np.int64)) for i in ix]) + return x.to(dev), y.to(dev) + +class RMSNorm(nn.Module): + def __init__(self, C, eps=1e-6): + super().__init__(); self.g = nn.Parameter(torch.ones(C)); self.eps = eps + def forward(self, x): + return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) * self.g + +class SwiGLU(nn.Module): + def __init__(self, C): + super().__init__() + h = ((8 * C // 3) + 63) // 64 * 64 + self.w1 = nn.Linear(C, h, bias=False); self.w3 = nn.Linear(C, h, bias=False) + self.w2 = nn.Linear(h, C, bias=False) + def forward(self, x): + return self.w2(F.silu(self.w1(x)) * self.w3(x)) + +class Olmo2Attn(nn.Module): + def __init__(self, C, H, T): + super().__init__() + self.H, self.hd = H, C // H + self.qkv = nn.Linear(C, 3 * C, bias=False); self.proj = nn.Linear(C, C, bias=False) + self.qn, self.kn = RMSNorm(C), RMSNorm(C) + inv = 1.0 / (500000.0 ** (torch.arange(0, self.hd, 2).float() / self.hd)) + fr = torch.outer(torch.arange(T).float(), inv) + self.register_buffer('rc', fr.cos(), persistent=False) + self.register_buffer('rs', fr.sin(), persistent=False) + def rope(self, x): + Tn = x.shape[2] + x1, x2 = x[..., ::2], x[..., 1::2] + c, s = self.rc[None, None, :Tn].to(x.dtype), self.rs[None, None, :Tn].to(x.dtype) + return torch.stack((x1 * c - x2 * s, x1 * s + x2 * c), dim=-1).flatten(-2) + def forward(self, x): + Bn, Tn, C = x.shape + q, k, v = self.qkv(x).split(C, dim=2) + q, k = self.qn(q), self.kn(k) + q = self.rope(q.view(Bn, Tn, self.H, self.hd).transpose(1, 2)) + k = self.rope(k.view(Bn, Tn, self.H, self.hd).transpose(1, 2)) + v = v.view(Bn, Tn, self.H, self.hd).transpose(1, 2) + y = F.scaled_dot_product_attention(q, k, v, is_causal=True) + return self.proj(y.transpose(1, 2).contiguous().view(Bn, Tn, C)) + +class Olmo2Block(nn.Module): + def __init__(self, C, H, T): + super().__init__() + self.attn = Olmo2Attn(C, H, T); self.ff = SwiGLU(C) + self.na, self.nf = RMSNorm(C), RMSNorm(C) + def forward(self, z): + z = z + self.na(self.attn(z)) + return z + self.nf(self.ff(z)) + +cfg = ck['config']; C, H, L = cfg['C'], cfg['H'], cfg['L'] +tok = nn.Embedding(vocab, C).to(dev); tok.load_state_dict(ck['tok']) +blocks = nn.ModuleList([Olmo2Block(C, H, T) for _ in range(L)]).to(dev) +blocks.load_state_dict(ck['blocks'], strict=False) +W_out = ck['wout'].to(dev) +ln_f = RMSNorm(C).to(dev); ln_f.load_state_dict(ck['lnf']) +NBT = B * T +params = [p for p in blocks.parameters()] +pcounts = [sum(1 for _ in blocks[l].parameters()) for l in range(L)] + +def readout(z): return ln_f(z) @ W_out.t() + +def flat(gs): return torch.cat([g.reshape(-1).double() for g in gs]) + +def cos(ga, gb): + va, vb = flat(ga), flat(gb) + return float((va @ vb) / (va.norm() * vb.norm() + 1e-30)) + +def cos_by_block(ga, gb): + out, i = [], 0 + for l in range(L): + sa = ga[i:i + pcounts[l]]; sb = gb[i:i + pcounts[l]] + out.append(cos(sa, sb)); i += pcounts[l] + return out + +def grads_from(outs_list, cots): + """sum_l -> d/dtheta. Positive scale/sign of cots irrelevant for cos.""" + obj = sum((o * c.detach()).sum() for o, c in zip(outs_list, cots)) + gs = torch.autograd.grad(obj, params, allow_unused=True, retain_graph=True) + return [g.float() if g is not None else torch.zeros_like(p) for g, p in zip(gs, params)] + +def run_case(x, y, beta): + # --- BP reference + exact cotangents c_l at free states (whole-chain graph) --- + z = tok(x) + zs_bp = [] + for b in blocks: + z = b(z); zs_bp.append(z) + ce = F.cross_entropy(readout(zs_bp[-1]).reshape(-1, vocab), y.reshape(-1)) + g_bp = [g.float() for g in torch.autograd.grad(ce, params, retain_graph=True, allow_unused=False)] + c = [g.detach() for g in torch.autograd.grad(ce, zs_bp, retain_graph=False)] + + # --- free block-wise pass (graph per block, EP-style detach between blocks) --- + f_ins, f_outs = [], [] + prev = tok(x).detach() + for b in blocks: + i = prev.detach().requires_grad_(True) + o = b(i) + f_ins.append(i); f_outs.append(o) + prev = o.detach() + + # self-check: (free, c) through this path must equal BP + g_chk = grads_from(f_outs, c) + chk = cos(g_chk, g_bp) + + # --- EP nudged relaxation (harness-faithful): K sweeps of derive-d + rebuild --- + ins, outs = f_ins, f_outs + zs = [o.detach().float() for o in f_outs] + d = [None] * L + for k in range(a.K): + zc = zs[L - 1].detach().requires_grad_(True) + ce_k = F.cross_entropy(readout(zc).reshape(-1, vocab), y.reshape(-1)) + d[L - 1] = (-beta * NBT * torch.autograd.grad(ce_k, zc)[0]).detach().float() + for l in range(L - 2, -1, -1): + d[l] = torch.autograd.grad(outs[l + 1], ins[l + 1], grad_outputs=d[l + 1], + retain_graph=True)[0].detach().float() + prev = tok(x).detach() + n_ins, n_outs = [], [] + for l in range(L): + i = prev.detach().requires_grad_(True) + o = blocks[l](i) + n_ins.append(i); n_outs.append(o) + zs[l] = (o.detach().float() + d[l]) + prev = zs[l] + ins, outs = n_ins, n_outs + + md = [-di for di in d] # -d ~ +beta*NBT*(transmitted cotangent); scale washes out in cos + g_ep = grads_from(outs, md) # (nudged anchor, EP d) = EP + g_trans = grads_from(f_outs, md) # (free anchor, EP d) = transmission-only error + g_loc = grads_from(outs, c) # (nudged anchor, exact c) = anchor-only error + return g_bp, g_ep, g_trans, g_loc, chk + +batches = [get_batch() for _ in range(a.nb)] +for beta in [float(s) for s in a.betas.split(',')]: + agg = {'ep': [], 'trans': [], 'loc': [], 'chk': []} + prof_ep, prof_trans, prof_loc = [], [], [] + for x, y in batches: + g_bp, g_ep, g_trans, g_loc, chk = run_case(x, y, beta) + agg['ep'].append(cos(g_ep, g_bp)); agg['trans'].append(cos(g_trans, g_bp)) + agg['loc'].append(cos(g_loc, g_bp)); agg['chk'].append(chk) + prof_ep.append(cos_by_block(g_ep, g_bp)) + prof_trans.append(cos_by_block(g_trans, g_bp)) + prof_loc.append(cos_by_block(g_loc, g_bp)) + del g_bp, g_ep, g_trans, g_loc + torch.cuda.empty_cache() + m = {k: sum(v) / len(v) for k, v in agg.items()} + print(f"[beta={beta:g} K={a.K}] selfcheck(free,c)={m['chk']:.6f} " + f"EP={m['ep']:.4f} TRANS-only={m['trans']:.4f} ANCHOR-only={m['loc']:.4f}", flush=True) + for name, prof in [('EP ', prof_ep), ('TRANS', prof_trans), ('ANCHR', prof_loc)]: + mp = [sum(p[l] for p in prof) / len(prof) for l in range(L)] + print(f" {name} by block (0=bottom..{L-1}=top): " + + " ".join(f"{v:.3f}" for v in mp), flush=True) +print("DONE_BLOCKCOS", flush=True) -- cgit v1.2.3