E-tier wave-1 ledger (RESULT 14 + HW map rows) + cost model v2 recalibrated on measured H200 datapoint

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014FAPDWQ49M5Ye3NpTndTpn

1 files changed, 227 insertions, 0 deletions

diff --git a/ep_run/etier_probe.py b/ep_run/etier_probe.py
new file mode 100644
index 0000000..343b77a
--- /dev/null
+++ b/ep_run/etier_probe.py
@@ -0,0 +1,227 @@
+"""E-tier wave-1: device-fault tolerance probes on the trained OLMo2 cascade (stage1b ckpt).
+For each fault x severity: (a) faulted free-forward val CE (inference survival),
+(b) cos(EP_faulted, BP_faulted) — estimator robustness on the faulted system,
+(c) cos(EP_faulted, BP_clean) — direction vs the clean-system gradient.
+Faults: wq (weight quant bits) | fnoise (dynamic block-output noise, mult) |
+divmis (RMSNorm divider mismatch, fixed per-channel) | gilbert (SwiGLU gate gain mismatch)
+| rope (phase error) | fbnoise (additive error-channel noise in the nudged feedback).
+Usage: etier_probe.py --shard {A,B,C}
+"""
+import argparse, math, pickle, copy
+import numpy as np, torch, torch.nn as nn, torch.nn.functional as F
+from pathlib import Path
+
+ap = argparse.ArgumentParser()
+ap.add_argument('--shard', required=True, choices=['A', 'B', 'C'])
+ap.add_argument('--ckpt', default='runs/stage1b_ep_muon_s55000.pt')
+ap.add_argument('--beta', type=float, default=1e-3)
+ap.add_argument('--K', type=int, default=3)
+ap.add_argument('--nb', type=int, default=4)
+a = ap.parse_args()
+dev = 'cuda' if torch.cuda.is_available() else 'cpu'
+torch.manual_seed(7)
+
+DD = Path('/home/yurenh2/ept/ep_run/data/tinystories_bpe')
+vocab = pickle.load(open(DD / 'meta.pkl', 'rb'))['vocab_size']
+B, T = 8, 256
+
+def get_batch():
+    data = np.memmap(DD / 'val.bin', dtype=np.uint16, mode='r')
+    ix = torch.randint(len(data) - T - 1, (B,))
+    x = torch.stack([torch.from_numpy(data[i:i + T].astype(np.int64)) for i in ix])
+    y = torch.stack([torch.from_numpy(data[i + 1:i + 1 + T].astype(np.int64)) for i in ix])
+    return x.to(dev), y.to(dev)
+
+# ---- model (OLMo2 cascade, matches trainer) with fault hooks ----
+FAULT = {'fnoise': 0.0, 'fbnoise': 0.0}
+
+class RMSNorm(nn.Module):
+    def __init__(self, C, eps=1e-6):
+        super().__init__(); self.g = nn.Parameter(torch.ones(C)); self.eps = eps
+    def forward(self, x):
+        return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) * self.g
+
+class SwiGLU(nn.Module):
+    def __init__(self, C):
+        super().__init__()
+        h = ((8 * C // 3) + 63) // 64 * 64
+        self.w1 = nn.Linear(C, h, bias=False); self.w3 = nn.Linear(C, h, bias=False)
+        self.w2 = nn.Linear(h, C, bias=False)
+        self.register_buffer('ggain', torch.ones(h), persistent=False)   # gilbert mismatch
+    def forward(self, x):
+        return self.w2(F.silu(self.w1(x)) * self.w3(x) * self.ggain)
+
+class Olmo2Attn(nn.Module):
+    def __init__(self, C, H, T):
+        super().__init__()
+        self.H, self.hd = H, C // H
+        self.qkv = nn.Linear(C, 3 * C, bias=False); self.proj = nn.Linear(C, C, bias=False)
+        self.qn, self.kn = RMSNorm(C), RMSNorm(C)
+        inv = 1.0 / (500000.0 ** (torch.arange(0, self.hd, 2).float() / self.hd))
+        fr = torch.outer(torch.arange(T).float(), inv)
+        self.register_buffer('rc', fr.cos(), persistent=False)
+        self.register_buffer('rs', fr.sin(), persistent=False)
+        self.register_buffer('fr', fr, persistent=False)
+    def rope(self, x):
+        Tn = x.shape[2]
+        x1, x2 = x[..., ::2], x[..., 1::2]
+        c, s = self.rc[None, None, :Tn], self.rs[None, None, :Tn]
+        return torch.stack((x1 * c - x2 * s, x1 * s + x2 * c), dim=-1).flatten(-2)
+    def forward(self, x):
+        Bn, Tn, C = x.shape
+        q, k, v = self.qkv(x).split(C, dim=2)
+        q, k = self.qn(q), self.kn(k)
+        q = self.rope(q.view(Bn, Tn, self.H, self.hd).transpose(1, 2))
+        k = self.rope(k.view(Bn, Tn, self.H, self.hd).transpose(1, 2))
+        v = v.view(Bn, Tn, self.H, self.hd).transpose(1, 2)
+        y = F.scaled_dot_product_attention(q, k, v, is_causal=True)
+        return self.proj(y.transpose(1, 2).contiguous().view(Bn, Tn, C))
+
+class Olmo2Block(nn.Module):
+    def __init__(self, C, H, T):
+        super().__init__()
+        self.attn = Olmo2Attn(C, H, T); self.ff = SwiGLU(C)
+        self.na, self.nf = RMSNorm(C), RMSNorm(C)
+    def forward(self, z):
+        z = z + self.na(self.attn(z))
+        out = z + self.nf(self.ff(z))
+        if FAULT['fnoise'] > 0:
+            out = out * (1.0 + FAULT['fnoise'] * torch.randn_like(out))
+        return out
+
+ck = torch.load(a.ckpt, map_location=dev, weights_only=False)
+cfg = ck['config']; C, H, L = cfg['C'], cfg['H'], cfg['L']
+tok = nn.Embedding(vocab, C).to(dev); tok.load_state_dict(ck['tok'])
+blocks0 = nn.ModuleList([Olmo2Block(C, H, T) for _ in range(L)]).to(dev)
+blocks0.load_state_dict(ck['blocks'], strict=False)   # ggain buffers are non-persistent extras
+W_out0 = ck['wout'].to(dev)
+ln_f0 = RMSNorm(C).to(dev); ln_f0.load_state_dict(ck['lnf'])
+NBT = B * T
+
+def apply_fault(kind, sev):
+    """return (blocks, W_out, ln_f) with the fault applied; also sets FAULT dict."""
+    FAULT['fnoise'] = 0.0; FAULT['fbnoise'] = 0.0
+    bl = copy.deepcopy(blocks0); Wo = W_out0.clone(); lf = copy.deepcopy(ln_f0)
+    g = torch.Generator(device='cpu').manual_seed(11)
+    if kind == 'clean':
+        pass
+    elif kind == 'wq':
+        bits = sev
+        with torch.no_grad():
+            for m in bl.modules():
+                if isinstance(m, nn.Linear):
+                    s = m.weight.abs().max() / (2 ** (bits - 1) - 1)
+                    m.weight.copy_(torch.round(m.weight / s) * s)
+            s = Wo.abs().max() / (2 ** (bits - 1) - 1)
+            Wo = torch.round(Wo / s) * s
+    elif kind == 'divmis':
+        with torch.no_grad():
+            for m in bl.modules():
+                if isinstance(m, RMSNorm):
+                    m.g.mul_(1.0 + sev * torch.randn(m.g.shape, generator=g).to(dev))
+            lf.g.mul_(1.0 + sev * torch.randn(lf.g.shape, generator=g).to(dev))
+    elif kind == 'gilbert':
+        with torch.no_grad():
+            for m in bl.modules():
+                if isinstance(m, SwiGLU):
+                    m.ggain.copy_(1.0 + sev * torch.randn(m.ggain.shape, generator=g).to(dev))
+    elif kind == 'rope':
+        with torch.no_grad():
+            for m in bl.modules():
+                if isinstance(m, Olmo2Attn):
+                    d = sev * torch.randn(m.fr.shape, generator=g).to(dev)
+                    m.rc.copy_((m.fr + d).cos()); m.rs.copy_((m.fr + d).sin())
+    elif kind == 'fnoise':
+        FAULT['fnoise'] = sev
+    elif kind == 'fbnoise':
+        FAULT['fbnoise'] = sev
+    return bl, Wo, lf
+
+def readout(z, Wo, lf): return lf(z) @ Wo.t()
+
+def ep_grad(bl, Wo, lf, x, y):
+    """single-sided fb EP grad on blocks params (matches trainer structure, K rounds)."""
+    with torch.no_grad():
+        z = tok(x)
+        zs_free = []
+        for b in bl: z = b(z); zs_free.append(z.clone())
+    # graphed free pass
+    ins, outs, zs = [], [], []
+    prev = zs_free[0] * 0 + tok(x).detach()
+    prev = tok(x).detach()
+    for b in bl:
+        i = prev.detach().requires_grad_(True)
+        o = b(i)
+        ins.append(i); outs.append(o); zs.append(o.detach())
+        prev = zs[-1]
+    d = [None] * L
+    for k in range(a.K):
+        zc = zs[L - 1].detach().requires_grad_(True)
+        ce = F.cross_entropy(readout(zc, Wo, lf).reshape(-1, vocab), y.reshape(-1))
+        d[L - 1] = (-a.beta * NBT * torch.autograd.grad(ce, zc)[0]).detach()
+        for l in range(L - 2, -1, -1):
+            d[l] = torch.autograd.grad(outs[l + 1], ins[l + 1], grad_outputs=d[l + 1])[0].detach()
+        if FAULT['fbnoise'] > 0:
+            for l in range(L):
+                d[l] = d[l] + FAULT['fbnoise'] * d[l].norm() / math.sqrt(d[l].numel()) * torch.randn_like(d[l])
+        last = (k + 1 == a.K)
+        prev = tok(x).detach()
+        n_ins, n_outs = [], []
+        for l in range(L):
+            i = prev.detach().requires_grad_(True)
+            o = bl[l](i)
+            n_ins.append(i); n_outs.append(o)
+            zs[l] = (o + d[l]).detach()
+            prev = zs[l]
+        ins, outs = n_ins, n_outs
+    E = 0.0
+    for z, o in zip(zs, outs): E = E + 0.5 * ((z.detach().float() - o.float()) ** 2).sum()
+    obj = E / (NBT * a.beta)
+    params = [p for p in bl.parameters()]
+    gs = torch.autograd.grad(obj, params, allow_unused=True)
+    return [g if g is not None else torch.zeros(1, device=dev) for g in gs]
+
+def bp_grad(bl, Wo, lf, x, y):
+    z = tok(x)
+    for b in bl: z = b(z)
+    ce = F.cross_entropy(readout(z, Wo, lf).reshape(-1, vocab), y.reshape(-1))
+    params = [p for p in bl.parameters()]
+    gs = torch.autograd.grad(ce, params, allow_unused=True)
+    return [g if g is not None else torch.zeros(1, device=dev) for g in gs]
+
+def val_ce(bl, Wo, lf, x, y):
+    with torch.no_grad():
+        z = tok(x)
+        for b in bl: z = b(z)
+        return float(F.cross_entropy(readout(z, Wo, lf).reshape(-1, vocab), y.reshape(-1)))
+
+def cos(ga, gb):
+    va = torch.cat([g.reshape(-1) for g in ga]); vb = torch.cat([g.reshape(-1) for g in gb])
+    return float((va @ vb) / (va.norm() * vb.norm() + 1e-30))
+
+SHARDS = {
+    'A': [('wq', 8), ('wq', 6), ('wq', 4), ('divmis', 0.01), ('divmis', 0.03), ('divmis', 0.10)],
+    'B': [('fnoise', 1e-3), ('fnoise', 3e-3), ('fnoise', 1e-2), ('rope', 0.01), ('rope', 0.03), ('rope', 0.10)],
+    'C': [('gilbert', 0.01), ('gilbert', 0.03), ('gilbert', 0.10), ('fbnoise', 1e-2), ('fbnoise', 1e-1), ('fbnoise', 3e-1)],
+}
+
+batches = [get_batch() for _ in range(a.nb)]
+blc, Woc, lfc = apply_fault('clean', 0)
+clean_ce = sum(val_ce(blc, Woc, lfc, x, y) for x, y in batches) / a.nb
+gclean = [[g.cpu() for g in bp_grad(blc, Woc, lfc, x, y)] for x, y in batches]
+torch.cuda.empty_cache()
+print(f"[clean] val CE {clean_ce:.4f}", flush=True)
+
+for kind, sev in SHARDS[a.shard]:
+    torch.cuda.empty_cache()
+    bl, Wo, lf = apply_fault(kind, sev)
+    ce = sum(val_ce(bl, Wo, lf, x, y) for x, y in batches) / a.nb
+    c_self, c_clean = [], []
+    for i, (x, y) in enumerate(batches):
+        ge = ep_grad(bl, Wo, lf, x, y)
+        gbf = bp_grad(bl, Wo, lf, x, y)
+        c_self.append(cos(ge, gbf))
+        c_clean.append(cos([g.cpu() for g in ge], gclean[i]))
+        del ge, gbf; torch.cuda.empty_cache()
+    print(f"[{kind}={sev}] valCE {ce:.4f} (Δ{ce-clean_ce:+.4f}) | cos(EP,BP_faulted) {sum(c_self)/a.nb:.4f} | cos(EP,BP_clean) {sum(c_clean)/a.nb:.4f}", flush=True)
+print(f"DONE_{a.shard}", flush=True)