OLMo2-standard architecture (user directive): norm-after-sublayer RMSNorm blocks, full-width QK-norm, RoPE(500k), SwiGLU, no-bias, untied head, final RMSNorm, 0.02 init, grouped wd, optional z-loss (plumbed through nudge+readout+gate for EP exactness); EP smoke 42.75M cos=1.0000

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014FAPDWQ49M5Ye3NpTndTpn

2 files changed, 173 insertions, 19 deletions

diff --git a/ep_run/casc_bp_train.py b/ep_run/casc_bp_train.py
index 069d858..987277f 100644
--- a/ep_run/casc_bp_train.py
+++ b/ep_run/casc_bp_train.py
@@ -22,7 +22,11 @@ ap.add_argument('--lr_min_ratio', type=float, default=0.1)
 ap.add_argument('--qk_norm', action='store_true')        # RMS-norm q,k per head before scores (OLMo2-style; bounds logits, analog-friendly)
 ap.add_argument('--final_ln', action='store_true')       # final LayerNorm before readout (standard GPT; bounds sig_tok growth -> keeps beta/estimator healthy on long runs)
 ap.add_argument('--resume', default='')                  # path to a ckpt (tok/pos/blocks) to continue from; step taken from ckpt
+ap.add_argument('--olmo2', action='store_true')          # OLMo2-standard block (see casc_eq_train.py)
+ap.add_argument('--wd', type=float, default=-1.0)        # >=0: grouped weight decay; <0 = legacy uniform 1e-4
+ap.add_argument('--zloss', type=float, default=0.0)      # z-loss coefficient; 0 = off
 args = ap.parse_args()
+if args.olmo2 and args.tok_init <= 0: args.tok_init = 0.02
 torch.manual_seed(args.seed)
 dev = 'cuda' if torch.cuda.is_available() else 'cpu'
 
@@ -68,26 +72,90 @@ class Block(nn.Module):
         z = z + self.attn(self.ln1(z))
         return z + self.ff(self.ln2(z))
 
+class RMSNorm(nn.Module):
+    def __init__(self, C, eps=1e-6):
+        super().__init__(); self.g = nn.Parameter(torch.ones(C)); self.eps = eps
+    def forward(self, x):
+        return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) * self.g
+
+class SwiGLU(nn.Module):
+    def __init__(self, C):
+        super().__init__()
+        h = ((8 * C // 3) + 63) // 64 * 64   # ~param-match the 4x-GELU MLP (8C^2)
+        self.w1 = nn.Linear(C, h, bias=False); self.w3 = nn.Linear(C, h, bias=False)
+        self.w2 = nn.Linear(h, C, bias=False)
+    def forward(self, x): return self.w2(F.silu(self.w1(x)) * self.w3(x))
+
+class Olmo2Attn(nn.Module):
+    """OLMo2 attention: no-bias projs, FULL-WIDTH RMS QK-norm (pre-head-split, HF Olmo2 order), then per-head RoPE."""
+    def __init__(self, C, H, T):
+        super().__init__()
+        self.H, self.hd = H, C // H
+        self.qkv = nn.Linear(C, 3 * C, bias=False); self.proj = nn.Linear(C, C, bias=False)
+        self.qn, self.kn = RMSNorm(C), RMSNorm(C)
+        inv = 1.0 / (500000.0 ** (torch.arange(0, self.hd, 2).float() / self.hd))
+        fr = torch.outer(torch.arange(T).float(), inv)
+        self.register_buffer('rc', fr.cos(), persistent=False)
+        self.register_buffer('rs', fr.sin(), persistent=False)
+    def rope(self, x):
+        x1, x2 = x[..., ::2], x[..., 1::2]
+        c, s = self.rc[None, None], self.rs[None, None]
+        return torch.stack((x1 * c - x2 * s, x1 * s + x2 * c), dim=-1).flatten(-2)
+    def forward(self, x):
+        B, T, C = x.shape
+        q, k, v = self.qkv(x).split(C, dim=2)
+        q, k = self.qn(q), self.kn(k)
+        q = self.rope(q.view(B, T, self.H, self.hd).transpose(1, 2))
+        k = self.rope(k.view(B, T, self.H, self.hd).transpose(1, 2))
+        v = v.view(B, T, self.H, self.hd).transpose(1, 2)
+        y = F.scaled_dot_product_attention(q, k, v, is_causal=True)
+        return self.proj(y.transpose(1, 2).contiguous().view(B, T, C))
+
+class Olmo2Block(nn.Module):
+    """OLMo2 reordered norm (norm AFTER each sublayer, inside the residual) — their training-stability change."""
+    def __init__(self, C, H, T):
+        super().__init__()
+        self.attn = Olmo2Attn(C, H, T); self.ff = SwiGLU(C)
+        self.na, self.nf = RMSNorm(C), RMSNorm(C)
+    def forward(self, z, mask=None):
+        z = z + self.na(self.attn(z))
+        return z + self.nf(self.ff(z))
+
 tok = nn.Embedding(vocab, args.C).to(dev)
 pos = nn.Embedding(args.T, args.C).to(dev)
 if args.tok_init > 0:
     with torch.no_grad():
         tok.weight.normal_(0, args.tok_init); pos.weight.normal_(0, args.tok_init)
-blocks = nn.ModuleList([Block(args.C, args.H, args.qk_norm) for _ in range(args.L)]).to(dev)
+blocks = nn.ModuleList([(Olmo2Block(args.C, args.H, args.T) if args.olmo2 else Block(args.C, args.H, args.qk_norm)) for _ in range(args.L)]).to(dev)
+if args.olmo2:
+    with torch.no_grad():
+        for m in blocks.modules():
+            if isinstance(m, nn.Linear): m.weight.normal_(0, 0.02)
+W_out = nn.Parameter(torch.randn(vocab, args.C, device=dev) * 0.02) if args.olmo2 else None
 mask = torch.triu(torch.full((args.T, args.T), float('-inf'), device=dev), 1)
-ln_f = nn.LayerNorm(args.C).to(dev) if args.final_ln else nn.Identity()
-params = list(tok.parameters()) + list(pos.parameters()) + list(blocks.parameters()) + list(ln_f.parameters())
+ln_f = (RMSNorm(args.C) if args.olmo2 else (nn.LayerNorm(args.C) if args.final_ln else nn.Identity())).to(dev)
+params = list(tok.parameters()) + ([] if args.olmo2 else list(pos.parameters())) + list(blocks.parameters()) + list(ln_f.parameters()) + ([W_out] if args.olmo2 else [])
 start_step = 0
 if args.resume:
     _ck = torch.load(args.resume, map_location=dev, weights_only=False)
     tok.load_state_dict(_ck['tok']); pos.load_state_dict(_ck['pos']); blocks.load_state_dict(_ck['blocks'])
+    if _ck.get('wout') is not None and args.olmo2:
+        with torch.no_grad(): W_out.copy_(_ck['wout'].to(dev))
+    if _ck.get('lnf') is not None and not isinstance(ln_f, nn.Identity): ln_f.load_state_dict(_ck['lnf'])
     start_step = int(_ck.get('step', 0))
     print(f'[resume] loaded {args.resume} at step {start_step}', flush=True)
 if args.opt == 'muon':
     from muon import build_hybrid
     opt, sched = build_hybrid(blocks, params, args.lr, args.muon_lr, args.warmup)
 else:
-    opt = torch.optim.AdamW(params, lr=args.lr, weight_decay=1e-4)
+    if args.wd >= 0:   # OLMo2-style grouped decay
+        nodecay = {id(p) for p in tok.parameters()} | {id(p) for p in pos.parameters()} | \
+                  {id(p) for p in blocks.parameters() if p.ndim < 2} | {id(p) for p in ln_f.parameters()}
+        opt = torch.optim.AdamW([
+            {'params': [p for p in params if id(p) not in nodecay], 'weight_decay': args.wd},
+            {'params': [p for p in params if id(p) in nodecay], 'weight_decay': 0.0}], lr=args.lr)
+    else:
+        opt = torch.optim.AdamW(params, lr=args.lr, weight_decay=1e-4)
     if args.cosine:
         def _lrlam(s):
             if s < args.warmup: return (s + 1) / max(args.warmup, 1)
@@ -98,9 +166,9 @@ else:
         sched = torch.optim.lr_scheduler.LambdaLR(opt, lambda s: min(1.0, (s + 1) / max(args.warmup, 1)))
 
 def fwd(x):
-    z = tok(x) + pos(torch.arange(args.T, device=dev))[None]
+    z = tok(x) if args.olmo2 else tok(x) + pos(torch.arange(args.T, device=dev))[None]
     for b in blocks: z = b(z, mask)
-    return ln_f(z) @ tok.weight.t()
+    return ln_f(z) @ (W_out.t() if args.olmo2 else tok.weight.t())
 
 @torch.no_grad()
 def evaluate(nb=6):
@@ -126,7 +194,10 @@ outdir = Path('runs'); outdir.mkdir(exist_ok=True)
 for _ in range(start_step): sched.step()   # advance LR schedule to the resumed step
 for step in range(start_step, args.steps + 1):
     x, y = get_batch('train')
-    loss = F.cross_entropy(fwd(x).reshape(-1, vocab), y.reshape(-1))
+    logits = fwd(x).reshape(-1, vocab)
+    loss = F.cross_entropy(logits, y.reshape(-1))
+    if args.zloss > 0:
+        loss = loss + args.zloss * (torch.logsumexp(logits.float(), -1) ** 2).mean()
     opt.zero_grad(set_to_none=True); loss.backward()
     torch.nn.utils.clip_grad_norm_(params, 1.0)
     opt.step(); sched.step()
@@ -139,6 +210,8 @@ for step in range(start_step, args.steps + 1):
             except Exception: pass
     if step % args.save_every == 0:
         torch.save({'tok': tok.state_dict(), 'pos': pos.state_dict(), 'blocks': blocks.state_dict(),
+                    'wout': (W_out.detach().cpu() if args.olmo2 else None),
+                    'lnf': (ln_f.state_dict() if not isinstance(ln_f, nn.Identity) else None),
                     'step': step, 'val': best, 'config': vars(args)}, outdir / f'{args.tag}_s{step}.pt')
 print(f'[{args.tag}] DONE best val CE {best:.4f} (random ln({vocab})={math.log(vocab):.3f})', flush=True)
 if wb is not None:
diff --git a/ep_run/casc_eq_train.py b/ep_run/casc_eq_train.py
index b85ac96..f3b8736 100644
--- a/ep_run/casc_eq_train.py
+++ b/ep_run/casc_eq_train.py
@@ -34,10 +34,16 @@ ap.add_argument('--qk_norm', action='store_true')        # RMS-norm q,k per head
 ap.add_argument('--final_ln', action='store_true')       # final LayerNorm before readout (standard GPT; bounds sig_tok growth -> keeps beta/estimator healthy on long runs)
 ap.add_argument('--resume', default='')                  # path to a ckpt (tok/pos/blocks) to continue from; step taken from ckpt
 ap.add_argument('--sig0', type=float, default=-1.0)      # override SIG0 (beta-schedule ref); needed on resume to restore original beta regime
+ap.add_argument('--olmo2', action='store_true')          # OLMo2-standard block: norm-AFTER-sublayer RMSNorm, full-width QK-norm, RoPE(500k), SwiGLU, no-bias, untied head, final RMSNorm, 0.02 init
+ap.add_argument('--wd', type=float, default=-1.0)        # >=0: grouped weight decay (linear weights+head decay; embeddings/norm-gains none). <0 = legacy uniform 1e-4
+ap.add_argument('--zloss', type=float, default=0.0)      # z-loss coefficient on train objective (OLMo2-style logit regularizer); 0 = off
 ap.add_argument('--dtop_every', type=int, default=1)    # 1 = exact (DEFAULT, BP-parity); 2 = fast mode (~20% cheaper, ~4% CE tax at high lr)
 ap.add_argument('--gate_every', type=int, default=200)  # in-training cos(EP,BP) telemetry; <=0 = fully BP-free (no bp_gate at all)
 ap.add_argument('--gate_govern', action='store_true')   # let gate cos adjust K/bscale (default: observe-only => training control is BP-free)
 args = ap.parse_args()
+if args.olmo2:
+    args.untie = True
+    if args.tok_init <= 0: args.tok_init = 0.02
 torch.manual_seed(args.seed)
 dev = 'cuda' if torch.cuda.is_available() else 'cpu'
 
@@ -83,12 +89,65 @@ class Block(nn.Module):
         z = z + self.attn(self.ln1(z))
         return z + self.ff(self.ln2(z))
 
+class RMSNorm(nn.Module):
+    def __init__(self, C, eps=1e-6):
+        super().__init__(); self.g = nn.Parameter(torch.ones(C)); self.eps = eps
+    def forward(self, x):
+        return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) * self.g
+
+class SwiGLU(nn.Module):
+    def __init__(self, C):
+        super().__init__()
+        h = ((8 * C // 3) + 63) // 64 * 64   # ~param-match the 4x-GELU MLP (8C^2)
+        self.w1 = nn.Linear(C, h, bias=False); self.w3 = nn.Linear(C, h, bias=False)
+        self.w2 = nn.Linear(h, C, bias=False)
+    def forward(self, x): return self.w2(F.silu(self.w1(x)) * self.w3(x))
+
+class Olmo2Attn(nn.Module):
+    """OLMo2 attention: no-bias projs, FULL-WIDTH RMS QK-norm (pre-head-split, HF Olmo2 order), then per-head RoPE."""
+    def __init__(self, C, H, T):
+        super().__init__()
+        self.H, self.hd = H, C // H
+        self.qkv = nn.Linear(C, 3 * C, bias=False); self.proj = nn.Linear(C, C, bias=False)
+        self.qn, self.kn = RMSNorm(C), RMSNorm(C)
+        inv = 1.0 / (500000.0 ** (torch.arange(0, self.hd, 2).float() / self.hd))
+        fr = torch.outer(torch.arange(T).float(), inv)
+        self.register_buffer('rc', fr.cos(), persistent=False)
+        self.register_buffer('rs', fr.sin(), persistent=False)
+    def rope(self, x):
+        x1, x2 = x[..., ::2], x[..., 1::2]
+        c, s = self.rc[None, None], self.rs[None, None]
+        return torch.stack((x1 * c - x2 * s, x1 * s + x2 * c), dim=-1).flatten(-2)
+    def forward(self, x):
+        B, T, C = x.shape
+        q, k, v = self.qkv(x).split(C, dim=2)
+        q, k = self.qn(q), self.kn(k)
+        q = self.rope(q.view(B, T, self.H, self.hd).transpose(1, 2))
+        k = self.rope(k.view(B, T, self.H, self.hd).transpose(1, 2))
+        v = v.view(B, T, self.H, self.hd).transpose(1, 2)
+        y = F.scaled_dot_product_attention(q, k, v, is_causal=True)
+        return self.proj(y.transpose(1, 2).contiguous().view(B, T, C))
+
+class Olmo2Block(nn.Module):
+    """OLMo2 reordered norm (norm AFTER each sublayer, inside the residual) — their training-stability change."""
+    def __init__(self, C, H, T):
+        super().__init__()
+        self.attn = Olmo2Attn(C, H, T); self.ff = SwiGLU(C)
+        self.na, self.nf = RMSNorm(C), RMSNorm(C)
+    def forward(self, z, mask=None):
+        z = z + self.na(self.attn(z))
+        return z + self.nf(self.ff(z))
+
 tok = nn.Embedding(vocab, args.C).to(dev)
 pos = nn.Embedding(args.T, args.C).to(dev)
 if args.tok_init > 0:
     with torch.no_grad():
         tok.weight.normal_(0, args.tok_init); pos.weight.normal_(0, args.tok_init)
-blocks = nn.ModuleList([Block(args.C, args.H, args.qk_norm) for _ in range(args.L)]).to(dev)
+blocks = nn.ModuleList([(Olmo2Block(args.C, args.H, args.T) if args.olmo2 else Block(args.C, args.H, args.qk_norm)) for _ in range(args.L)]).to(dev)
+if args.olmo2:
+    with torch.no_grad():
+        for m in blocks.modules():
+            if isinstance(m, nn.Linear): m.weight.normal_(0, 0.02)
 if args.compile:
     try:
         for i in range(args.L): blocks[i] = torch.compile(blocks[i], mode='reduce-overhead')
@@ -97,20 +156,32 @@ if args.compile:
         print(f'[compile] disabled ({e})', flush=True)
 mask = torch.triu(torch.full((args.T, args.T), float('-inf'), device=dev), 1)
 W_out = nn.Parameter(torch.randn(vocab, args.C, device=dev) * 0.02) if args.untie else None
-ln_f = nn.LayerNorm(args.C).to(dev) if args.final_ln else nn.Identity()
+ln_f = (RMSNorm(args.C) if args.olmo2 else (nn.LayerNorm(args.C) if args.final_ln else nn.Identity())).to(dev)
+def emb(x):
+    return tok(x) if args.olmo2 else tok(x) + pos(torch.arange(args.T, device=dev))[None]
 readout = (lambda z: ln_f(z) @ W_out.t()) if args.untie else (lambda z: ln_f(z) @ tok.weight.t())
-all_params = list(tok.parameters()) + list(pos.parameters()) + list(blocks.parameters()) + list(ln_f.parameters()) + ([W_out] if args.untie else [])
+all_params = list(tok.parameters()) + ([] if args.olmo2 else list(pos.parameters())) + list(blocks.parameters()) + list(ln_f.parameters()) + ([W_out] if args.untie else [])
 start_step = 0
 if args.resume:
     _ck = torch.load(args.resume, map_location=dev, weights_only=False)
     tok.load_state_dict(_ck['tok']); pos.load_state_dict(_ck['pos']); blocks.load_state_dict(_ck['blocks'])
+    if _ck.get('wout') is not None and args.untie:
+        with torch.no_grad(): W_out.copy_(_ck['wout'].to(dev))
+    if _ck.get('lnf') is not None and not isinstance(ln_f, nn.Identity): ln_f.load_state_dict(_ck['lnf'])
     start_step = int(_ck.get('step', 0))
     print(f'[resume] loaded {args.resume} at step {start_step}', flush=True)
 if args.opt == 'muon':
     from muon import build_hybrid
     opt, sched = build_hybrid(blocks, all_params, args.lr, args.muon_lr, args.warmup)
 else:
-    opt = torch.optim.AdamW(all_params, lr=args.lr, weight_decay=1e-4)
+    if args.wd >= 0:   # OLMo2-style grouped decay: linear weights + head decay; embeddings/norm-gains none
+        nodecay = {id(p) for p in tok.parameters()} | {id(p) for p in pos.parameters()} | \
+                  {id(p) for p in blocks.parameters() if p.ndim < 2} | {id(p) for p in ln_f.parameters()}
+        opt = torch.optim.AdamW([
+            {'params': [p for p in all_params if id(p) not in nodecay], 'weight_decay': args.wd},
+            {'params': [p for p in all_params if id(p) in nodecay], 'weight_decay': 0.0}], lr=args.lr)
+    else:
+        opt = torch.optim.AdamW(all_params, lr=args.lr, weight_decay=1e-4)
     if args.cosine:
         def _lrlam(s):
             if s < args.warmup: return (s + 1) / max(args.warmup, 1)
@@ -121,10 +192,18 @@ else:
         sched = torch.optim.lr_scheduler.LambdaLR(opt, lambda s: min(1.0, (s + 1) / max(args.warmup, 1)))
 NBT = args.B * args.T
 
+def obj_loss(logits2d, y1d):
+    """train objective: CE (+ optional z-loss). Used in the nudge force, theta-readout and bp_gate
+    so EP tracks BP on the SAME objective; evaluate() stays pure CE for comparability."""
+    l = F.cross_entropy(logits2d, y1d)
+    if args.zloss > 0:
+        l = l + args.zloss * (torch.logsumexp(logits2d.float(), -1) ** 2).mean()
+    return l
+
 def free_states_graphed(x):
     """free forward, keeping per-layer graphs (in_l, out_l) so round-1 backward vjps reuse them."""
     with torch.no_grad():
-        z0 = (tok(x) + pos(torch.arange(args.T, device=dev))[None])
+        z0 = emb(x)
     ins, outs, zs = [], [], []
     prev = z0
     for b in blocks:
@@ -154,7 +233,7 @@ def relax(z0, zs, ins, outs, y, beta, K, x):
     for k in range(K):
         if k % args.dtop_every == 0 or d[args.L - 1] is None:
             zc = zs[args.L - 1].detach().requires_grad_(True)
-            ce = F.cross_entropy(readout(zc).reshape(-1, vocab), y.reshape(-1))
+            ce = obj_loss(readout(zc).reshape(-1, vocab), y.reshape(-1))
             d[args.L - 1] = (-beta * NBT * torch.autograd.grad(ce, zc)[0]).detach()
         for l in range(args.L - 2, -1, -1):
             d[l] = torch.autograd.grad(outs[l + 1], ins[l + 1], grad_outputs=d[l + 1])[0].detach()
@@ -163,7 +242,7 @@ def relax(z0, zs, ins, outs, y, beta, K, x):
         n_ins, n_outs = [], []
         for l in range(args.L):
             if last and l == 0:
-                i = tok(x) + pos(torch.arange(args.T, device=dev))[None]   # graphed emb for the readout's E-path
+                i = emb(x)   # graphed emb for the readout's E-path
             else:
                 i = prev.detach().requires_grad_(True)
             o = blocks[l](i, mask)
@@ -178,7 +257,7 @@ def dFdtheta(zs, x, y, beta):
     graphs); kept as the INVARIANT-TEST surface for test_bp_free.py. Self-sealing: inputs
     are detached here so the local-graph property holds for any caller."""
     zs = [z.detach() for z in zs]
-    prev = tok(x) + pos(torch.arange(args.T, device=dev))[None]
+    prev = emb(x)
     E = 0.0
     for z, b in zip(zs, blocks):
         E = E + 0.5 * ((z - b(prev, mask)) ** 2).sum()
@@ -217,7 +296,7 @@ def ep_step(x, y):
     GOV['drift'] = drift
     E = 0.0
     for z, o in zip(zp, last_outs): E = E + 0.5 * ((z.detach() - o) ** 2).sum()
-    obj = E / (NBT * beta_t) + F.cross_entropy(readout(zp[-1].detach()).reshape(-1, vocab), y.reshape(-1))
+    obj = E / (NBT * beta_t) + obj_loss(readout(zp[-1].detach()).reshape(-1, vocab), y.reshape(-1))
     gs = torch.autograd.grad(obj, all_params, allow_unused=True)
     gn = 0.0
     for g in gs:
@@ -236,9 +315,9 @@ def ep_step(x, y):
 
 def bp_gate(x, y):
     """true BP grads for telemetry cos (called before opt.step; reads p.grad separately)."""
-    z = tok(x) + pos(torch.arange(args.T, device=dev))[None]
+    z = emb(x)
     for b in blocks: z = b(z, mask)
-    ce = F.cross_entropy(readout(z).reshape(-1, vocab), y.reshape(-1))
+    ce = obj_loss(readout(z).reshape(-1, vocab), y.reshape(-1))
     return list(torch.autograd.grad(ce, all_params, allow_unused=True))
 
 @torch.no_grad()
@@ -246,7 +325,7 @@ def evaluate(nb=6):
     tot = 0.0
     for _ in range(nb):
         x, y = get_batch('val')
-        z = tok(x) + pos(torch.arange(args.T, device=dev))[None]
+        z = emb(x)
         for b in blocks: z = b(z, mask)
         tot += F.cross_entropy(readout(z).reshape(-1, vocab), y.reshape(-1)).item()
     return tot / nb
@@ -297,6 +376,8 @@ for step in range(start_step, args.steps + 1):
             except Exception: pass
     if step % args.save_every == 0 and step > 0:
         torch.save({'tok': tok.state_dict(), 'pos': pos.state_dict(), 'blocks': blocks.state_dict(),
+                    'wout': (W_out.detach().cpu() if args.untie else None),
+                    'lnf': (ln_f.state_dict() if not isinstance(ln_f, nn.Identity) else None),
                     'step': step, 'val': best, 'config': vars(args)}, Path('runs') / f'{args.tag}_s{step}.pt')
 print(f'[{args.tag}] DONE best val CE {best:.4f} (BP twin 2.9746; zil-diagnostic 3.3236)', flush=True)
 if wb is not None: