From 4e0d730765a8287cdd574dc39b8dafc16006b266 Mon Sep 17 00:00:00 2001 From: Yuren Hao Date: Fri, 10 Jul 2026 08:24:03 -0500 Subject: OLMo2-standard architecture (user directive): norm-after-sublayer RMSNorm blocks, full-width QK-norm, RoPE(500k), SwiGLU, no-bias, untied head, final RMSNorm, 0.02 init, grouped wd, optional z-loss (plumbed through nudge+readout+gate for EP exactness); EP smoke 42.75M cos=1.0000 Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014FAPDWQ49M5Ye3NpTndTpn --- ep_run/casc_bp_train.py | 87 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 80 insertions(+), 7 deletions(-) (limited to 'ep_run/casc_bp_train.py') diff --git a/ep_run/casc_bp_train.py b/ep_run/casc_bp_train.py index 069d858..987277f 100644 --- a/ep_run/casc_bp_train.py +++ b/ep_run/casc_bp_train.py @@ -22,7 +22,11 @@ ap.add_argument('--lr_min_ratio', type=float, default=0.1) ap.add_argument('--qk_norm', action='store_true') # RMS-norm q,k per head before scores (OLMo2-style; bounds logits, analog-friendly) ap.add_argument('--final_ln', action='store_true') # final LayerNorm before readout (standard GPT; bounds sig_tok growth -> keeps beta/estimator healthy on long runs) ap.add_argument('--resume', default='') # path to a ckpt (tok/pos/blocks) to continue from; step taken from ckpt +ap.add_argument('--olmo2', action='store_true') # OLMo2-standard block (see casc_eq_train.py) +ap.add_argument('--wd', type=float, default=-1.0) # >=0: grouped weight decay; <0 = legacy uniform 1e-4 +ap.add_argument('--zloss', type=float, default=0.0) # z-loss coefficient; 0 = off args = ap.parse_args() +if args.olmo2 and args.tok_init <= 0: args.tok_init = 0.02 torch.manual_seed(args.seed) dev = 'cuda' if torch.cuda.is_available() else 'cpu' @@ -68,26 +72,90 @@ class Block(nn.Module): z = z + self.attn(self.ln1(z)) return z + self.ff(self.ln2(z)) +class RMSNorm(nn.Module): + def __init__(self, C, eps=1e-6): + super().__init__(); self.g = nn.Parameter(torch.ones(C)); self.eps = eps + def forward(self, x): + return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) * self.g + +class SwiGLU(nn.Module): + def __init__(self, C): + super().__init__() + h = ((8 * C // 3) + 63) // 64 * 64 # ~param-match the 4x-GELU MLP (8C^2) + self.w1 = nn.Linear(C, h, bias=False); self.w3 = nn.Linear(C, h, bias=False) + self.w2 = nn.Linear(h, C, bias=False) + def forward(self, x): return self.w2(F.silu(self.w1(x)) * self.w3(x)) + +class Olmo2Attn(nn.Module): + """OLMo2 attention: no-bias projs, FULL-WIDTH RMS QK-norm (pre-head-split, HF Olmo2 order), then per-head RoPE.""" + def __init__(self, C, H, T): + super().__init__() + self.H, self.hd = H, C // H + self.qkv = nn.Linear(C, 3 * C, bias=False); self.proj = nn.Linear(C, C, bias=False) + self.qn, self.kn = RMSNorm(C), RMSNorm(C) + inv = 1.0 / (500000.0 ** (torch.arange(0, self.hd, 2).float() / self.hd)) + fr = torch.outer(torch.arange(T).float(), inv) + self.register_buffer('rc', fr.cos(), persistent=False) + self.register_buffer('rs', fr.sin(), persistent=False) + def rope(self, x): + x1, x2 = x[..., ::2], x[..., 1::2] + c, s = self.rc[None, None], self.rs[None, None] + return torch.stack((x1 * c - x2 * s, x1 * s + x2 * c), dim=-1).flatten(-2) + def forward(self, x): + B, T, C = x.shape + q, k, v = self.qkv(x).split(C, dim=2) + q, k = self.qn(q), self.kn(k) + q = self.rope(q.view(B, T, self.H, self.hd).transpose(1, 2)) + k = self.rope(k.view(B, T, self.H, self.hd).transpose(1, 2)) + v = v.view(B, T, self.H, self.hd).transpose(1, 2) + y = F.scaled_dot_product_attention(q, k, v, is_causal=True) + return self.proj(y.transpose(1, 2).contiguous().view(B, T, C)) + +class Olmo2Block(nn.Module): + """OLMo2 reordered norm (norm AFTER each sublayer, inside the residual) — their training-stability change.""" + def __init__(self, C, H, T): + super().__init__() + self.attn = Olmo2Attn(C, H, T); self.ff = SwiGLU(C) + self.na, self.nf = RMSNorm(C), RMSNorm(C) + def forward(self, z, mask=None): + z = z + self.na(self.attn(z)) + return z + self.nf(self.ff(z)) + tok = nn.Embedding(vocab, args.C).to(dev) pos = nn.Embedding(args.T, args.C).to(dev) if args.tok_init > 0: with torch.no_grad(): tok.weight.normal_(0, args.tok_init); pos.weight.normal_(0, args.tok_init) -blocks = nn.ModuleList([Block(args.C, args.H, args.qk_norm) for _ in range(args.L)]).to(dev) +blocks = nn.ModuleList([(Olmo2Block(args.C, args.H, args.T) if args.olmo2 else Block(args.C, args.H, args.qk_norm)) for _ in range(args.L)]).to(dev) +if args.olmo2: + with torch.no_grad(): + for m in blocks.modules(): + if isinstance(m, nn.Linear): m.weight.normal_(0, 0.02) +W_out = nn.Parameter(torch.randn(vocab, args.C, device=dev) * 0.02) if args.olmo2 else None mask = torch.triu(torch.full((args.T, args.T), float('-inf'), device=dev), 1) -ln_f = nn.LayerNorm(args.C).to(dev) if args.final_ln else nn.Identity() -params = list(tok.parameters()) + list(pos.parameters()) + list(blocks.parameters()) + list(ln_f.parameters()) +ln_f = (RMSNorm(args.C) if args.olmo2 else (nn.LayerNorm(args.C) if args.final_ln else nn.Identity())).to(dev) +params = list(tok.parameters()) + ([] if args.olmo2 else list(pos.parameters())) + list(blocks.parameters()) + list(ln_f.parameters()) + ([W_out] if args.olmo2 else []) start_step = 0 if args.resume: _ck = torch.load(args.resume, map_location=dev, weights_only=False) tok.load_state_dict(_ck['tok']); pos.load_state_dict(_ck['pos']); blocks.load_state_dict(_ck['blocks']) + if _ck.get('wout') is not None and args.olmo2: + with torch.no_grad(): W_out.copy_(_ck['wout'].to(dev)) + if _ck.get('lnf') is not None and not isinstance(ln_f, nn.Identity): ln_f.load_state_dict(_ck['lnf']) start_step = int(_ck.get('step', 0)) print(f'[resume] loaded {args.resume} at step {start_step}', flush=True) if args.opt == 'muon': from muon import build_hybrid opt, sched = build_hybrid(blocks, params, args.lr, args.muon_lr, args.warmup) else: - opt = torch.optim.AdamW(params, lr=args.lr, weight_decay=1e-4) + if args.wd >= 0: # OLMo2-style grouped decay + nodecay = {id(p) for p in tok.parameters()} | {id(p) for p in pos.parameters()} | \ + {id(p) for p in blocks.parameters() if p.ndim < 2} | {id(p) for p in ln_f.parameters()} + opt = torch.optim.AdamW([ + {'params': [p for p in params if id(p) not in nodecay], 'weight_decay': args.wd}, + {'params': [p for p in params if id(p) in nodecay], 'weight_decay': 0.0}], lr=args.lr) + else: + opt = torch.optim.AdamW(params, lr=args.lr, weight_decay=1e-4) if args.cosine: def _lrlam(s): if s < args.warmup: return (s + 1) / max(args.warmup, 1) @@ -98,9 +166,9 @@ else: sched = torch.optim.lr_scheduler.LambdaLR(opt, lambda s: min(1.0, (s + 1) / max(args.warmup, 1))) def fwd(x): - z = tok(x) + pos(torch.arange(args.T, device=dev))[None] + z = tok(x) if args.olmo2 else tok(x) + pos(torch.arange(args.T, device=dev))[None] for b in blocks: z = b(z, mask) - return ln_f(z) @ tok.weight.t() + return ln_f(z) @ (W_out.t() if args.olmo2 else tok.weight.t()) @torch.no_grad() def evaluate(nb=6): @@ -126,7 +194,10 @@ outdir = Path('runs'); outdir.mkdir(exist_ok=True) for _ in range(start_step): sched.step() # advance LR schedule to the resumed step for step in range(start_step, args.steps + 1): x, y = get_batch('train') - loss = F.cross_entropy(fwd(x).reshape(-1, vocab), y.reshape(-1)) + logits = fwd(x).reshape(-1, vocab) + loss = F.cross_entropy(logits, y.reshape(-1)) + if args.zloss > 0: + loss = loss + args.zloss * (torch.logsumexp(logits.float(), -1) ** 2).mean() opt.zero_grad(set_to_none=True); loss.backward() torch.nn.utils.clip_grad_norm_(params, 1.0) opt.step(); sched.step() @@ -139,6 +210,8 @@ for step in range(start_step, args.steps + 1): except Exception: pass if step % args.save_every == 0: torch.save({'tok': tok.state_dict(), 'pos': pos.state_dict(), 'blocks': blocks.state_dict(), + 'wout': (W_out.detach().cpu() if args.olmo2 else None), + 'lnf': (ln_f.state_dict() if not isinstance(ln_f, nn.Identity) else None), 'step': step, 'val': best, 'config': vars(args)}, outdir / f'{args.tag}_s{step}.pt') print(f'[{args.tag}] DONE best val CE {best:.4f} (random ln({vocab})={math.log(vocab):.3f})', flush=True) if wb is not None: -- cgit v1.2.3