path: root/ep_run/casc_bp_train.py

"""BP-train a small cascade-form standard transformer (L distinct blocks), saving ckpts
every --save_every for the A0.2 on-trajectory gradient gate (cascade_probe.py --ckpt).
Plain LLM training — this is also the BP twin for the C-tier money runs."""
import argparse, math, pickle, time, json
import numpy as np, torch, torch.nn as nn, torch.nn.functional as F
from pathlib import Path

ap = argparse.ArgumentParser()
ap.add_argument('--tag', default='casc_bp6')
ap.add_argument('--L', type=int, default=6); ap.add_argument('--C', type=int, default=256)
ap.add_argument('--H', type=int, default=8); ap.add_argument('--T', type=int, default=256)
ap.add_argument('--B', type=int, default=24); ap.add_argument('--steps', type=int, default=4000)
ap.add_argument('--lr', type=float, default=3e-4); ap.add_argument('--warmup', type=int, default=200)
ap.add_argument('--seed', type=int, default=0)
ap.add_argument('--save_every', type=int, default=500); ap.add_argument('--log', type=int, default=200)
ap.add_argument('--wandb', default='ept-cascade')   # ON BY DEFAULT (user directive 07-13); --wandb '' to disable
ap.add_argument('--wandb_run', default='')
ap.add_argument('--amp', action='store_true')   # bf16 autocast fwd/loss (no scaler needed for bf16)
ap.add_argument('--opt', choices=['adamw', 'muon'], default='adamw')
ap.add_argument('--muon_lr', type=float, default=0.02)
ap.add_argument('--tok_init', type=float, default=0.0)  # >0: init tok/pos std (GPT-standard 0.02)
ap.add_argument('--cosine', action='store_true')         # warmup then cosine decay to lr_min_ratio*lr over --steps (long runs)
ap.add_argument('--lr_min_ratio', type=float, default=0.1)
ap.add_argument('--qk_norm', action='store_true')        # RMS-norm q,k per head before scores (OLMo2-style; bounds logits, analog-friendly)
ap.add_argument('--final_ln', action='store_true')       # final LayerNorm before readout (standard GPT; bounds sig_tok growth -> keeps beta/estimator healthy on long runs)
ap.add_argument('--resume', default='')                  # path to a ckpt (tok/pos/blocks) to continue from; step taken from ckpt
ap.add_argument('--olmo2', action='store_true')          # OLMo2-standard block (see casc_eq_train.py)
ap.add_argument('--wd', type=float, default=-1.0)        # >=0: grouped weight decay; <0 = legacy uniform 1e-4
ap.add_argument('--zloss', type=float, default=0.0)      # z-loss coefficient; 0 = off
ap.add_argument('--data', default='tinystories_bpe')     # dataset dir under ep_run/data
args = ap.parse_args()
if args.olmo2 and args.tok_init <= 0: args.tok_init = 0.02
torch.manual_seed(args.seed)
dev = 'cuda' if torch.cuda.is_available() else 'cpu'

DD = Path('/home/yurenh2/ept/ep_run/data') / args.data
vocab = pickle.load(open(DD / 'meta.pkl', 'rb'))['vocab_size']

def get_batch(split):
    data = np.memmap(DD / ('train.bin' if split == 'train' else 'val.bin'), dtype=np.uint16, mode='r')
    ix = torch.randint(len(data) - args.T - 1, (args.B,))
    x = torch.stack([torch.from_numpy(data[i:i + args.T].astype(np.int64)) for i in ix])
    y = torch.stack([torch.from_numpy(data[i + 1:i + 1 + args.T].astype(np.int64)) for i in ix])
    return x.to(dev), y.to(dev)

class CausalSelfAttn(nn.Module):
    """explicit MHA (SDPA-backed) so we can QK-norm q,k per head before the scores."""
    def __init__(self, C, H, qk_norm=False):
        super().__init__()
        self.H, self.hd, self.qk_norm = H, C // H, qk_norm
        self.qkv = nn.Linear(C, 3 * C)
        self.proj = nn.Linear(C, C)
        if qk_norm:
            self.q_g = nn.Parameter(torch.ones(self.hd))
            self.k_g = nn.Parameter(torch.ones(self.hd))
    def forward(self, x):
        B, T, C = x.shape
        q, k, v = self.qkv(x).split(C, dim=2)
        q = q.view(B, T, self.H, self.hd).transpose(1, 2)
        k = k.view(B, T, self.H, self.hd).transpose(1, 2)
        v = v.view(B, T, self.H, self.hd).transpose(1, 2)
        if self.qk_norm:  # RMS-norm over head_dim (OLMo2-style), learnable per-dim gain
            q = q * torch.rsqrt(q.pow(2).mean(-1, keepdim=True) + 1e-6) * self.q_g
            k = k * torch.rsqrt(k.pow(2).mean(-1, keepdim=True) + 1e-6) * self.k_g
        y = F.scaled_dot_product_attention(q, k, v, is_causal=True)
        return self.proj(y.transpose(1, 2).contiguous().view(B, T, C))

class Block(nn.Module):
    def __init__(self, C, H, qk_norm=False):
        super().__init__()
        self.ln1, self.ln2 = nn.LayerNorm(C), nn.LayerNorm(C)
        self.attn = CausalSelfAttn(C, H, qk_norm)
        self.ff = nn.Sequential(nn.Linear(C, 4 * C), nn.GELU(), nn.Linear(4 * C, C))
    def forward(self, z, mask=None):
        z = z + self.attn(self.ln1(z))
        return z + self.ff(self.ln2(z))

class RMSNorm(nn.Module):
    def __init__(self, C, eps=1e-6):
        super().__init__(); self.g = nn.Parameter(torch.ones(C)); self.eps = eps
    def forward(self, x):
        return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) * self.g

class SwiGLU(nn.Module):
    def __init__(self, C):
        super().__init__()
        h = ((8 * C // 3) + 63) // 64 * 64   # ~param-match the 4x-GELU MLP (8C^2)
        self.w1 = nn.Linear(C, h, bias=False); self.w3 = nn.Linear(C, h, bias=False)
        self.w2 = nn.Linear(h, C, bias=False)
    def forward(self, x): return self.w2(F.silu(self.w1(x)) * self.w3(x))

class Olmo2Attn(nn.Module):
    """OLMo2 attention: no-bias projs, FULL-WIDTH RMS QK-norm (pre-head-split, HF Olmo2 order), then per-head RoPE."""
    def __init__(self, C, H, T):
        super().__init__()
        self.H, self.hd = H, C // H
        self.qkv = nn.Linear(C, 3 * C, bias=False); self.proj = nn.Linear(C, C, bias=False)
        self.qn, self.kn = RMSNorm(C), RMSNorm(C)
        inv = 1.0 / (500000.0 ** (torch.arange(0, self.hd, 2).float() / self.hd))
        fr = torch.outer(torch.arange(T).float(), inv)
        self.register_buffer('rc', fr.cos(), persistent=False)
        self.register_buffer('rs', fr.sin(), persistent=False)
    def rope(self, x):
        x1, x2 = x[..., ::2], x[..., 1::2]
        c, s = self.rc[None, None], self.rs[None, None]
        return torch.stack((x1 * c - x2 * s, x1 * s + x2 * c), dim=-1).flatten(-2)
    def forward(self, x):
        B, T, C = x.shape
        q, k, v = self.qkv(x).split(C, dim=2)
        q, k = self.qn(q), self.kn(k)
        q = self.rope(q.view(B, T, self.H, self.hd).transpose(1, 2))
        k = self.rope(k.view(B, T, self.H, self.hd).transpose(1, 2))
        v = v.view(B, T, self.H, self.hd).transpose(1, 2)
        y = F.scaled_dot_product_attention(q, k, v, is_causal=True)
        return self.proj(y.transpose(1, 2).contiguous().view(B, T, C))

class Olmo2Block(nn.Module):
    """OLMo2 reordered norm (norm AFTER each sublayer, inside the residual) — their training-stability change."""
    def __init__(self, C, H, T):
        super().__init__()
        self.attn = Olmo2Attn(C, H, T); self.ff = SwiGLU(C)
        self.na, self.nf = RMSNorm(C), RMSNorm(C)
    def forward(self, z, mask=None):
        z = z + self.na(self.attn(z))
        return z + self.nf(self.ff(z))

tok = nn.Embedding(vocab, args.C).to(dev)
pos = nn.Embedding(args.T, args.C).to(dev)
if args.tok_init > 0:
    with torch.no_grad():
        tok.weight.normal_(0, args.tok_init); pos.weight.normal_(0, args.tok_init)
blocks = nn.ModuleList([(Olmo2Block(args.C, args.H, args.T) if args.olmo2 else Block(args.C, args.H, args.qk_norm)) for _ in range(args.L)]).to(dev)
if args.olmo2:
    with torch.no_grad():
        for m in blocks.modules():
            if isinstance(m, nn.Linear): m.weight.normal_(0, 0.02)
W_out = nn.Parameter(torch.randn(vocab, args.C, device=dev) * 0.02) if args.olmo2 else None
mask = torch.triu(torch.full((args.T, args.T), float('-inf'), device=dev), 1)
ln_f = (RMSNorm(args.C) if args.olmo2 else (nn.LayerNorm(args.C) if args.final_ln else nn.Identity())).to(dev)
params = list(tok.parameters()) + ([] if args.olmo2 else list(pos.parameters())) + list(blocks.parameters()) + list(ln_f.parameters()) + ([W_out] if args.olmo2 else [])
start_step = 0
if args.resume:
    _ck = torch.load(args.resume, map_location=dev, weights_only=False)
    tok.load_state_dict(_ck['tok']); pos.load_state_dict(_ck['pos']); blocks.load_state_dict(_ck['blocks'])
    if _ck.get('wout') is not None and args.olmo2:
        with torch.no_grad(): W_out.copy_(_ck['wout'].to(dev))
    if _ck.get('lnf') is not None and not isinstance(ln_f, nn.Identity): ln_f.load_state_dict(_ck['lnf'])
    start_step = int(_ck.get('step', 0))
    print(f'[resume] loaded {args.resume} at step {start_step}', flush=True)
if args.opt == 'muon':
    from muon import build_hybrid
    opt, sched = build_hybrid(blocks, params, args.lr, args.muon_lr, args.warmup,
                              total_steps=(args.steps if args.cosine else 0), lr_min_ratio=args.lr_min_ratio)
else:
    if args.wd >= 0:   # OLMo2-style grouped decay
        nodecay = {id(p) for p in tok.parameters()} | {id(p) for p in pos.parameters()} | \
                  {id(p) for p in blocks.parameters() if p.ndim < 2} | {id(p) for p in ln_f.parameters()}
        opt = torch.optim.AdamW([
            {'params': [p for p in params if id(p) not in nodecay], 'weight_decay': args.wd},
            {'params': [p for p in params if id(p) in nodecay], 'weight_decay': 0.0}], lr=args.lr)
    else:
        opt = torch.optim.AdamW(params, lr=args.lr, weight_decay=1e-4)
    if args.cosine:
        def _lrlam(s):
            if s < args.warmup: return (s + 1) / max(args.warmup, 1)
            p = min(1.0, (s - args.warmup) / max(1, args.steps - args.warmup))
            return args.lr_min_ratio + 0.5 * (1 - args.lr_min_ratio) * (1 + math.cos(math.pi * p))
        sched = torch.optim.lr_scheduler.LambdaLR(opt, _lrlam)
    else:
        sched = torch.optim.lr_scheduler.LambdaLR(opt, lambda s: min(1.0, (s + 1) / max(args.warmup, 1)))

def fwd(x):
    z = tok(x) if args.olmo2 else tok(x) + pos(torch.arange(args.T, device=dev))[None]
    for b in blocks: z = b(z, mask)
    return ln_f(z) @ (W_out.t() if args.olmo2 else tok.weight.t())

@torch.no_grad()
def evaluate(nb=6):
    tot = 0.0
    for _ in range(nb):
        x, y = get_batch('val')
        tot += F.cross_entropy(fwd(x).reshape(-1, vocab), y.reshape(-1)).item()
    return tot / nb

wb = None
if args.wandb:
    try:
        import wandb as _w
        wb = _w.init(entity='eqprop-llm-training', project=args.wandb, name=args.wandb_run or args.tag, id=args.wandb_run or args.tag,
                     resume='allow', config=vars(args))
    except Exception as e:
        print(f'[wandb] disabled ({e})', flush=True)

n = sum(p.numel() for p in params)
print(f'[{args.tag}] cascade-BP L{args.L} C{args.C} H{args.H} T{args.T} | {n/1e6:.2f}M params | {dev}', flush=True)
best, t0 = 1e9, time.time()
outdir = Path('runs'); outdir.mkdir(exist_ok=True)
for _ in range(start_step): sched.step()   # advance LR schedule to the resumed step
for step in range(start_step, args.steps + 1):
    x, y = get_batch('train')
    with torch.autocast('cuda', dtype=torch.bfloat16, enabled=args.amp):
        logits = fwd(x).reshape(-1, vocab)
        loss = F.cross_entropy(logits, y.reshape(-1))
        if args.zloss > 0:
            loss = loss + args.zloss * (torch.logsumexp(logits.float(), -1) ** 2).mean()
    opt.zero_grad(set_to_none=True); loss.backward()
    torch.nn.utils.clip_grad_norm_(params, 1.0)
    opt.step(); sched.step()
    if step % args.log == 0:
        val = evaluate(); best = min(best, val)
        print(f'step {step:5d}/{args.steps} | train {loss.item():.4f} val {val:.4f} (best {best:.4f}) '
              f'| {step/max(time.time()-t0,1e-9):.2f} it/s', flush=True)
        if wb is not None:
            try: wb.log({'train_ce': loss.item(), 'val_ce': val, 'best': best}, step=step)
            except Exception: pass
    if step % args.save_every == 0:
        torch.save({'tok': tok.state_dict(), 'pos': pos.state_dict(), 'blocks': blocks.state_dict(),
                    'wout': (W_out.detach().cpu() if args.olmo2 else None),
                    'lnf': (ln_f.state_dict() if not isinstance(ln_f, nn.Identity) else None),
                    'step': step, 'val': best, 'config': vars(args)}, outdir / f'{args.tag}_s{step}.pt')
print(f'[{args.tag}] DONE best val CE {best:.4f} (random ln({vocab})={math.log(vocab):.3f})', flush=True)
if wb is not None:
    try: wb.summary['best_val_ce'] = best; wb.finish()
    except Exception: pass