summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--docs/campaign/CASCADE_ABLATION_PLAN.md20
-rw-r--r--ep_run/casc_bp_train.py3
-rw-r--r--ep_run/casc_eq_train.py3
-rw-r--r--ep_run/casc_gen.py95
-rw-r--r--ep_run/muon.py14
5 files changed, 130 insertions, 5 deletions
diff --git a/docs/campaign/CASCADE_ABLATION_PLAN.md b/docs/campaign/CASCADE_ABLATION_PLAN.md
index ac602cf..b456ccd 100644
--- a/docs/campaign/CASCADE_ABLATION_PLAN.md
+++ b/docs/campaign/CASCADE_ABLATION_PLAN.md
@@ -464,3 +464,23 @@ UNTOUCHED (the control measuring erosion damage vs BP twins); PARALLEL bf1e3 con
from ckpt-40000 on the farm (floor 1e-3 for the remaining 18.8k steps). Endpoint comparison becomes a
clean quad: EP-control(3e-4) / EP-floor-lift(1e-3 from 40k) / BP-s1 / BP-s2 — quantifies BOTH the
erosion damage AND the fix's recovery in one shot.
+
+### RESULT 10 (2026-07-10 19:0x): EPOCH ENDPOINTS + "NENG KAN" GATE PASSED + stage1b (improved recipe) launched.
+**Epoch endpoints (58,800 steps / 361M tokens, OLMo2, AdamW):**
+| arm | best val CE |
+|---|---|
+| BP s1 / s2 | **1.2750 / 1.2509** |
+| EP (floor 3e-4 fixed) | **1.4802** (zero guard events end-to-end) |
+| EP bf1e3-cont (floor->1e-3 @40k) | 1.4835@46k, running to 58.8k |
+**EP-BP gap at epoch scale = +0.22** (was +0.03 at 4k): the late-SNR cos erosion (1.0 -> ~0.92-0.95)
+is a REAL, horizon-growing CE cost with fixed floor 3e-4. Mechanism + dial both established
+(dose-response); the improved recipe is designed to close this.
+**GENERATION GATE ("neng kan") PASSED:** casc_gen.py (new; plain-forward standard-LLM inference) from
+EP s55000: coherent multi-paragraph TinyStories — named characters, balanced-quote dialogue,
+cause-effect, emotional arc (minor charm-defects vs BP's tighter coherence, consistent with +0.22).
+**A 42.75M standard 12-layer transformer trained end-to-end WITHOUT backprop tells coherent stories;
+inference is a plain forward pass.** Task #15 demo artifact exists.
+**stage1b launched (the improved-recipe head-to-head):** stage1b_ep_muon (local GPU1: Muon + floor
+3e-4 + bf_late 1e-3@15k + kretry + cosine[now also on Muon via build_hybrid total_steps]) vs
+stage1b_bp_muon (farm GPU6: Muon + cosine). Expectation: EP ~1.25-1.35 (Muon -0.13 and erosion fix
+~-0.1+), BP+Muon anchor moves too. ~8h both.
diff --git a/ep_run/casc_bp_train.py b/ep_run/casc_bp_train.py
index 987277f..d45327b 100644
--- a/ep_run/casc_bp_train.py
+++ b/ep_run/casc_bp_train.py
@@ -146,7 +146,8 @@ if args.resume:
print(f'[resume] loaded {args.resume} at step {start_step}', flush=True)
if args.opt == 'muon':
from muon import build_hybrid
- opt, sched = build_hybrid(blocks, params, args.lr, args.muon_lr, args.warmup)
+ opt, sched = build_hybrid(blocks, params, args.lr, args.muon_lr, args.warmup,
+ total_steps=(args.steps if args.cosine else 0), lr_min_ratio=args.lr_min_ratio)
else:
if args.wd >= 0: # OLMo2-style grouped decay
nodecay = {id(p) for p in tok.parameters()} | {id(p) for p in pos.parameters()} | \
diff --git a/ep_run/casc_eq_train.py b/ep_run/casc_eq_train.py
index 6bd83f8..5a3e640 100644
--- a/ep_run/casc_eq_train.py
+++ b/ep_run/casc_eq_train.py
@@ -175,7 +175,8 @@ if args.resume:
print(f'[resume] loaded {args.resume} at step {start_step}', flush=True)
if args.opt == 'muon':
from muon import build_hybrid
- opt, sched = build_hybrid(blocks, all_params, args.lr, args.muon_lr, args.warmup)
+ opt, sched = build_hybrid(blocks, all_params, args.lr, args.muon_lr, args.warmup,
+ total_steps=(args.steps if args.cosine else 0), lr_min_ratio=args.lr_min_ratio)
else:
if args.wd >= 0: # OLMo2-style grouped decay: linear weights + head decay; embeddings/norm-gains none
nodecay = {id(p) for p in tok.parameters()} | {id(p) for p in pos.parameters()} | \
diff --git a/ep_run/casc_gen.py b/ep_run/casc_gen.py
new file mode 100644
index 0000000..e535df8
--- /dev/null
+++ b/ep_run/casc_gen.py
@@ -0,0 +1,95 @@
+"""Generate TinyStories samples from a cascade (OLMo2-arch) checkpoint — plain forward, standard LLM inference."""
+import argparse, pickle, sys
+import torch, torch.nn as nn, torch.nn.functional as F
+from pathlib import Path
+
+ap = argparse.ArgumentParser()
+ap.add_argument('--ckpt', required=True)
+ap.add_argument('--n', type=int, default=3)
+ap.add_argument('--len', type=int, default=180)
+ap.add_argument('--temp', type=float, default=0.8)
+ap.add_argument('--topk', type=int, default=40)
+ap.add_argument('--prompt', default='Once upon a time')
+args = ap.parse_args()
+
+DD = Path('/home/yurenh2/ept/ep_run/data/tinystories_bpe')
+vocab = pickle.load(open(DD / 'meta.pkl', 'rb'))['vocab_size']
+from tokenizers import Tokenizer
+tk = Tokenizer.from_file(str(DD / 'tokenizer.json'))
+
+class RMSNorm(nn.Module):
+ def __init__(self, C, eps=1e-6):
+ super().__init__(); self.g = nn.Parameter(torch.ones(C)); self.eps = eps
+ def forward(self, x):
+ return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) * self.g
+
+class SwiGLU(nn.Module):
+ def __init__(self, C):
+ super().__init__()
+ h = ((8 * C // 3) + 63) // 64 * 64
+ self.w1 = nn.Linear(C, h, bias=False); self.w3 = nn.Linear(C, h, bias=False)
+ self.w2 = nn.Linear(h, C, bias=False)
+ def forward(self, x): return self.w2(F.silu(self.w1(x)) * self.w3(x))
+
+class Olmo2Attn(nn.Module):
+ def __init__(self, C, H, T):
+ super().__init__()
+ self.H, self.hd = H, C // H
+ self.qkv = nn.Linear(C, 3 * C, bias=False); self.proj = nn.Linear(C, C, bias=False)
+ self.qn, self.kn = RMSNorm(C), RMSNorm(C)
+ inv = 1.0 / (500000.0 ** (torch.arange(0, self.hd, 2).float() / self.hd))
+ fr = torch.outer(torch.arange(T).float(), inv)
+ self.register_buffer('rc', fr.cos(), persistent=False)
+ self.register_buffer('rs', fr.sin(), persistent=False)
+ def rope(self, x):
+ T = x.shape[2]
+ x1, x2 = x[..., ::2], x[..., 1::2]
+ c, s = self.rc[None, None, :T], self.rs[None, None, :T]
+ return torch.stack((x1 * c - x2 * s, x1 * s + x2 * c), dim=-1).flatten(-2)
+ def forward(self, x):
+ B, T, C = x.shape
+ q, k, v = self.qkv(x).split(C, dim=2)
+ q, k = self.qn(q), self.kn(k)
+ q = self.rope(q.view(B, T, self.H, self.hd).transpose(1, 2))
+ k = self.rope(k.view(B, T, self.H, self.hd).transpose(1, 2))
+ v = v.view(B, T, self.H, self.hd).transpose(1, 2)
+ y = F.scaled_dot_product_attention(q, k, v, is_causal=True)
+ return self.proj(y.transpose(1, 2).contiguous().view(B, T, C))
+
+class Olmo2Block(nn.Module):
+ def __init__(self, C, H, T):
+ super().__init__()
+ self.attn = Olmo2Attn(C, H, T); self.ff = SwiGLU(C)
+ self.na, self.nf = RMSNorm(C), RMSNorm(C)
+ def forward(self, z):
+ z = z + self.na(self.attn(z))
+ return z + self.nf(self.ff(z))
+
+dev = 'cuda' if torch.cuda.is_available() else 'cpu'
+ck = torch.load(args.ckpt, map_location=dev, weights_only=False)
+cfg = ck['config']; C, H, T, L = cfg['C'], cfg['H'], cfg['T'], cfg['L']
+print(f"[gen] {args.ckpt} | step {ck.get('step')} val {ck.get('val'):.4f} | L{L} C{C}", flush=True)
+assert ck.get('wout') is not None, 'need untied head (olmo2 ckpt)'
+tok = nn.Embedding(vocab, C).to(dev); tok.load_state_dict(ck['tok'])
+blocks = nn.ModuleList([Olmo2Block(C, H, T) for _ in range(L)]).to(dev)
+blocks.load_state_dict(ck['blocks'])
+W_out = ck['wout'].to(dev)
+ln_f = RMSNorm(C).to(dev); ln_f.load_state_dict(ck['lnf'])
+for m in [tok, blocks, ln_f]: m.eval()
+
+@torch.no_grad()
+def gen_one(seed):
+ torch.manual_seed(seed)
+ ids = tk.encode(args.prompt).ids
+ for _ in range(args.len):
+ x = torch.tensor(ids[-T:], device=dev)[None]
+ z = tok(x)
+ for b in blocks: z = b(z)
+ logits = (ln_f(z[:, -1]) @ W_out.t()) / args.temp
+ v, _ = torch.topk(logits, args.topk)
+ logits[logits < v[:, -1:]] = -float('inf')
+ ids.append(int(torch.multinomial(F.softmax(logits, -1), 1)))
+ return tk.decode(ids)
+
+for i in range(args.n):
+ print(f'--- sample {i+1} ---'); print(gen_one(1234 + i), flush=True)
diff --git a/ep_run/muon.py b/ep_run/muon.py
index 5ac6fc6..3116030 100644
--- a/ep_run/muon.py
+++ b/ep_run/muon.py
@@ -52,13 +52,21 @@ class MultiSched:
for s in self.scheds: s.step()
-def build_hybrid(blocks, other_params, lr_adamw, lr_muon, warmup):
- """Muon(2D block matrices) + AdamW(everything else), with linear-warmup scheds for both."""
+def build_hybrid(blocks, other_params, lr_adamw, lr_muon, warmup, total_steps=0, lr_min_ratio=0.1):
+ """Muon(2D block matrices) + AdamW(everything else). Scheds: linear warmup, then cosine decay to
+ lr_min_ratio*peak if total_steps>0 (long runs), else constant after warmup (legacy)."""
+ import math as _m
mats = [p for p in blocks.parameters() if p.ndim == 2]
mat_ids = {id(p) for p in mats}
rest = [p for p in other_params if id(p) not in mat_ids]
om = Muon(mats, lr=lr_muon)
oa = torch.optim.AdamW(rest, lr=lr_adamw, weight_decay=1e-4)
- fn = lambda s: min(1.0, (s + 1) / max(warmup, 1))
+ if total_steps > 0:
+ def fn(s):
+ if s < warmup: return (s + 1) / max(warmup, 1)
+ p = min(1.0, (s - warmup) / max(1, total_steps - warmup))
+ return lr_min_ratio + 0.5 * (1 - lr_min_ratio) * (1 + _m.cos(_m.pi * p))
+ else:
+ fn = lambda s: min(1.0, (s + 1) / max(warmup, 1))
scheds = [torch.optim.lr_scheduler.LambdaLR(om, fn), torch.optim.lr_scheduler.LambdaLR(oa, fn)]
return MultiOpt([om, oa]), MultiSched(scheds)
generated by cgit v1.2.3 (git 2.25.1) at 2026-08-18 14:12:10 +0000