From 82da86dbd49a093a24e0827331667e8e9e6217df Mon Sep 17 00:00:00 2001 From: Yuren Hao Date: Wed, 15 Jul 2026 10:29:06 -0500 Subject: Alexi primer deck: 10 slides (arch op-by-op, EP phases, schedule/parallelism, 72M blowup from real logs, loop-gain + beta-window hypotheses, diagnostics table, in-flight outcomes, $50k ladder) + figure scripts Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014FAPDWQ49M5Ye3NpTndTpn --- ep_run/build_alexi_deck.py | 159 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 159 insertions(+) create mode 100644 ep_run/build_alexi_deck.py (limited to 'ep_run/build_alexi_deck.py') diff --git a/ep_run/build_alexi_deck.py b/ep_run/build_alexi_deck.py new file mode 100644 index 0000000..461b8fb --- /dev/null +++ b/ep_run/build_alexi_deck.py @@ -0,0 +1,159 @@ +"""Alexi technical-primer deck: architecture -> EP update -> schedule -> problems -> +hypotheses -> diagnostics -> likely outcomes -> $50k plan. 16:9, minimal styling.""" +from pptx import Presentation +from pptx.util import Inches, Pt +from pptx.dml.color import RGBColor +from pptx.enum.text import PP_ALIGN + +INK = RGBColor(0x3a, 0x3a, 0x3a); GRAY = RGBColor(0x8a, 0x8a, 0x8a) +BLUE = RGBColor(0x2c, 0x6f, 0xbb); ORAN = RGBColor(0xd9, 0x5f, 0x02) +RED = RGBColor(0xb0, 0x3a, 0x2e); GREEN = RGBColor(0x2e, 0x7d, 0x32) +A = '/home/yurenh2/ept/assets/' + +prs = Presentation() +prs.slide_width = Inches(13.333); prs.slide_height = Inches(7.5) +BLANK = prs.slide_layouts[6] + +def slide(): + return prs.slides.add_slide(BLANK) + +def tbox(s, x, y, w, h): + tb = s.shapes.add_textbox(Inches(x), Inches(y), Inches(w), Inches(h)) + tb.text_frame.word_wrap = True + return tb.text_frame + +def title(s, text, color=INK): + tf = tbox(s, 0.45, 0.22, 12.5, 0.75) + p = tf.paragraphs[0]; r = p.add_run(); r.text = text + r.font.size = Pt(25); r.font.bold = True; r.font.color.rgb = color + +def para(tf, text, size=13, color=INK, bold=False, before=6, bullet=True, first=False): + p = tf.paragraphs[0] if first else tf.add_paragraph() + p.space_before = Pt(0 if first else before) + r = p.add_run(); r.text = ('• ' if bullet else '') + text + r.font.size = Pt(size); r.font.color.rgb = color; r.font.bold = bold + return p + +# ---------- 1 cover ---------- +s = slide() +tf = tbox(s, 0.9, 2.35, 11.5, 2.6) +p = tf.paragraphs[0]; r = p.add_run() +r.text = 'Training transformer LMs without backpropagation' +r.font.size = Pt(34); r.font.bold = True; r.font.color.rgb = INK +p2 = tf.add_paragraph(); p2.space_before = Pt(14); r = p2.add_run() +r.text = 'Standard architecture at inference · equilibrium-propagation gradients in training · GPU now, analog next' +r.font.size = Pt(17); r.font.color.rgb = GRAY +p3 = tf.add_paragraph(); p3.space_before = Pt(30); r = p3.add_run() +r.text = 'Yuren Hao · July 2026 · technical primer' +r.font.size = Pt(14); r.font.color.rgb = INK + +# ---------- 2 architecture ---------- +s = slide(); title(s, 'The model is a stock OLMo2 decoder — op by op') +s.shapes.add_picture(A + 'fig_primer_block.png', Inches(0.5), Inches(1.0), height=Inches(6.3)) +tf = tbox(s, 5.6, 1.35, 7.3, 5.8) +para(tf, 'Decoder-only transformer: L12, C512, H8, T256. RMSNorm, RoPE, QK-norm, SwiGLU, untied readout. No biases.', 14, first=True) +para(tf, 'Two trained instances: 42.75M (TinyStories, 4k BPE) and 72.11M (FineWeb-Edu, 32k BPE).', 14) +para(tf, 'Nothing in the inference graph is modified for our training method — the checkpoint is indistinguishable in form from a conventionally trained model.', 14, bold=True) +para(tf, 'Color code, kept through the whole deck: blue = weight×activation matmuls (7 logical — crossbar-mappable on analog hardware); purple = activation×activation (qkᵀ, Av — must be computed on the fly); cream = RMSNorm; gray = elementwise.', 13, GRAY) +para(tf, 'One honest architecture–method coupling: QK-norm and the final RMSNorm bound internal state scales, which the training estimator relies on. Both are stock components.', 13, GRAY) + +# ---------- 3 EP phases ---------- +s = slide(); title(s, 'Training = two settles and a difference measurement') +s.shapes.add_picture(A + 'fig_primer_phases.png', Inches(0.35), Inches(1.05), height=Inches(6.1)) +tf = tbox(s, 7.35, 1.5, 5.6, 5.6) +para(tf, 'The stack is treated as a system that settles: each layer gets a state zₗ, and the energy penalizes disagreement between zₗ and fₗ(zₗ₋₁).', 14, first=True) +para(tf, 'Free phase: settle with no label. The exact minimum IS the ordinary forward pass (E = 0). Inference is untouched.', 14) +para(tf, 'Nudged phase: weights frozen, states move — the loss pulls the top state with strength β, the whole stack re-settles, every layer picks up a mismatch dₗ.', 14) +para(tf, 'Update: states frozen, weights move — each layer absorbs its own mismatch: Δθₗ ∝ ⟨dₗ, ∂fₗ/∂θₗ⟩/β. Local quantities only; no global tape.', 14) +para(tf, 'The nudged state becomes the new free state: today’s pulled answer is tomorrow’s natural forward output.', 14, ORAN, bold=True) + +# ---------- 4 schedule ---------- +s = slide(); title(s, 'The update schedule: local, pipelinable — and native to physics') +s.shapes.add_picture(A + 'fig_p3_schedule.png', Inches(0.45), Inches(1.15), width=Inches(12.4)) +tf = tbox(s, 0.7, 6.55, 12.0, 0.8) +para(tf, 'Everything that follows is the price of emulating this settle with discrete sweeps on a GPU. Two problems fall out — the physical system has neither.', 14, RED, bold=True, first=True, bullet=False) + +# ---------- 5 what happened ---------- +s = slide(); title(s, 'What actually happened at 72M (real run, real logs)') +s.shapes.add_picture(A + 'fig_p4_blowup.png', Inches(0.4), Inches(1.1), height=Inches(5.5)) +tf = tbox(s, 9.05, 1.35, 4.0, 5.9) +para(tf, 'Same β = 1e-3: safe for 135k steps, lethal at 195k. Something moved under the schedule.', 13.5, RED, bold=True, first=True) +para(tf, 'Guards contained it (15k step-skips, no NaN), a governor finished the token budget — but β starved to 2e-5 and best never improved after 185k.', 13) +para(tf, 'Headline stands: 72.11M × 1.44B tokens fully BP-free, best 3.7117 vs BP twin 3.2884.', 13) +para(tf, 'The 0.43 gap is a BLOWN-SCHEDULE number, not a method number. The honest rerun is in flight (last slide but one).', 13, bold=True) + +# ---------- 6 problem 1 ---------- +s = slide(); title(s, 'Problem 1 — the nudge closes a loop; discrete sweeps can amplify it') +s.shapes.add_picture(A + 'fig_p5_loop.png', Inches(1.35), Inches(1.15), height=Inches(5.6)) +tf = tbox(s, 0.7, 6.85, 12.0, 0.55) +para(tf, 'Why the SAME β became lethal: ρ ≈ β·σ(Wₒᵤₜ)²·‖J‖² crossed 1 as σ and ‖J‖ grew with fit depth.', 14, first=True, bullet=False) + +# ---------- 7 problem 2 ---------- +s = slide(); title(s, 'Problem 2 — β is squeezed from both sides, and the window narrows') +s.shapes.add_picture(A + 'fig_p6_window.png', Inches(0.4), Inches(1.1), height=Inches(5.5)) +tf = tbox(s, 9.05, 1.3, 4.0, 6.0) +para(tf, 'Floor is real and RISING: on the 42M testbed, tail at flat 1e-3 beats decaying to 3e-4 (1.2678 vs 1.2808). Momentum as a substitute for β refuted in 3 doses.', 13, first=True) +para(tf, 'Ceiling is real and FALLING: measured crossings at 195k (≤1e-3), 222k (≤3e-4), late (~2e-5).', 13) +para(tf, 'Window-aware recipe: centered ±β estimator (bias O(β²) → ride high) + a loop-gain governor capping β near ITS ceiling.', 13, bold=True) +para(tf, 'fw72m_cent, in flight, is exactly this recipe.', 13, ORAN, bold=True) + +# ---------- 8 diagnostics ---------- +s = slide(); title(s, 'Diagnostics already done (every number is a run you can open in wandb)') +rows = [ + ('DDP gradient equivalence', 'cos 0.999999999 vs single-GPU big batch (manual all-reduce, synced guards)'), + ('centered + bf16 + DDP', 'cos 1.000000000 vs sequential centered; packed ±β variant identical (relerr 2e-5)'), + ('Error-source decomposition', 'within-block read EXACT (1.0000); ALL bias sits in between-block transmission (0.9987 = the whole EP deficit); K=1 degenerates to BP; K3 = K8 (converged, not truncated)'), + ('Wall-2 probes — 6 arms @72M', 'damping refuted in 3 doses; lowering β sails through; governor completes the budget'), + ('Tail-SNR arms — 7 @42M', 'centered@3e-3 wins CE and cos; ĝ-momentum refuted ×3; Richardson dominated; AdamW = tie'), + ('Gap-vs-size ladder — 8 runs, pre-registered', 'gap tracks FIT DEPTH, not width: 0.004 / 0.004 / 0.015 / 0.013 / 0.050 (C128→C512); C128 has gate cos 0.805 yet ZERO gap — direction noise alone costs nothing until the fit is deep'), + ('bf16 mixed precision', 'epoch-validated lossless (Δ +0.006), 1.56× wall-clock'), + ('Cost of centered', '1.72× step time (1.64× packed); tail-only switching amortizes to ~1.13×'), +] +tbl = s.shapes.add_table(len(rows) + 1, 2, Inches(0.45), Inches(1.15), Inches(12.45), Inches(5.9)).table +tbl.columns[0].width = Inches(3.6); tbl.columns[1].width = Inches(8.85) +hdr = ('diagnostic', 'verdict') +for j in range(2): + c = tbl.cell(0, j); c.text = hdr[j] + c.text_frame.paragraphs[0].runs[0].font.size = Pt(12) + c.text_frame.paragraphs[0].runs[0].font.bold = True +for i, (a, b) in enumerate(rows): + for j, t in enumerate((a, b)): + c = tbl.cell(i + 1, j); c.text = t + r = c.text_frame.paragraphs[0].runs[0] + r.font.size = Pt(10.5); r.font.bold = (j == 0) + r.font.color.rgb = INK + +# ---------- 9 likely outcomes ---------- +s = slide(); title(s, 'In flight right now — and what the outcomes would mean') +tf = tbox(s, 0.55, 1.2, 12.4, 6.0) +para(tf, 'stage1b_cent — 42M TinyStories, centered for the FULL epoch (at 30k/58.8k: best 1.3642, ahead of both originals at matched step).', 14, bold=True, first=True) +para(tf, 'reads: best ≤ ~1.24 ⇒ the 0.050 gap was a noise-recipe account and the zero-gap line extends to C512 · 1.25–1.26 ⇒ a residual needs window management · >1.27 ⇒ an unknown term (falsifier).', 13, GRAY) +para(tf, 'fw72m_cent — 72M FineWeb, the window-aware recipe from scratch (at 48k/234k: best 3.6890 — already below the original’s FULL-RUN best 3.7117, at 20% of the budget).', 14, bold=True, before=14) +para(tf, 'registered predictions: no 195k-style blow; honest gap vs BP twin lands 0.25–0.35 (current trajectory suggests better).', 13, GRAY) +para(tf, 'If both hold: the EP–BP gap is an ENGINEERING account — schedule + estimator — not an architectural tax. The frontier is late-phase signal-to-noise, and we hold validated levers for it.', 14, GREEN, bold=True, before=16) +para(tf, 'Queued next: 3 seeds at C256/C512 (certify the ladder tiers); the est-switch rule (when to start paying for centered); 300M × 6B.', 13, before=14) + +# ---------- 10 the $50k ---------- +s = slide(); title(s, 'How to spend the $50k — a laddered plan with kill-gates') +rows = [ + ('rung', 'est. cost', 'what it buys', 'gate to the next rung'), + ('300M × 6B tokens (rented H100, days)', '~$0.3k', 'recipe + window governor at 4× params; DDP at scale', 'gap & stability hold'), + ('1B × 10B', '~$5k', 'first B-class BP-free LM; the LAST affordable BP twin', 'gap ≤ target at 1B'), + ('7B × 20B (flagship)', '~$30k', 'headline result; compare to published baselines (no twin)', '—'), + ('reserve', '$10–15k', 'seeds, reruns, ablations, surprises', '—'), +] +tbl = s.shapes.add_table(len(rows), 4, Inches(0.45), Inches(1.25), Inches(12.45), Inches(4.2)).table +widths = (3.6, 1.3, 4.6, 2.95) +for j, w in enumerate(widths): tbl.columns[j].width = Inches(w) +for i, row in enumerate(rows): + for j, t in enumerate(row): + c = tbl.cell(i, j); c.text = t + r = c.text_frame.paragraphs[0].runs[0] + r.font.size = Pt(12 if i == 0 else 12.5); r.font.bold = (i == 0 or j == 0) + r.font.color.rgb = INK +tf = tbox(s, 0.55, 5.9, 12.4, 1.3) +para(tf, 'Prices (sourced, July 2026): market H100 $1.87–2.99/GPU·h; AWS p5e $4.97/GPU·h. All estimates include bf16 (1.56×) and EP ≈ 3.2× BP wall-clock; tail-only centered adds ~1.13×.', 12, GRAY, first=True, bullet=False) +para(tf, 'Alternative one-shot: 3B-Chinchilla ≈ $40k fits the envelope alone — but the ladder buys three publishable points and de-risks the flagship.', 12, GRAY, bullet=False) + +prs.save(A + 'alexi_primer_deck.pptx') +print('deck saved:', A + 'alexi_primer_deck.pptx') -- cgit v1.2.3