1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
|
"""Alexi technical-primer deck: architecture -> EP update -> schedule -> problems ->
hypotheses -> diagnostics -> likely outcomes -> $50k plan. 16:9, minimal styling."""
from pptx import Presentation
from pptx.util import Inches, Pt
from pptx.dml.color import RGBColor
from pptx.enum.text import PP_ALIGN
INK = RGBColor(0x3a, 0x3a, 0x3a); GRAY = RGBColor(0x8a, 0x8a, 0x8a)
BLUE = RGBColor(0x2c, 0x6f, 0xbb); ORAN = RGBColor(0xd9, 0x5f, 0x02)
RED = RGBColor(0xb0, 0x3a, 0x2e); GREEN = RGBColor(0x2e, 0x7d, 0x32)
A = '/home/yurenh2/ept/assets/figs/'
prs = Presentation()
prs.slide_width = Inches(13.333); prs.slide_height = Inches(7.5)
BLANK = prs.slide_layouts[6]
def slide():
return prs.slides.add_slide(BLANK)
def tbox(s, x, y, w, h):
tb = s.shapes.add_textbox(Inches(x), Inches(y), Inches(w), Inches(h))
tb.text_frame.word_wrap = True
return tb.text_frame
def title(s, text, color=INK):
tf = tbox(s, 0.45, 0.22, 12.5, 0.75)
p = tf.paragraphs[0]; r = p.add_run(); r.text = text
r.font.size = Pt(25); r.font.bold = True; r.font.color.rgb = color
def para(tf, text, size=13, color=INK, bold=False, before=6, bullet=True, first=False):
p = tf.paragraphs[0] if first else tf.add_paragraph()
p.space_before = Pt(0 if first else before)
r = p.add_run(); r.text = ('• ' if bullet else '') + text
r.font.size = Pt(size); r.font.color.rgb = color; r.font.bold = bold
return p
# ---------- 1 cover ----------
s = slide()
tf = tbox(s, 0.9, 2.35, 11.5, 2.6)
p = tf.paragraphs[0]; r = p.add_run()
r.text = 'Training transformer LMs without backpropagation'
r.font.size = Pt(34); r.font.bold = True; r.font.color.rgb = INK
p2 = tf.add_paragraph(); p2.space_before = Pt(14); r = p2.add_run()
r.text = 'Standard architecture at inference · equilibrium-propagation gradients in training · GPU now, analog next'
r.font.size = Pt(17); r.font.color.rgb = GRAY
p3 = tf.add_paragraph(); p3.space_before = Pt(30); r = p3.add_run()
r.text = 'Yuren Hao · July 2026 · technical primer'
r.font.size = Pt(14); r.font.color.rgb = INK
# ---------- 2 architecture ----------
s = slide(); title(s, 'The model is a stock OLMo2 decoder — op by op')
s.shapes.add_picture(A + 'fig_primer_block.png', Inches(0.5), Inches(1.0), height=Inches(6.3))
tf = tbox(s, 5.6, 1.35, 7.3, 5.8)
para(tf, 'Decoder-only transformer: L12, C512, H8, T256. RMSNorm, RoPE, QK-norm, SwiGLU, untied readout. No biases.', 14, first=True)
para(tf, 'Two trained instances: 42.75M (TinyStories, 4k BPE) and 72.11M (FineWeb-Edu, 32k BPE).', 14)
para(tf, 'Nothing in the inference graph is modified for our training method — the checkpoint is indistinguishable in form from a conventionally trained model.', 14, bold=True)
para(tf, 'Color code, kept through the whole deck: blue = weight×activation matmuls (7 logical — crossbar-mappable on analog hardware); purple = activation×activation (qkᵀ, Av — must be computed on the fly); cream = RMSNorm; gray = elementwise.', 13, GRAY)
para(tf, 'One honest architecture–method coupling: QK-norm and the final RMSNorm bound internal state scales, which the training estimator relies on. Both are stock components.', 13, GRAY)
# ---------- 3 EP phases ----------
s = slide(); title(s, 'Training = two settles and a difference measurement')
s.shapes.add_picture(A + 'fig_primer_phases.png', Inches(0.35), Inches(1.05), height=Inches(6.1))
tf = tbox(s, 7.35, 1.5, 5.6, 5.6)
para(tf, 'The stack is treated as a system that settles: each layer gets a state zₗ, and the energy penalizes disagreement between zₗ and fₗ(zₗ₋₁).', 14, first=True)
para(tf, 'Free phase: settle with no label. The exact minimum IS the ordinary forward pass (E = 0). Inference is untouched.', 14)
para(tf, 'Nudged phase: weights frozen, states move — the loss pulls the top state with strength β, the whole stack re-settles, every layer picks up a mismatch dₗ.', 14)
para(tf, 'Update: states frozen, weights move — each layer absorbs its own mismatch: Δθₗ ∝ ⟨dₗ, ∂fₗ/∂θₗ⟩/β. Local quantities only; no global tape.', 14)
para(tf, 'The nudged state becomes the new free state: today’s pulled answer is tomorrow’s natural forward output.', 14, ORAN, bold=True)
# ---------- 4 schedule ----------
s = slide(); title(s, 'The update schedule: local, pipelinable — and native to physics')
s.shapes.add_picture(A + 'fig_p3_schedule.png', Inches(0.45), Inches(1.15), width=Inches(12.4))
tf = tbox(s, 0.7, 6.55, 12.0, 0.8)
para(tf, 'Everything that follows is the price of emulating this settle with discrete sweeps on a GPU. Two problems fall out — the physical system has neither.', 14, RED, bold=True, first=True, bullet=False)
# ---------- 5 what happened ----------
s = slide(); title(s, 'What actually happened at 72M (real run, real logs)')
s.shapes.add_picture(A + 'fig_p4_blowup.png', Inches(0.4), Inches(1.1), height=Inches(5.5))
tf = tbox(s, 9.05, 1.35, 4.0, 5.9)
para(tf, 'Same β = 1e-3: safe for 135k steps, lethal at 195k. Something moved under the schedule.', 13.5, RED, bold=True, first=True)
para(tf, 'Guards contained it (15k step-skips, no NaN), a governor finished the token budget — but β starved to 2e-5 and best never improved after 185k.', 13)
para(tf, 'Headline stands: 72.11M × 1.44B tokens fully BP-free, best 3.7117 vs BP twin 3.2884.', 13)
para(tf, 'The 0.43 gap is a BLOWN-SCHEDULE number, not a method number. The honest rerun is in flight (last slide but one).', 13, bold=True)
# ---------- 6 problem 1 ----------
s = slide(); title(s, 'Problem 1 — the nudge closes a loop; discrete sweeps can amplify it')
s.shapes.add_picture(A + 'fig_p5_loop.png', Inches(1.35), Inches(1.15), height=Inches(5.6))
tf = tbox(s, 0.7, 6.85, 12.0, 0.55)
para(tf, 'Why the SAME β became lethal: ρ ≈ β·σ(Wₒᵤₜ)²·‖J‖² crossed 1 as σ and ‖J‖ grew with fit depth.', 14, first=True, bullet=False)
# ---------- 7 problem 2 ----------
s = slide(); title(s, 'Problem 2 — β is squeezed from both sides, and the window narrows')
s.shapes.add_picture(A + 'fig_p6_window.png', Inches(0.4), Inches(1.1), height=Inches(5.5))
tf = tbox(s, 9.05, 1.3, 4.0, 6.0)
para(tf, 'Floor is real and RISING: on the 42M testbed, tail at flat 1e-3 beats decaying to 3e-4 (1.2678 vs 1.2808). Momentum as a substitute for β refuted in 3 doses.', 13, first=True)
para(tf, 'Ceiling is real and FALLING: measured crossings at 195k (≤1e-3), 222k (≤3e-4), late (~2e-5).', 13)
para(tf, 'Window-aware recipe: centered ±β estimator (bias O(β²) → ride high) + a loop-gain governor capping β near ITS ceiling.', 13, bold=True)
para(tf, 'fw72m_cent, in flight, is exactly this recipe.', 13, ORAN, bold=True)
# ---------- 8 diagnostics ----------
s = slide(); title(s, 'Diagnostics already done (every number is a run you can open in wandb)')
rows = [
('DDP gradient equivalence', 'cos 0.999999999 vs single-GPU big batch (manual all-reduce, synced guards)'),
('centered + bf16 + DDP', 'cos 1.000000000 vs sequential centered; packed ±β variant identical (relerr 2e-5)'),
('Error-source decomposition', 'within-block read EXACT (1.0000); ALL bias sits in between-block transmission (0.9987 = the whole EP deficit); K=1 degenerates to BP; K3 = K8 (converged, not truncated)'),
('Wall-2 probes — 6 arms @72M', 'damping refuted in 3 doses; lowering β sails through; governor completes the budget'),
('Tail-SNR arms — 7 @42M', 'centered@3e-3 wins CE and cos; ĝ-momentum refuted ×3; Richardson dominated; AdamW = tie'),
('Gap-vs-size ladder — 8 runs, pre-registered', 'gap tracks FIT DEPTH, not width: 0.004 / 0.004 / 0.015 / 0.013 / 0.050 (C128→C512); C128 has gate cos 0.805 yet ZERO gap — direction noise alone costs nothing until the fit is deep'),
('bf16 mixed precision', 'epoch-validated lossless (Δ +0.006), 1.56× wall-clock'),
('Cost of centered', '1.72× step time (1.64× packed); tail-only switching amortizes to ~1.13×'),
]
tbl = s.shapes.add_table(len(rows) + 1, 2, Inches(0.45), Inches(1.15), Inches(12.45), Inches(5.9)).table
tbl.columns[0].width = Inches(3.6); tbl.columns[1].width = Inches(8.85)
hdr = ('diagnostic', 'verdict')
for j in range(2):
c = tbl.cell(0, j); c.text = hdr[j]
c.text_frame.paragraphs[0].runs[0].font.size = Pt(12)
c.text_frame.paragraphs[0].runs[0].font.bold = True
for i, (a, b) in enumerate(rows):
for j, t in enumerate((a, b)):
c = tbl.cell(i + 1, j); c.text = t
r = c.text_frame.paragraphs[0].runs[0]
r.font.size = Pt(10.5); r.font.bold = (j == 0)
r.font.color.rgb = INK
# ---------- 9 likely outcomes ----------
s = slide(); title(s, 'In flight right now — and what the outcomes would mean')
tf = tbox(s, 0.55, 1.2, 12.4, 6.0)
para(tf, 'stage1b_cent — 42M TinyStories, centered for the FULL epoch (at 30k/58.8k: best 1.3642, ahead of both originals at matched step).', 14, bold=True, first=True)
para(tf, 'reads: best ≤ ~1.24 ⇒ the 0.050 gap was a noise-recipe account and the zero-gap line extends to C512 · 1.25–1.26 ⇒ a residual needs window management · >1.27 ⇒ an unknown term (falsifier).', 13, GRAY)
para(tf, 'fw72m_cent — 72M FineWeb, the window-aware recipe from scratch (at 48k/234k: best 3.6890 — already below the original’s FULL-RUN best 3.7117, at 20% of the budget).', 14, bold=True, before=14)
para(tf, 'registered predictions: no 195k-style blow; honest gap vs BP twin lands 0.25–0.35 (current trajectory suggests better).', 13, GRAY)
para(tf, 'If both hold: the EP–BP gap is an ENGINEERING account — schedule + estimator — not an architectural tax. The frontier is late-phase signal-to-noise, and we hold validated levers for it.', 14, GREEN, bold=True, before=16)
para(tf, 'Queued next: 3 seeds at C256/C512 (certify the ladder tiers); the est-switch rule (when to start paying for centered); 300M × 6B.', 13, before=14)
# ---------- 10 the $50k ----------
s = slide(); title(s, 'How to spend the $50k — a laddered plan with kill-gates')
rows = [
('rung', 'est. cost', 'what it buys', 'gate to the next rung'),
('300M × 6B tokens (rented H100, days)', '~$0.3k', 'recipe + window governor at 4× params; DDP at scale', 'gap & stability hold'),
('1B × 10B', '~$5k', 'first B-class BP-free LM; the LAST affordable BP twin', 'gap ≤ target at 1B'),
('7B × 20B (flagship)', '~$30k', 'headline result; compare to published baselines (no twin)', '—'),
('reserve', '$10–15k', 'seeds, reruns, ablations, surprises', '—'),
]
tbl = s.shapes.add_table(len(rows), 4, Inches(0.45), Inches(1.25), Inches(12.45), Inches(4.2)).table
widths = (3.6, 1.3, 4.6, 2.95)
for j, w in enumerate(widths): tbl.columns[j].width = Inches(w)
for i, row in enumerate(rows):
for j, t in enumerate(row):
c = tbl.cell(i, j); c.text = t
r = c.text_frame.paragraphs[0].runs[0]
r.font.size = Pt(12 if i == 0 else 12.5); r.font.bold = (i == 0 or j == 0)
r.font.color.rgb = INK
tf = tbox(s, 0.55, 5.9, 12.4, 1.3)
para(tf, 'Prices (sourced, July 2026): market H100 $1.87–2.99/GPU·h; AWS p5e $4.97/GPU·h. All estimates include bf16 (1.56×) and EP ≈ 3.2× BP wall-clock; tail-only centered adds ~1.13×.', 12, GRAY, first=True, bullet=False)
para(tf, 'Alternative one-shot: 3B-Chinchilla ≈ $40k fits the envelope alone — but the ladder buys three publishable points and de-risks the flagship.', 12, GRAY, bullet=False)
prs.save('/home/yurenh2/ept/assets/decks/alexi_primer_deck.pptx')
print('deck saved: /home/yurenh2/ept/assets/decks/alexi_primer_deck.pptx')
|
