1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
|
"""Alexi technical-primer deck: architecture -> EP update -> schedule -> problems ->
hypotheses -> diagnostics -> likely outcomes -> $50k plan. 16:9, minimal styling."""
from pptx import Presentation
from pptx.util import Inches, Pt
from pptx.dml.color import RGBColor
from pptx.enum.text import PP_ALIGN
INK = RGBColor(0x3a, 0x3a, 0x3a); GRAY = RGBColor(0x8a, 0x8a, 0x8a)
BLUE = RGBColor(0x2c, 0x6f, 0xbb); ORAN = RGBColor(0xd9, 0x5f, 0x02)
RED = RGBColor(0xb0, 0x3a, 0x2e); GREEN = RGBColor(0x2e, 0x7d, 0x32)
A = '/home/yurenh2/ept/assets/figs/'
prs = Presentation()
prs.slide_width = Inches(13.333); prs.slide_height = Inches(7.5)
BLANK = prs.slide_layouts[6]
def slide():
return prs.slides.add_slide(BLANK)
def tbox(s, x, y, w, h):
tb = s.shapes.add_textbox(Inches(x), Inches(y), Inches(w), Inches(h))
tb.text_frame.word_wrap = True
return tb.text_frame
def title(s, text, color=INK):
tf = tbox(s, 0.45, 0.22, 12.5, 0.75)
p = tf.paragraphs[0]; r = p.add_run(); r.text = text
r.font.size = Pt(25); r.font.bold = True; r.font.color.rgb = color
def para(tf, text, size=13, color=INK, bold=False, before=6, bullet=True, first=False):
p = tf.paragraphs[0] if first else tf.add_paragraph()
p.space_before = Pt(0 if first else before)
r = p.add_run(); r.text = ('• ' if bullet else '') + text
r.font.size = Pt(size); r.font.color.rgb = color; r.font.bold = bold
return p
# ---------- 1 cover ----------
s = slide()
tf = tbox(s, 0.9, 2.35, 11.5, 2.6)
p = tf.paragraphs[0]; r = p.add_run()
r.text = 'Training transformer LMs without backpropagation'
r.font.size = Pt(34); r.font.bold = True; r.font.color.rgb = INK
p2 = tf.add_paragraph(); p2.space_before = Pt(14); r = p2.add_run()
r.text = 'Standard architecture at inference · equilibrium-propagation gradients in training · GPU now, analog next'
r.font.size = Pt(17); r.font.color.rgb = GRAY
p3 = tf.add_paragraph(); p3.space_before = Pt(30); r = p3.add_run()
r.text = 'Yuren Hao · July 2026 · technical primer'
r.font.size = Pt(14); r.font.color.rgb = INK
# ---------- 2 architecture ----------
s = slide(); title(s, 'The model is a stock OLMo2 decoder — op by op')
s.shapes.add_picture(A + 'fig_primer_block.png', Inches(0.5), Inches(1.0), height=Inches(6.3))
tf = tbox(s, 5.6, 1.35, 7.3, 5.8)
para(tf, 'Decoder-only transformer: L12, C512, H8, T256. RMSNorm, RoPE, QK-norm, SwiGLU, untied readout. No biases.', 14, first=True)
para(tf, 'Two trained instances: 42.75M (TinyStories, 4k BPE) and 72.11M (FineWeb-Edu, 32k BPE).', 14)
para(tf, 'Nothing in the inference graph is modified for our training method — the checkpoint is indistinguishable in form from a conventionally trained model.', 14, bold=True)
para(tf, 'Color code, kept through the whole deck: blue = weight×activation matmuls (7 logical — crossbar-mappable on analog hardware); purple = activation×activation (qkᵀ, Av — must be computed on the fly); cream = RMSNorm; gray = elementwise.', 13, GRAY)
para(tf, 'One honest architecture–method coupling: QK-norm and the final RMSNorm bound internal state scales, which the training estimator relies on. Both are stock components.', 13, GRAY)
# ---------- 3 EP phases ----------
s = slide(); title(s, 'Training = two settles and a difference measurement')
s.shapes.add_picture(A + 'fig_primer_phases.png', Inches(0.35), Inches(1.05), height=Inches(6.1))
tf = tbox(s, 7.35, 1.5, 5.6, 5.6)
para(tf, 'The stack is treated as a system that settles: each layer gets a state zₗ, and the energy penalizes disagreement between zₗ and fₗ(zₗ₋₁).', 14, first=True)
para(tf, 'Free phase: settle with no label. The exact minimum IS the ordinary forward pass (E = 0). Inference is untouched.', 14)
para(tf, 'Nudged phase: weights frozen, states move — the loss pulls the top state with strength β, the whole stack re-settles, every layer picks up a mismatch dₗ.', 14)
para(tf, 'Update: states frozen, weights move — each layer absorbs its own mismatch: Δθₗ ∝ ⟨dₗ, ∂fₗ/∂θₗ⟩/β. Local quantities only; no global tape.', 14)
para(tf, 'The nudged state becomes the new free state: today’s pulled answer is tomorrow’s natural forward output.', 14, ORAN, bold=True)
# ---------- 4 schedule ----------
s = slide(); title(s, 'The update schedule: local, pipelinable — and native to physics')
s.shapes.add_picture(A + 'fig_p3_schedule.png', Inches(0.45), Inches(1.15), width=Inches(12.4))
tf = tbox(s, 0.7, 6.55, 12.0, 0.8)
para(tf, 'Everything that follows is the price of emulating this settle with discrete sweeps on a GPU. Two problems fall out — the physical system has neither.', 14, RED, bold=True, first=True, bullet=False)
# ---------- 5 what happened ----------
s = slide(); title(s, 'What actually happened at 72M (real run, real logs)')
s.shapes.add_picture(A + 'fig_p4_blowup.png', Inches(0.4), Inches(1.1), height=Inches(5.5))
tf = tbox(s, 9.05, 1.35, 4.0, 5.9)
para(tf, 'Same β = 1e-3: safe for 135k steps, lethal at 195k. Something moved under the schedule.', 13.5, RED, bold=True, first=True)
para(tf, 'Guards contained it (15k step-skips, no NaN), a governor finished the token budget — but β starved to 2e-5 and best never improved after 185k.', 13)
para(tf, 'Headline stands: 72.11M × 1.44B tokens fully BP-free, best 3.7117 vs BP twin 3.2884.', 13)
para(tf, 'The 0.43 gap was a BLOWN-SCHEDULE number. The honest rerun (window-aware recipe) finished: gap 0.043 — ten times smaller. Details two slides ahead.', 13, bold=True)
# ---------- 6 problem 1 ----------
s = slide(); title(s, 'Problem 1 — the nudge closes a loop; discrete sweeps can amplify it')
s.shapes.add_picture(A + 'fig_p5_loop.png', Inches(1.35), Inches(1.15), height=Inches(5.6))
tf = tbox(s, 0.7, 6.85, 12.0, 0.55)
para(tf, 'Why the SAME β became lethal: ρ ≈ β·σ(Wₒᵤₜ)²·‖J‖² crossed 1 as σ and ‖J‖ grew with fit depth.', 14, first=True, bullet=False)
# ---------- 7 problem 2 ----------
s = slide(); title(s, 'Problem 2 — β is squeezed from both sides, and the window narrows')
s.shapes.add_picture(A + 'fig_p6_window.png', Inches(0.4), Inches(1.1), height=Inches(5.5))
tf = tbox(s, 9.05, 1.3, 4.0, 6.0)
para(tf, 'Floor is real and RISING: on the 42M testbed, tail at flat 1e-3 beats decaying to 3e-4 (1.2678 vs 1.2808). Momentum as a substitute for β refuted in 3 doses.', 13, first=True)
para(tf, 'Ceiling is real and FALLING: measured crossings at 195k (≤1e-3), 222k (≤3e-4), late (~2e-5).', 13)
para(tf, 'Window-aware recipe: centered ±β estimator (bias O(β²) → ride high) + a loop-gain governor capping β near ITS ceiling.', 13, bold=True)
para(tf, 'fw72m_cent ran exactly this recipe: beta rode 3e-3 for all 234k steps, zero guard events, final gap 0.043.', 13, ORAN, bold=True)
# ---------- 8 diagnostics ----------
s = slide(); title(s, 'Diagnostics already done (every number is a run you can open in wandb)')
rows = [
('DDP gradient equivalence', 'cos 0.999999999 vs single-GPU big batch (manual all-reduce, synced guards)'),
('centered + bf16 + DDP', 'cos 1.000000000 vs sequential centered; packed ±β variant identical (relerr 2e-5)'),
('Error-source decomposition', 'within-block read EXACT (1.0000); ALL bias sits in between-block transmission (0.9987 = the whole EP deficit); K=1 degenerates to BP; K3 = K8 (converged, not truncated)'),
('Wall-2 probes — 6 arms @72M', 'damping refuted in 3 doses; lowering β sails through; governor completes the budget'),
('Tail-SNR arms — 7 @42M', 'centered@3e-3 wins CE and cos; ĝ-momentum refuted ×3; Richardson dominated; AdamW = tie'),
('Gap-vs-size ladder — 8 runs, pre-registered', 'gap tracks FIT DEPTH, not width: 0.004 / 0.004 / 0.015 / 0.013 / 0.050 (C128→C512); C128 has gate cos 0.805 yet ZERO gap — direction noise alone costs nothing until the fit is deep'),
('bf16 mixed precision', 'epoch-validated lossless (Δ +0.006), 1.56× wall-clock'),
('Cost of centered', '1.72× step time (1.64× packed; 1.39× mirror warm-start — gradient-identical)'),
('K-saturation across training', 'cos(EP,BP) vs K∈{1,2,3,8} at ckpts 5k/50k/100k: K3 = K8 to the 4th decimal at EVERY stage — the fixed point does not recede as training roughens the landscape'),
('Quantization Δ-vs-Δ matrix (EP and BP mirrors)', 'at the 8-bit compute + digital-master operating point: ZERO tax for both; below it the tax is EP-specific (finite-β measurement chain) — sets the hardware acceptance bar'),
]
tbl = s.shapes.add_table(len(rows) + 1, 2, Inches(0.45), Inches(1.15), Inches(12.45), Inches(5.9)).table
tbl.columns[0].width = Inches(3.6); tbl.columns[1].width = Inches(8.85)
hdr = ('diagnostic', 'verdict')
for j in range(2):
c = tbl.cell(0, j); c.text = hdr[j]
c.text_frame.paragraphs[0].runs[0].font.size = Pt(12)
c.text_frame.paragraphs[0].runs[0].font.bold = True
for i, (a, b) in enumerate(rows):
for j, t in enumerate((a, b)):
c = tbl.cell(i + 1, j); c.text = t
r = c.text_frame.paragraphs[0].runs[0]
r.font.size = Pt(10.5); r.font.bold = (j == 0)
r.font.color.rgb = INK
# ---------- 9 outcomes ----------
s = slide(); title(s, 'Both verdicts landed — the gap is an engineering account, and it is paid')
tf = tbox(s, 0.55, 1.2, 12.4, 6.0)
para(tf, '42M TinyStories, full-epoch window recipe: best 1.2334 vs BP twin 1.2311 — +0.0023, inside the ±0.006 seed band: STATISTICAL ZERO-GAP at C512.', 14.5, bold=True, first=True)
para(tf, '72M FineWeb, from scratch, 234k steps: best 3.3318 vs BP twin 3.2884 — gap 0.043 (~1.3% relative), ZERO guard events, best still improving in the final quarter. Registered prediction (0.25–0.35) beaten 8×.', 14.5, bold=True, before=12)
para(tf, 'Decoupling twist (the control we almost skipped): at the 42M tail, plain single-sided @3e-3 EQUALS centered @3e-3 (1.2590 vs 1.2591) — the win is the β schedule itself; the O(β) bias is measurable in cosine but costless in CE. Third independent confirmation that only SNR prices CE.', 13, ORAN, before=14)
para(tf, 'In flight now: plain-estimator full-epoch (does zero-gap cost 1.0×?); 72M centered-vs-plain A/B; a ceiling-riding β governor (β climbs until the live loop-gain meter objects).', 13, GRAY, before=10)
para(tf, 'If plain holds: the whole recipe is "standard transformer + EP + one β governor" — no estimator tricks. And the knob that closes the gap (a large nudge) is exactly the knob analog hardware gets for free.', 14, GREEN, bold=True, before=14)
# ---------- 10 the $50k ----------
s = slide(); title(s, 'How to spend the $50k — a laddered plan with kill-gates')
rows = [
('rung', 'est. cost', 'what it buys', 'gate to the next rung'),
('300M × 6B tokens (rented H100, days)', '~$0.3k', 'recipe + window governor at 4× params; DDP at scale', 'gap & stability hold'),
('1B × 10B', '~$5k', 'first B-class BP-free LM; the LAST affordable BP twin', 'gap ≤ target at 1B'),
('7B × 20B (flagship)', '~$30k', 'headline result; compare to published baselines (no twin)', '—'),
('reserve', '$10–15k', 'seeds, reruns, ablations, surprises', '—'),
]
tbl = s.shapes.add_table(len(rows), 4, Inches(0.45), Inches(1.25), Inches(12.45), Inches(4.2)).table
widths = (3.6, 1.3, 4.6, 2.95)
for j, w in enumerate(widths): tbl.columns[j].width = Inches(w)
for i, row in enumerate(rows):
for j, t in enumerate(row):
c = tbl.cell(i, j); c.text = t
r = c.text_frame.paragraphs[0].runs[0]
r.font.size = Pt(12 if i == 0 else 12.5); r.font.bold = (i == 0 or j == 0)
r.font.color.rgb = INK
tf = tbox(s, 0.55, 5.9, 12.4, 1.3)
para(tf, 'Prices (sourced, July 2026): market H100 $1.87–2.99/GPU·h; AWS p5e $4.97/GPU·h. All estimates include bf16 (1.56×) and EP ≈ 3.2× BP wall-clock; tail-only centered adds ~1.13×.', 12, GRAY, first=True, bullet=False)
para(tf, 'Alternative one-shot: 3B-Chinchilla ≈ $40k fits the envelope alone — but the ladder buys three publishable points and de-risks the flagship.', 12, GRAY, bullet=False)
prs.save('/home/yurenh2/ept/assets/decks/alexi_primer_deck.pptx')
print('deck saved: /home/yurenh2/ept/assets/decks/alexi_primer_deck.pptx')
|
