summaryrefslogtreecommitdiff
path: root/ep_run
diff options
context:
space:
mode:
authorYuren Hao <yurenh2@illinois.edu>2026-07-16 12:58:26 -0500
committerYuren Hao <yurenh2@illinois.edu>2026-07-16 12:58:26 -0500
commit44e4238f7b3e1bca91faa7aec5e78109d7dd2966 (patch)
tree782b563631ba4b549a066994bf81ba8a9c2bf8f3 /ep_run
parent4c7cdeeba856e360f6be29cf03ccb8c73f1049d3 (diff)
Deck finalized: both verdicts sealed (C512 zero-gap, 72M gap 0.043), decoupling twist, K-saturation + quant-matrix rows; window fig cent line SEALED
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014FAPDWQ49M5Ye3NpTndTpn
Diffstat (limited to 'ep_run')
-rw-r--r--ep_run/build_alexi_deck.py24
-rw-r--r--ep_run/fig_primer_pack2.py4
2 files changed, 14 insertions, 14 deletions
diff --git a/ep_run/build_alexi_deck.py b/ep_run/build_alexi_deck.py
index d206e8d..d90dcbc 100644
--- a/ep_run/build_alexi_deck.py
+++ b/ep_run/build_alexi_deck.py
@@ -80,7 +80,7 @@ tf = tbox(s, 9.05, 1.35, 4.0, 5.9)
para(tf, 'Same β = 1e-3: safe for 135k steps, lethal at 195k. Something moved under the schedule.', 13.5, RED, bold=True, first=True)
para(tf, 'Guards contained it (15k step-skips, no NaN), a governor finished the token budget — but β starved to 2e-5 and best never improved after 185k.', 13)
para(tf, 'Headline stands: 72.11M × 1.44B tokens fully BP-free, best 3.7117 vs BP twin 3.2884.', 13)
-para(tf, 'The 0.43 gap is a BLOWN-SCHEDULE number, not a method number. The honest rerun is in flight (last slide but one).', 13, bold=True)
+para(tf, 'The 0.43 gap was a BLOWN-SCHEDULE number. The honest rerun (window-aware recipe) finished: gap 0.043 — ten times smaller. Details two slides ahead.', 13, bold=True)
# ---------- 6 problem 1 ----------
s = slide(); title(s, 'Problem 1 — the nudge closes a loop; discrete sweeps can amplify it')
@@ -95,7 +95,7 @@ tf = tbox(s, 9.05, 1.3, 4.0, 6.0)
para(tf, 'Floor is real and RISING: on the 42M testbed, tail at flat 1e-3 beats decaying to 3e-4 (1.2678 vs 1.2808). Momentum as a substitute for β refuted in 3 doses.', 13, first=True)
para(tf, 'Ceiling is real and FALLING: measured crossings at 195k (≤1e-3), 222k (≤3e-4), late (~2e-5).', 13)
para(tf, 'Window-aware recipe: centered ±β estimator (bias O(β²) → ride high) + a loop-gain governor capping β near ITS ceiling.', 13, bold=True)
-para(tf, 'fw72m_cent, in flight, is exactly this recipe.', 13, ORAN, bold=True)
+para(tf, 'fw72m_cent ran exactly this recipe: beta rode 3e-3 for all 234k steps, zero guard events, final gap 0.043.', 13, ORAN, bold=True)
# ---------- 8 diagnostics ----------
s = slide(); title(s, 'Diagnostics already done (every number is a run you can open in wandb)')
@@ -107,7 +107,9 @@ rows = [
('Tail-SNR arms — 7 @42M', 'centered@3e-3 wins CE and cos; ĝ-momentum refuted ×3; Richardson dominated; AdamW = tie'),
('Gap-vs-size ladder — 8 runs, pre-registered', 'gap tracks FIT DEPTH, not width: 0.004 / 0.004 / 0.015 / 0.013 / 0.050 (C128→C512); C128 has gate cos 0.805 yet ZERO gap — direction noise alone costs nothing until the fit is deep'),
('bf16 mixed precision', 'epoch-validated lossless (Δ +0.006), 1.56× wall-clock'),
- ('Cost of centered', '1.72× step time (1.64× packed); tail-only switching amortizes to ~1.13×'),
+ ('Cost of centered', '1.72× step time (1.64× packed; 1.39× mirror warm-start — gradient-identical)'),
+ ('K-saturation across training', 'cos(EP,BP) vs K∈{1,2,3,8} at ckpts 5k/50k/100k: K3 = K8 to the 4th decimal at EVERY stage — the fixed point does not recede as training roughens the landscape'),
+ ('Quantization Δ-vs-Δ matrix (EP and BP mirrors)', 'at the 8-bit compute + digital-master operating point: ZERO tax for both; below it the tax is EP-specific (finite-β measurement chain) — sets the hardware acceptance bar'),
]
tbl = s.shapes.add_table(len(rows) + 1, 2, Inches(0.45), Inches(1.15), Inches(12.45), Inches(5.9)).table
tbl.columns[0].width = Inches(3.6); tbl.columns[1].width = Inches(8.85)
@@ -123,16 +125,14 @@ for i, (a, b) in enumerate(rows):
r.font.size = Pt(10.5); r.font.bold = (j == 0)
r.font.color.rgb = INK
-# ---------- 9 likely outcomes ----------
-s = slide(); title(s, 'In flight right now — and what the outcomes would mean')
+# ---------- 9 outcomes ----------
+s = slide(); title(s, 'Both verdicts landed — the gap is an engineering account, and it is paid')
tf = tbox(s, 0.55, 1.2, 12.4, 6.0)
-para(tf, 'stage1b_cent — 42M TinyStories, centered for the FULL epoch (at 30k/58.8k: best 1.3642, ahead of both originals at matched step).', 14, bold=True, first=True)
-para(tf, 'reads: best ≤ ~1.24 ⇒ the 0.050 gap was a noise-recipe account and the zero-gap line extends to C512 · 1.25–1.26 ⇒ a residual needs window management · >1.27 ⇒ an unknown term (falsifier).', 13, GRAY)
-para(tf, 'fw72m_cent — 72M FineWeb, the window-aware recipe from scratch (at 48k/234k: best 3.6890 — already below the original’s FULL-RUN best 3.7117, at 20% of the budget).', 14, bold=True, before=14)
-para(tf, 'registered predictions: no 195k-style blow; honest gap vs BP twin lands 0.25–0.35 (current trajectory suggests better).', 13, GRAY)
-para(tf, 'If both hold: the EP–BP gap is an ENGINEERING account — schedule + estimator — not an architectural tax. The frontier is late-phase signal-to-noise, and we hold validated levers for it.', 14, GREEN, bold=True, before=16)
-para(tf, 'Queued next: 3 seeds at C256/C512 (certify the ladder tiers); the est-switch rule (when to start paying for centered); 300M × 6B.', 13, before=14)
-
+para(tf, '42M TinyStories, full-epoch window recipe: best 1.2334 vs BP twin 1.2311 — +0.0023, inside the ±0.006 seed band: STATISTICAL ZERO-GAP at C512.', 14.5, bold=True, first=True)
+para(tf, '72M FineWeb, from scratch, 234k steps: best 3.3318 vs BP twin 3.2884 — gap 0.043 (~1.3% relative), ZERO guard events, best still improving in the final quarter. Registered prediction (0.25–0.35) beaten 8×.', 14.5, bold=True, before=12)
+para(tf, 'Decoupling twist (the control we almost skipped): at the 42M tail, plain single-sided @3e-3 EQUALS centered @3e-3 (1.2590 vs 1.2591) — the win is the β schedule itself; the O(β) bias is measurable in cosine but costless in CE. Third independent confirmation that only SNR prices CE.', 13, ORAN, before=14)
+para(tf, 'In flight now: plain-estimator full-epoch (does zero-gap cost 1.0×?); 72M centered-vs-plain A/B; a ceiling-riding β governor (β climbs until the live loop-gain meter objects).', 13, GRAY, before=10)
+para(tf, 'If plain holds: the whole recipe is "standard transformer + EP + one β governor" — no estimator tricks. And the knob that closes the gap (a large nudge) is exactly the knob analog hardware gets for free.', 14, GREEN, bold=True, before=14)
# ---------- 10 the $50k ----------
s = slide(); title(s, 'How to spend the $50k — a laddered plan with kill-gates')
rows = [
diff --git a/ep_run/fig_primer_pack2.py b/ep_run/fig_primer_pack2.py
index 47be501..74850c8 100644
--- a/ep_run/fig_primer_pack2.py
+++ b/ep_run/fig_primer_pack2.py
@@ -184,8 +184,8 @@ xm = x[x <= 185]
cent = np.where(xm < 20, 3e-4, 3e-3)
cent = np.minimum(cent, np.exp(np.interp(xm, cx, np.log(cy))) * 0.85)
ax2.plot(xm, cent, color=ORAN, lw=2.2, ls='--')
-ax2.text(24, 4.25e-3, 'fw72m_cent (in flight): centered + $\\beta$ rides ITS OWN ceiling via bcap 0.9 — where that ceiling sits is being measured now (48k: ahead)', fontsize=8.6, color=ORAN, ha='left')
-ax2.text(186.5, 1.62e-3, '→ ?', fontsize=10, color=ORAN, fontweight='bold')
+ax2.text(24, 4.25e-3, 'fw72m_cent (SEALED): rode 3e-3 for all 234k steps — zero guard events, final gap 0.043', fontsize=8.6, color=ORAN, ha='left')
+ax2.text(186.5, 1.62e-3, '✓', fontsize=12, color=ORAN, fontweight='bold')
# measured ceiling points
for (mx, my, lab) in [(195, 1e-3, '≤1e-3'), (222, 3e-4, '≤3e-4'), (228, 2.5e-5, '≈2e-5')]:
ax2.plot([mx], [my], marker='v', ms=7, color=RED, zorder=5)