From a6d88a18a4bfd46ab0dd487730b72dcbc3b8b5b8 Mon Sep 17 00:00:00 2001 From: Yuren Hao Date: Wed, 15 Jul 2026 10:32:26 -0500 Subject: assets/: organize into decks|figs|hw|refs|misc; scripts + gitignore repointed Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014FAPDWQ49M5Ye3NpTndTpn --- assets/figs/fig_ep_phases.png | Bin 0 -> 159757 bytes assets/figs/fig_p3_schedule.png | Bin 0 -> 148284 bytes assets/figs/fig_p4_blowup.png | Bin 0 -> 171798 bytes assets/figs/fig_p5_loop.png | Bin 0 -> 130618 bytes assets/figs/fig_p6_window.png | Bin 0 -> 160498 bytes assets/figs/fig_primer_arch.emf | Bin 0 -> 539020 bytes assets/figs/fig_primer_arch.png | Bin 0 -> 389335 bytes assets/figs/fig_primer_arch.svg | 1327 ++++++++++++++++++++++++ assets/figs/fig_primer_block.emf | Bin 0 -> 826816 bytes assets/figs/fig_primer_block.png | Bin 0 -> 266630 bytes assets/figs/fig_primer_block.svg | 1956 ++++++++++++++++++++++++++++++++++++ assets/figs/fig_primer_phases.emf | Bin 0 -> 358368 bytes assets/figs/fig_primer_phases.png | Bin 0 -> 204700 bytes assets/figs/fig_primer_phases.svg | 925 +++++++++++++++++ assets/figs/frozen_vs_adaptive.png | Bin 0 -> 217748 bytes 15 files changed, 4208 insertions(+) create mode 100644 assets/figs/fig_ep_phases.png create mode 100644 assets/figs/fig_p3_schedule.png create mode 100644 assets/figs/fig_p4_blowup.png create mode 100644 assets/figs/fig_p5_loop.png create mode 100644 assets/figs/fig_p6_window.png create mode 100644 assets/figs/fig_primer_arch.emf create mode 100644 assets/figs/fig_primer_arch.png create mode 100644 assets/figs/fig_primer_arch.svg create mode 100644 assets/figs/fig_primer_block.emf create mode 100644 assets/figs/fig_primer_block.png create mode 100644 assets/figs/fig_primer_block.svg create mode 100644 assets/figs/fig_primer_phases.emf create mode 100644 assets/figs/fig_primer_phases.png create mode 100644 assets/figs/fig_primer_phases.svg create mode 100644 assets/figs/frozen_vs_adaptive.png (limited to 'assets/figs') diff --git a/assets/figs/fig_ep_phases.png b/assets/figs/fig_ep_phases.png new file mode 100644 index 0000000..e787f58 Binary files /dev/null and b/assets/figs/fig_ep_phases.png differ diff --git a/assets/figs/fig_p3_schedule.png b/assets/figs/fig_p3_schedule.png new file mode 100644 index 0000000..c4ad8b3 Binary files /dev/null and b/assets/figs/fig_p3_schedule.png differ diff --git a/assets/figs/fig_p4_blowup.png b/assets/figs/fig_p4_blowup.png new file mode 100644 index 0000000..53b9d3c Binary files /dev/null and b/assets/figs/fig_p4_blowup.png differ diff --git a/assets/figs/fig_p5_loop.png b/assets/figs/fig_p5_loop.png new file mode 100644 index 0000000..18964ad Binary files /dev/null and b/assets/figs/fig_p5_loop.png differ diff --git a/assets/figs/fig_p6_window.png b/assets/figs/fig_p6_window.png new file mode 100644 index 0000000..9519e39 Binary files /dev/null and b/assets/figs/fig_p6_window.png differ diff --git a/assets/figs/fig_primer_arch.emf b/assets/figs/fig_primer_arch.emf new file mode 100644 index 0000000..87ba265 Binary files /dev/null and b/assets/figs/fig_primer_arch.emf differ diff --git a/assets/figs/fig_primer_arch.png b/assets/figs/fig_primer_arch.png new file mode 100644 index 0000000..a7e7adf Binary files /dev/null and b/assets/figs/fig_primer_arch.png differ diff --git a/assets/figs/fig_primer_arch.svg b/assets/figs/fig_primer_arch.svg new file mode 100644 index 0000000..24e64f1 --- /dev/null +++ b/assets/figs/fig_primer_arch.svg @@ -0,0 +1,1327 @@ + + + + + + + + 2026-07-15T09:29:43.786622 + image/svg+xml + + + Matplotlib v3.10.8, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + (A) The model — a stock OLMo2-style decoder + + + Inference is one ordinary forward pass. + + + logits + + + + + + + + + + l + i + n + e + a + r +   + r + e + a + d + o + u + t +   +   +   + ( + u + n + t + i + e + d + ) + W + o + u + t + + + + + + + + + final RMSNorm + + + + + + + + + + t + r + a + n + s + f + o + r + m + e + r +   + b + l + o + c + k +   +   + ( + ) +   +   +   + = + × + 1 + 2 + f + l + + + + + + + + h + z + z + = + + + R + M + S + N + o + r + m + ( + A + t + t + n + ( + ) + ) + + + + + + + + z + h + h + 0 + = + + + R + M + S + N + o + r + m + ( + S + w + i + G + L + U + ( + ) + ) + + + + + causal SDPA · QK-RMSNorm · RoPE + + + + + + + token embedding + + + 42.75M (TinyStories 4k BPE) / 72.11M (FineWeb-Edu 32k BPE) + + + Nothing in this graph is modified for training: the trained checkpoint + is indistinguishable in form from a conventionally trained model. + + + (B) Free phase (= inference) + + + + + + g + i + v + e +   + e + a + c + h +   + l + a + y + e + r +   + a +   + s + t + a + t + e +   +   + ( + 3 +   + o + f +   +   + d + r + a + w + n + ) + ; +   + s + e + t + t + l + e +   + t + h + e +   + d + i + s + a + g + r + e + e + m + e + n + t +   + e + n + e + r + g + y + z + L + l + + + + + + + + E + z + z + f + z + ( + ) + = + ‖ + − + ( + ) + ‖ + ∑ + l + l + l + l + 1 + 2 + − + 1 + 2 + + + + + + + + + z + f + z + 3 + 3 + 2 + = + ( + ) + + + + + term = 0 + + + + + + z + f + z + 2 + 2 + 1 + = + ( + ) + + + + + term = 0 + + + + + + z + f + 1 + 1 + = + ( + e + m + b + ) + + + + + term = 0 + + + + + + + + + + + + + + + + + + e + m + b + ( + ) + x + + + + + + + + T + h + e +   + m + i + n + i + m + u + m +   + i + s +   + e + x + a + c + t + : +   +   + = + 0 + , +   +   + s + t + a + t + e + s +   + ≡ +   + f + o + r + w + a + r + d +   + a + c + t + i + v + a + t + i + o + n + s + , + E + + + + + and one bottom-up pass reaches it. + + + The free phase adds nothing and changes nothing at inference. + + + (C) Nudged phase (training only) + + + + + + a + d + d +   + t + h + e +   + l + o + s + s +   + a + t +   + s + t + r + e + n + g + t + h +   + ≪ + 1 +   + a + n + d +   + s + e + t + t + l + e +   + a + g + a + i + n + : + β + + + + + + + + E + z + β + z + y + ( + ) + + + ⋅ + C + E + ( + l + o + g + i + t + s + ( + ) + , + ) + 3 + + + + + + + + p + u + l + l +   + − + ∇ + C + E + β + + + + + (the only place the label enters) + + + + + + + + + + z + z + d + β + 3 + 3 + 3 + = + + + + + + + + + + z + z + d + β + 2 + 2 + 2 + = + + + + + + + + + + z + z + d + β + 1 + 1 + 1 + = + + + + + + + + + + + + + + + + + + + + + + e + m + b + ( + ) + x + + + + + + + + + + + + d + J + d + 2 + 3 + ⊤ + 3 + = + + + + + + + + + + + + d + J + d + 1 + 2 + ⊤ + 2 + = + + + + + + + + T + h + e +   + t + o + p +   + s + t + a + t + e +   + i + s +   + p + u + l + l + e + d +   + t + o + w + a + r + d +   + l + o + w + e + r +   + l + o + s + s + ; +   + e + a + c + h +   + l + a + y + e + r + ' + s +   + m + i + s + m + a + t + c + h +   + d + l + + + + + transmits DOWN through the same weights, and the stack re-settles. + + + + + + ( +   + = +   + t + h + e +   + t + r + a + n + s + p + o + s + e +   + r + e + a + d +   + a +   + b + i + d + i + r + e + c + t + i + o + n + a + l +   + p + h + y + s + i + c + a + l +   + d + e + v + i + c + e +   + p + r + o + v + i + d + e + s + ) + J + ⊤ + + + + + (D) The update — a difference measurement between the two settled states + + + + + + ̂ + = + ∂ + ( + ) + − + ∂ + ( + ) + / + ( + ∂ + ( + ) + ≡ + 0 + h + e + r + e + , + s + i + n + c + e + = + 0 + ) + g + E + z + E + z + β + E + z + E + [ + ] + θ + β + θ + θ + 0 + 0 + + + + + + + + p + e + r +   + l + a + y + e + r + : +   +   + Δ + ∝ + ⟨ + , + ∂ + ( + ) + / + ∂ + ⟩ + / + θ + d + f + z + θ + β + l + l + l + l + l + − + 1 + + + + + Each layer updates from its own boundary mismatch — no global backward graph, no global tape, no loss + + + + d + e + r + i + v + a + t + i + v + e + s +   + e + x + c + e + p + t +   + t + h + e +   + t + o + p +   + n + u + d + g + e + . +   +   + → + 0 +   + g + i + v + e + s +   + t + h + e +   + e + x + a + c + t +   + g + r + a + d + i + e + n + t + ; +   + b + i + a + s +   + i + s +   + ( + ) + ; +   + a +   + ± +   + t + w + o + - + s + i + d + e + d +   + r + e + a + d +   + c + a + n + c + e + l + s +   + i + t +   + t + o +   + ( + ) + . + β + O + β + β + O + β + 2 + + + + + (E) This is not predictive coding — same energy family, different measurement + + + · PC (as typically run): ONE settled phase with the target clamped hard; the update uses the raw errors of that single state → finite-clamp + + + bias. Its "exact-BP" results require freezing predictions during error transport (fixed-prediction) — backprop re-expressed in local variables. + + + · EP (here): TWO phases and an infinitesimal nudge; the update is a difference quotient in β → bias is measured and controllable, and the + + + settle stays fully self-consistent — which is what physical hardware actually does. + + + · The free phase is the zero-reference of the measurement: on analog hardware the subtraction cancels state-independent device offsets. + + + + + + + + + diff --git a/assets/figs/fig_primer_block.emf b/assets/figs/fig_primer_block.emf new file mode 100644 index 0000000..7993cc9 Binary files /dev/null and b/assets/figs/fig_primer_block.emf differ diff --git a/assets/figs/fig_primer_block.png b/assets/figs/fig_primer_block.png new file mode 100644 index 0000000..b5eb58d Binary files /dev/null and b/assets/figs/fig_primer_block.png differ diff --git a/assets/figs/fig_primer_block.svg b/assets/figs/fig_primer_block.svg new file mode 100644 index 0000000..7a901a9 --- /dev/null +++ b/assets/figs/fig_primer_block.svg @@ -0,0 +1,1956 @@ + + + + + + + + 2026-07-15T09:29:45.734111 + image/svg+xml + + + Matplotlib v3.10.8, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + One transformer block, op by op + + + + + + O + L + M + o + 2 +   + o + r + d + e + r + i + n + g +   +   + R + M + S + N + o + r + m +   + a + f + t + e + r +   + e + a + c + h +   + s + u + b + l + a + y + e + r + , +   + i + n + s + i + d + e +   + t + h + e +   + r + e + s + i + d + u + a + l + . +   +   +   +   +   + = + 5 + 1 + 2 +   +   +   + = + 8 +   +   +   + = + 6 + 4 +   +   +   + = + 2 + 5 + 6 +   +   +   + = + 1 + 4 + 0 + 8 + — + C + H + h + T + h + d + f + f + + + + + attention sublayer + + + + + + i + n + p + u + t +   +   +   +   + z + B + T + C + ( + , + , + ) + + + + + + + + + residual rail + + + + + + + + + + m + a + t + m + u + l +   +   + : + → + 3 + W + C + C + q + k + v + + + + + + + + ( + , + , + 3 + ) +   +   + s + p + l + i + t +   + i + n + t + o +   + , + , + B + T + C + q + k + v + — + + + + + + + + q + + + + + + + + k + + + + + + + + v + + + + + + + + + + + + + + + + + + + + R + M + S + N + o + r + m + q + + + + + + + + R + M + S + N + o + r + m + k + + + + + + + + s + p + l + i + t +   + h + e + a + d + s +   + ( + = + 8 + ) + H + + + + + + + + s + p + l + i + t +   + h + e + a + d + s +   + ( + = + 8 + ) + H + + + + + + + + s + p + l + i + t +   + h + e + a + d + s +   + ( + = + 8 + ) + H + + + + + RoPE + + + RoPE + + + + + + θ + = + 5 + ⋅ + 1 + 0 + 5 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + m + a + t + m + u + l +   +   + = + ⋅ + 1 + / + S + q + k + h + ⊤ + √ + d + + + + + + + + + ( + , + , + , + ) + B + H + T + T + + + + + + + + + causal mask + + + + + + + softmax over keys + + + + + + + + + + m + a + t + m + u + l +   +   + = + y + A + v + + + + + + + + ( + , + , + , + ) + B + H + T + h + d + + + + + + + + + merge heads + + + + + + ( + , + , + ) + B + T + C + + + + + + + + + + + + m + a + t + m + u + l +   +   + : + → + W + C + C + p + r + o + j + + + + + + + + + + + + R + M + S + N + o + r + m + a + t + t + n + + + + + ◄ OLMo2: norm on the sublayer OUTPUT, + not on its input + + + + + + + + + + + + + + + + + + + + + z + 0 + + + + + SwiGLU sublayer + + + + + + + + + + + + + + + + + + m + a + t + m + u + l +   +   + : + → + W + C + h + 1 + f + f + + + + + + + + m + a + t + m + u + l +   +   + : + → + W + C + h + 3 + f + f + + + + + + + + h + f + f + = + 1 + 4 + 0 + 8 + + + + + + + + + SiLU + + + + + + + + + + + ⊙ elementwise gate + + + + + + + + + + m + a + t + m + u + l +   +   + : + → + W + h + C + 2 + f + f + + + + + + + + + + + + R + M + S + N + o + r + m + f + f + + + + + ◄ norm after the sublayer, again + + + + + + + + + + + + + + + + + + + + + o + u + t + p + u + t +   +   +   +   +   + o + n + e +   + f + u + l + l +   +   + d + o + n + e + z + f + 0 + 0 + — + l + + + + + + + + w + e + i + g + h + t +   + × +   + a + c + t + i + v + a + t + i + o + n +   + m + a + t + m + u + l +   +   + 7 +   + l + o + g + i + c + a + l +   + ( +   + f + u + s + e + s +   + , + , + ) + ; +   + c + r + o + s + s + b + a + r + - + m + a + p + p + a + b + l + e +   + o + n +   + a + n + a + l + o + g +   + h + a + r + d + w + a + r + e + — + W + W + W + W + q + k + v + q + k + v + + + + + + + + a + c + t + i + v + a + t + i + o + n +   + × +   + a + c + t + i + v + a + t + i + o + n +   + m + a + t + m + u + l +   +   + 2 +   + ( +   + a + n + d +   + ) + ; +   + c + o + m + p + u + t + e + d +   + o + n +   + t + h + e +   + f + l + y + : +   + t + h + e +   + n + o + n + - + c + r + o + s + s + b + a + r +   + p + a + r + t +   + o + f +   + a + t + t + e + n + t + i + o + n + — + q + k + A + v + ⊤ + + + + + RMSNorm (learned gain) + + + elementwise / reshape + + + + + + N + o +   + b + i + a + s + e + s +   + a + n + y + w + h + e + r + e +   + i + n +   + t + h + e +   + n + e + t + w + o + r + k + . +   +   + T + h + e +   + 1 + / +   + s + c + a + l + e +   + i + s +   + f + o + l + d + e + d +   + i + n + t + o +   + ; +   + m + a + s + k +   + a + n + d +   + s + o + f + t + m + a + x +   + r + u + n +   + a + l + o + n + g +   + t + h + e +   + k + e + y +   + a + x + i + s + . + √ + h + S + d + + + + + + + + + h + C + C + f + f + = + 1 + 4 + 0 + 8 + ≈ + 8 + / + 3 + , +   + r + o + u + n + d + e + d +   + u + p +   + t + o +   + a +   + m + u + l + t + i + p + l + e +   + o + f +   + 6 + 4 + . +   +   + Q + K + - + n + o + r + m +   + a + c + t + s +   + o + n +   + t + h + e +   + f + u + l + l +   + w + i + d + t + h +   + , +   + b + e + f + o + r + e +   + t + h + e +   + h + e + a + d +   + s + p + l + i + t + . + + + + + + + + T + h + e +   + b + o + x + e + d +   + p + i + p + e + l + i + n + e + , +   + i + n + p + u + t +   +   + t + o +   + o + u + t + p + u + t +   + , +   + i + s +   + e + x + a + c + t + l + y +   +   + o + n +   + t + h + e +   + c + o + m + p + a + n + i + o + n +   + p + a + g + e + ; +   + t + h + e +   + s + t + a + t + e +   +   + l + i + v + e + s +   + o + n +   + t + h + e +   + r + e + s + i + d + u + a + l +   + r + a + i + l + . + z + z + f + z + 0 + 0 + l + l + + + + + + + + + + + diff --git a/assets/figs/fig_primer_phases.emf b/assets/figs/fig_primer_phases.emf new file mode 100644 index 0000000..c28c459 Binary files /dev/null and b/assets/figs/fig_primer_phases.emf differ diff --git a/assets/figs/fig_primer_phases.png b/assets/figs/fig_primer_phases.png new file mode 100644 index 0000000..a6db577 Binary files /dev/null and b/assets/figs/fig_primer_phases.png differ diff --git a/assets/figs/fig_primer_phases.svg b/assets/figs/fig_primer_phases.svg new file mode 100644 index 0000000..da9ed6a --- /dev/null +++ b/assets/figs/fig_primer_phases.svg @@ -0,0 +1,925 @@ + + + + + + + + 2026-07-15T09:44:53.517354 + image/svg+xml + + + Matplotlib v3.10.8, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Free phase (= inference) + + + + + + g + i + v + e +   + e + a + c + h +   + l + a + y + e + r +   + a +   + s + t + a + t + e +   +   + ( + 3 +   + o + f +   +   + d + r + a + w + n + ) + ; +   + s + e + t + t + l + e +   + t + h + e +   + d + i + s + a + g + r + e + e + m + e + n + t +   + e + n + e + r + g + y + z + L + l + + + + + + + + E + z + z + f + z + ( + ) + = + ‖ + − + ( + ) + ‖ + ∑ + l + l + l + l + 1 + 2 + − + 1 + 2 + + + + + + + + + z + f + z + 3 + 3 + 2 + = + ( + ) + + + + + term = 0 + + + + + + z + f + z + 2 + 2 + 1 + = + ( + ) + + + + + term = 0 + + + + + + z + f + 1 + 1 + = + ( + e + m + b + ) + + + + + term = 0 + + + + + + + + + + + + + + + + + + e + m + b + ( + ) + x + + + + + + + + T + h + e +   + m + i + n + i + m + u + m +   + i + s +   + e + x + a + c + t + : +   +   + = + 0 + , +   +   + s + t + a + t + e + s +   + ≡ +   + f + o + r + w + a + r + d +   + a + c + t + i + v + a + t + i + o + n + s + , + E + + + + + and one bottom-up pass reaches it. + + + The free phase adds nothing and changes nothing at inference. + + + Nudged phase (training only) + + + + + + a + d + d +   + t + h + e +   + l + o + s + s +   + a + t +   + s + t + r + e + n + g + t + h +   + ≪ + 1 +   + a + n + d +   + s + e + t + t + l + e +   + a + g + a + i + n + : + β + + + + + + + + E + z + β + z + y + ( + ) + + + ⋅ + C + E + ( + l + o + g + i + t + s + ( + ) + , + ) + 3 + + + + + + + + p + u + l + l +   + − + ∇ + C + E + β + + + + + (the only place the label enters) + + + + + + + + + + z + z + d + β + 3 + 3 + 3 + = + + + + + + + + + + z + z + d + β + 2 + 2 + 2 + = + + + + + + + + + + z + z + d + β + 1 + 1 + 1 + = + + + + + + + + + + + + + + + + + + + + + + e + m + b + ( + ) + x + + + + + + + + + + + + d + J + d + 2 + 3 + ⊤ + 3 + = + + + + + + + + + + + + d + J + d + 1 + 2 + ⊤ + 2 + = + + + + + + + + T + h + e +   + t + o + p +   + s + t + a + t + e +   + i + s +   + p + u + l + l + e + d +   + t + o + w + a + r + d +   + l + o + w + e + r +   + l + o + s + s + ; +   + e + a + c + h +   + l + a + y + e + r + ' + s +   + m + i + s + m + a + t + c + h +   + d + l + + + + + transmits DOWN through the same weights, and the stack re-settles. + + + + + + ( +   + = +   + t + h + e +   + t + r + a + n + s + p + o + s + e +   + r + e + a + d +   + a +   + b + i + d + i + r + e + c + t + i + o + n + a + l +   + p + h + y + s + i + c + a + l +   + d + e + v + i + c + e +   + p + r + o + v + i + d + e + s + ) + J + ⊤ + + + + + The update — a difference measurement between the two settled states + + + + + + ̂ + = + ∂ + ( + ) + − + ∂ + ( + ) + / + ( + ∂ + ( + ) + ≡ + 0 + h + e + r + e + , + s + i + n + c + e + = + 0 + ) + g + E + z + E + z + β + E + z + E + [ + ] + θ + β + θ + θ + 0 + 0 + + + + + + + + p + e + r +   + l + a + y + e + r + : +   +   + Δ + ∝ + ⟨ + , + ∂ + ( + ) + / + ∂ + ⟩ + / + θ + d + f + z + θ + β + l + l + l + l + l + − + 1 + + + + + Each layer updates from its own boundary mismatch — no global backward graph, no global tape, no loss derivatives except the top nudge. + + + + + + + + + diff --git a/assets/figs/frozen_vs_adaptive.png b/assets/figs/frozen_vs_adaptive.png new file mode 100644 index 0000000..e45e77b Binary files /dev/null and b/assets/figs/frozen_vs_adaptive.png differ -- cgit v1.2.3