1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
|
"""Slide variant of fig_primer_arch: free/nudged phases + update only (no model panel,
no PC contrast). PDF/PNG/SVG to ../assets/."""
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
plt.rcParams['svg.fonttype'] = 'none' # keep text editable in PPT
from matplotlib.patches import FancyBboxPatch
INK = '#3a3a3a'
GRAY = '#8a8a8a'
BOXF = '#f4f4f4'
BOXE = '#9a9a9a'
BLUE = '#2c6fbb'
BLUEF = '#eaf1fa'
ORAN = '#d95f02'
RED = '#b03a2e'
CREAM = '#faf6ee'
CREME = '#c9b895'
fig = plt.figure(figsize=(12.2, 11.0))
ax = fig.add_axes([0, 0, 1, 1]); ax.set_xlim(0, 100); ax.set_ylim(16, 105); ax.axis('off')
def box(x, y, w, h, fc=BOXF, ec=BOXE, lw=1.0, r=0.6):
b = FancyBboxPatch((x, y), w, h, boxstyle=f'round,pad=0.25,rounding_size={r}',
fc=fc, ec=ec, lw=lw)
ax.add_patch(b); return b
def txt(x, y, s, size=8.5, color=INK, ha='center', va='center', w='normal', style='normal'):
ax.text(x, y, s, fontsize=size, color=color, ha=ha, va=va,
fontweight=w, fontstyle=style)
def arrow(x0, y0, x1, y1, color=INK, lw=1.2, style='-|>', ms=11):
ax.annotate('', xy=(x1, y1), xytext=(x0, y0),
arrowprops=dict(arrowstyle=style, color=color, lw=lw,
mutation_scale=ms, shrinkA=0.5, shrinkB=0.5))
def state_col(x0, ys, labels, notes, fc=BLUEF, ec=BLUE, note_c=GRAY):
for y, lab, note in zip(ys, labels, notes):
box(x0 - 10.5, y - 2.2, 21, 4.4, fc=fc, ec=ec, lw=1.2)
txt(x0, y, lab, 10, color=INK)
if note: txt(x0 + 12.2, y, note, 8.4, note_c, ha='left')
for ya, yb in zip(ys[1:], ys[:-1]):
arrow(x0, ya + 2.7, x0, yb - 2.8, lw=1.15)
# ======================== free phase ========================
bx = 25
txt(bx, 102.3, 'Free phase (= inference)', 11.5, w='bold')
txt(bx, 99.4, 'give each layer a state $z_l$ (3 of $L$ drawn); settle the disagreement energy', 8.8, GRAY)
txt(bx, 95.4, r'$E(z)\ =\ \sum_l\ \frac{1}{2}\,\|\,z_l - f_l(z_{l-1})\,\|^2$', 11.5)
ys = [85, 73, 61]
state_col(bx - 6, ys, [r'$z_3 = f_3(z_2)$', r'$z_2 = f_2(z_1)$', r'$z_1 = f_1(\mathrm{emb})$'],
['term = 0', 'term = 0', 'term = 0'])
arrow(bx - 6, 52.4, bx - 6, 58.0, lw=1.15)
txt(bx - 6, 50.2, r'$\mathrm{emb}(x)$', 10)
txt(bx, 43.8, 'The minimum is exact: $E=0$, states $\\equiv$ forward activations,', 9)
txt(bx, 40.8, 'and one bottom-up pass reaches it.', 9)
txt(bx, 37.2, 'The free phase adds nothing and changes nothing at inference.', 8.6, GRAY)
# ======================== nudged phase ========================
nx = 70
txt(nx, 102.3, 'Nudged phase (training only)', 11.5, w='bold')
txt(nx, 99.4, r'add the loss at strength $\beta \ll 1$ and settle again:', 8.8, GRAY)
txt(nx, 95.4, r'$E(z)\ +\ \beta\cdot\mathrm{CE}(\mathrm{logits}(z_3),\,y)$', 11.5)
txt(nx + 3.0, 91.8, r'pull $-\beta\,\nabla\mathrm{CE}$', 9.4, RED, ha='left')
txt(nx + 3.0, 89.4, '(the only place the label enters)', 7.8, RED, ha='left')
arrow(nx + 2.0, 90.9, nx - 3.5, 87.8, color=RED, lw=1.6)
nys = [85, 73, 61]
state_col(nx - 6, nys,
[r'$z_3^{\beta} = z_3 + d_3$', r'$z_2^{\beta} = z_2 + d_2$', r'$z_1^{\beta} = z_1 + d_1$'],
[None, None, None], fc='#fdeee2', ec=ORAN)
arrow(nx - 6, 52.4, nx - 6, 58.0, lw=1.15)
txt(nx - 6, 50.2, r'$\mathrm{emb}(x)$', 10)
for (ya, yb, lab) in [(83.0, 75.8, r'$d_2 = J_3^{\top} d_3$'),
(71.0, 63.8, r'$d_1 = J_2^{\top} d_2$')]:
arrow(nx + 6.4, ya, nx + 6.4, yb, color=ORAN, lw=1.6)
txt(nx + 8.2, (ya + yb) / 2, lab, 9.4, ORAN, ha='left')
txt(nx, 43.8, 'The top state is pulled toward lower loss; each layer\'s mismatch $d_l$', 9)
txt(nx, 40.8, 'transmits DOWN through the same weights, and the stack re-settles.', 9)
txt(nx, 37.2, r'($J^{\top}$ = the transpose read a bidirectional physical device provides)', 8.6, GRAY)
# ======================== the update ========================
box(2, 18.5, 96, 15.0, fc=CREAM, ec=CREME, lw=1.2)
txt(4, 30.9, 'The update — a difference measurement between the two settled states', 11, w='bold', ha='left')
txt(50, 27.1, r'$\hat{g}\ =\ \left[\ \partial_\theta E(z^{\beta})\ -\ \partial_\theta E(z^{0})\ \right]\,/\,\beta$'
r'$\qquad\qquad(\partial_\theta E(z^0)\equiv 0\ \mathrm{here,\ since}\ E=0)$', 11)
txt(50, 23.4, r'per layer: $\Delta\theta_l\ \propto\ \langle\ d_l\ ,\ \partial f_l(z_{l-1})/\partial\theta_l\ \rangle\ /\ \beta$', 10)
txt(50, 20.4, r'Each layer updates from its own boundary mismatch — no global backward graph, no global tape, no loss'
r' derivatives except the top nudge.', 8.6)
fig.savefig('/home/yurenh2/ept/assets/figs/fig_primer_phases.pdf')
fig.savefig('/home/yurenh2/ept/assets/figs/fig_primer_phases.png', dpi=185)
fig.savefig('/home/yurenh2/ept/assets/figs/fig_primer_phases.svg')
print('saved fig_primer_phases.{pdf,png,svg}')
|
