diff options
Diffstat (limited to 'assets')
23 files changed, 5035 insertions, 0 deletions
diff --git a/assets/decks/alexi_primer_deck.pptx b/assets/decks/alexi_primer_deck.pptx Binary files differnew file mode 100644 index 0000000..9dcea29 --- /dev/null +++ b/assets/decks/alexi_primer_deck.pptx diff --git a/assets/decks/zhai_update_0714.pptx b/assets/decks/zhai_update_0714.pptx Binary files differnew file mode 100644 index 0000000..7f284f7 --- /dev/null +++ b/assets/decks/zhai_update_0714.pptx diff --git a/assets/figs/fig_ep_phases.png b/assets/figs/fig_ep_phases.png Binary files differnew file mode 100644 index 0000000..e787f58 --- /dev/null +++ b/assets/figs/fig_ep_phases.png diff --git a/assets/figs/fig_p3_schedule.png b/assets/figs/fig_p3_schedule.png Binary files differnew file mode 100644 index 0000000..c4ad8b3 --- /dev/null +++ b/assets/figs/fig_p3_schedule.png diff --git a/assets/figs/fig_p4_blowup.png b/assets/figs/fig_p4_blowup.png Binary files differnew file mode 100644 index 0000000..53b9d3c --- /dev/null +++ b/assets/figs/fig_p4_blowup.png diff --git a/assets/figs/fig_p5_loop.png b/assets/figs/fig_p5_loop.png Binary files differnew file mode 100644 index 0000000..18964ad --- /dev/null +++ b/assets/figs/fig_p5_loop.png diff --git a/assets/figs/fig_p6_window.png b/assets/figs/fig_p6_window.png Binary files differnew file mode 100644 index 0000000..9519e39 --- /dev/null +++ b/assets/figs/fig_p6_window.png diff --git a/assets/figs/fig_primer_arch.emf b/assets/figs/fig_primer_arch.emf Binary files differnew file mode 100644 index 0000000..87ba265 --- /dev/null +++ b/assets/figs/fig_primer_arch.emf diff --git a/assets/figs/fig_primer_arch.png b/assets/figs/fig_primer_arch.png Binary files differnew file mode 100644 index 0000000..a7e7adf --- /dev/null +++ b/assets/figs/fig_primer_arch.png diff --git a/assets/figs/fig_primer_arch.svg b/assets/figs/fig_primer_arch.svg new file mode 100644 index 0000000..24e64f1 --- /dev/null +++ b/assets/figs/fig_primer_arch.svg @@ -0,0 +1,1327 @@ +<?xml version="1.0" encoding="utf-8" standalone="no"?> +<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" + "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"> +<svg xmlns:xlink="http://www.w3.org/1999/xlink" width="950.4pt" height="720pt" viewBox="0 0 950.4 720" xmlns="http://www.w3.org/2000/svg" version="1.1"> + <metadata> + <rdf:RDF xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:cc="http://creativecommons.org/ns#" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"> + <cc:Work> + <dc:type rdf:resource="http://purl.org/dc/dcmitype/StillImage"/> + <dc:date>2026-07-15T09:29:43.786622</dc:date> + <dc:format>image/svg+xml</dc:format> + <dc:creator> + <cc:Agent> + <dc:title>Matplotlib v3.10.8, https://matplotlib.org/</dc:title> + </cc:Agent> + </dc:creator> + </cc:Work> + </rdf:RDF> + </metadata> + <defs> + <style type="text/css">*{stroke-linejoin: round; stroke-linecap: butt}</style> + </defs> + <g id="figure_1"> + <g id="patch_1"> + <path d="M 0 720 +L 950.4 720 +L 950.4 0 +L 0 0 +z +" style="fill: #ffffff"/> + </g> + <g id="axes_1"> + <g id="patch_2"> + <path d="M 41.3424 141.6 +L 224.7696 141.6 +Q 230.472 141.6 230.472 137.485714 +L 230.472 114.857143 +Q 230.472 110.742857 224.7696 110.742857 +L 41.3424 110.742857 +Q 35.64 110.742857 35.64 114.857143 +L 35.64 137.485714 +Q 35.64 141.6 41.3424 141.6 +z +" clip-path="url(#p7fae8feff8)" style="fill: #f4f4f4; stroke: #9a9a9a; stroke-linejoin: miter"/> + </g> + <g id="patch_3"> + <path d="M 41.3424 193.714286 +L 224.7696 193.714286 +Q 230.472 193.714286 230.472 189.6 +L 230.472 166.971429 +Q 230.472 162.857143 224.7696 162.857143 +L 41.3424 162.857143 +Q 35.64 162.857143 35.64 166.971429 +L 35.64 189.6 +Q 35.64 193.714286 41.3424 193.714286 +z +" clip-path="url(#p7fae8feff8)" style="fill: #f4f4f4; stroke: #9a9a9a; stroke-linejoin: miter"/> + </g> + <g id="patch_4"> + <path d="M 17.5824 358.285714 +L 248.5296 358.285714 +Q 254.232 358.285714 254.232 354.171429 +L 254.232 231.428571 +Q 254.232 227.314286 248.5296 227.314286 +L 17.5824 227.314286 +Q 11.88 227.314286 11.88 231.428571 +L 11.88 354.171429 +Q 11.88 358.285714 17.5824 358.285714 +z +" clip-path="url(#p7fae8feff8)" style="fill: #eeeeee; stroke: #777777; stroke-width: 1.3; stroke-linejoin: miter"/> + </g> + <g id="patch_5"> + <path d="M 41.3424 420 +L 224.7696 420 +Q 230.472 420 230.472 415.885714 +L 230.472 393.257143 +Q 230.472 389.142857 224.7696 389.142857 +L 41.3424 389.142857 +Q 35.64 389.142857 35.64 393.257143 +L 35.64 415.885714 +Q 35.64 420 41.3424 420 +z +" clip-path="url(#p7fae8feff8)" style="fill: #f4f4f4; stroke: #9a9a9a; stroke-linejoin: miter"/> + </g> + <g id="patch_6"> + <path d="M 302.7024 145.714286 +L 476.6256 145.714286 +Q 482.328 145.714286 482.328 141.6 +L 482.328 118.971429 +Q 482.328 114.857143 476.6256 114.857143 +L 302.7024 114.857143 +Q 297 114.857143 297 118.971429 +L 297 141.6 +Q 297 145.714286 302.7024 145.714286 +z +" clip-path="url(#p7fae8feff8)" style="fill: #eaf1fa; stroke: #2c6fbb; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_7"> + <path d="M 302.7024 228 +L 476.6256 228 +Q 482.328 228 482.328 223.885714 +L 482.328 201.257143 +Q 482.328 197.142857 476.6256 197.142857 +L 302.7024 197.142857 +Q 297 197.142857 297 201.257143 +L 297 223.885714 +Q 297 228 302.7024 228 +z +" clip-path="url(#p7fae8feff8)" style="fill: #eaf1fa; stroke: #2c6fbb; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_8"> + <path d="M 302.7024 310.285714 +L 476.6256 310.285714 +Q 482.328 310.285714 482.328 306.171429 +L 482.328 283.542857 +Q 482.328 279.428571 476.6256 279.428571 +L 302.7024 279.428571 +Q 297 279.428571 297 283.542857 +L 297 306.171429 +Q 297 310.285714 302.7024 310.285714 +z +" clip-path="url(#p7fae8feff8)" style="fill: #eaf1fa; stroke: #2c6fbb; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_9"> + <path d="M 625.8384 145.714286 +L 799.7616 145.714286 +Q 805.464 145.714286 805.464 141.6 +L 805.464 118.971429 +Q 805.464 114.857143 799.7616 114.857143 +L 625.8384 114.857143 +Q 620.136 114.857143 620.136 118.971429 +L 620.136 141.6 +Q 620.136 145.714286 625.8384 145.714286 +z +" clip-path="url(#p7fae8feff8)" style="fill: #fdeee2; stroke: #d95f02; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_10"> + <path d="M 625.8384 228 +L 799.7616 228 +Q 805.464 228 805.464 223.885714 +L 805.464 201.257143 +Q 805.464 197.142857 799.7616 197.142857 +L 625.8384 197.142857 +Q 620.136 197.142857 620.136 201.257143 +L 620.136 223.885714 +Q 620.136 228 625.8384 228 +z +" clip-path="url(#p7fae8feff8)" style="fill: #fdeee2; stroke: #d95f02; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_11"> + <path d="M 625.8384 310.285714 +L 799.7616 310.285714 +Q 805.464 310.285714 805.464 306.171429 +L 805.464 283.542857 +Q 805.464 279.428571 799.7616 279.428571 +L 625.8384 279.428571 +Q 620.136 279.428571 620.136 283.542857 +L 620.136 306.171429 +Q 620.136 310.285714 625.8384 310.285714 +z +" clip-path="url(#p7fae8feff8)" style="fill: #fdeee2; stroke: #d95f02; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_12"> + <path d="M 22.3344 581.142857 +L 928.0656 581.142857 +Q 933.768 581.142857 933.768 577.028571 +L 933.768 489.257143 +Q 933.768 485.142857 928.0656 485.142857 +L 22.3344 485.142857 +Q 16.632 485.142857 16.632 489.257143 +L 16.632 577.028571 +Q 16.632 581.142857 22.3344 581.142857 +z +" clip-path="url(#p7fae8feff8)" style="fill: #faf6ee; stroke: #c9b895; stroke-width: 1.2; stroke-linejoin: miter"/> + </g> + <g id="patch_13"> + <path d="M 22.3344 708 +L 928.0656 708 +Q 933.768 708 933.768 703.885714 +L 933.768 597.6 +Q 933.768 593.485714 928.0656 593.485714 +L 22.3344 593.485714 +Q 16.632 593.485714 16.632 597.6 +L 16.632 703.885714 +Q 16.632 708 22.3344 708 +z +" clip-path="url(#p7fae8feff8)" style="fill: #f7f7f7; stroke: #9a9a9a; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="text_1"> + <text style="font-weight: 700; font-size: 10.5px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="152.064" y="21.370614" transform="rotate(-0 152.064 21.370614)">(A) The model — a stock OLMo2-style decoder</text> + </g> + <g id="text_2"> + <text style="font-size: 8.5px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #8a8a8a" x="152.064" y="39.37404" transform="rotate(-0 152.064 39.37404)">Inference is one ordinary forward pass.</text> + </g> + <g id="text_3"> + <text style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="133.056" y="81.44144" transform="rotate(-0 133.056 81.44144)">logits</text> + </g> + <g id="patch_14"> + <path d="M 133.056 111.95625 +Q 133.056 101.485232 133.056 92.355855 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.2; stroke-linecap: round"/> + <path d="M 130.856 96.755855 +L 133.056 92.355855 +L 135.256 96.755855 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.2; stroke-linecap: round"/> + </g> + <g id="text_4"> + <!-- linear readout $W_{\mathrm{out}}$ (untied) --> + <g style="fill: #3a3a3a" transform="translate(72.9185 128.516897)"> + <text> + <tspan x="0" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="2.361572" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="4.723145" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="10.110352" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="15.339844" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="20.548584" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="24.043213" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="26.745117" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="30.239746" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="35.469238" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="40.677979" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="46.073486" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="51.273926" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="56.661133" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="59.993896" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="81.090723" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="83.792627" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="86.494531" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="89.810693" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="95.1979" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="100.585107" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="103.917871" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="106.279443" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="111.508936" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="116.904443" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="62.695801" y="-0.15625" style="font-style: oblique; font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">W</tspan> + <tspan x="71.100342" y="1.320312" style="font-size: 5.95px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="74.740649" y="1.320312" style="font-size: 5.95px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="78.511694" y="1.320312" style="font-size: 5.95px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + </text> + </g> + </g> + <g id="patch_15"> + <path d="M 133.056 164.070536 +Q 133.056 153.599518 133.056 144.470141 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.2; stroke-linecap: round"/> + <path d="M 130.856 148.870141 +L 133.056 144.470141 +L 135.256 148.870141 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.2; stroke-linecap: round"/> + </g> + <g id="text_5"> + <text style="font-size: 8.5px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="133.056" y="180.631183" transform="rotate(-0 133.056 180.631183)">final RMSNorm</text> + </g> + <g id="patch_16"> + <path d="M 133.056 228.530525 +Q 133.056 211.886221 133.056 196.583559 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.2; stroke-linecap: round"/> + <path d="M 130.856 200.983559 +L 133.056 196.583559 +L 135.256 200.983559 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.2; stroke-linecap: round"/> + </g> + <g id="text_6"> + <!-- transformer block ($=f_l$) $\times\,12$ --> + <g style="fill: #3a3a3a" transform="translate(57.141 253.454866)"> + <text> + <tspan x="0" y="-0.15625" style="font-weight: 700; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="4.302246" y="-0.15625" style="font-weight: 700; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="8.740723" y="-0.15625" style="font-weight: 700; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="14.813965" y="-0.15625" style="font-weight: 700; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="21.221191" y="-0.15625" style="font-weight: 700; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="26.578125" y="-0.15625" style="font-weight: 700; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="30.493652" y="-0.15625" style="font-weight: 700; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="36.676758" y="-0.15625" style="font-weight: 700; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="41.115234" y="-0.15625" style="font-weight: 700; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="50.493164" y="-0.15625" style="font-weight: 700; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="56.597168" y="-0.15625" style="font-weight: 700; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="61.035645" y="-0.15625" style="font-weight: 700; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="64.168945" y="-0.15625" style="font-weight: 700; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">b</tspan> + <tspan x="70.611328" y="-0.15625" style="font-weight: 700; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="73.696289" y="-0.15625" style="font-weight: 700; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="79.879395" y="-0.15625" style="font-weight: 700; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="85.214355" y="-0.15625" style="font-weight: 700; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">k</tspan> + <tspan x="91.199707" y="-0.15625" style="font-weight: 700; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="94.333008" y="-0.15625" style="font-weight: 700; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="97.466309" y="-0.15625" style="font-weight: 700; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="117.792334" y="-0.15625" style="font-weight: 700; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="121.905615" y="-0.15625" style="font-weight: 700; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="125.038916" y="-0.15625" style="font-weight: 700; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="128.172217" y="-0.15625" style="font-weight: 700; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="103.333008" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="131.305518" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">×</tspan> + <tspan x="140.307744" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">1</tspan> + <tspan x="146.033818" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">2</tspan> + <tspan x="112.627441" y="-0.15625" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="115.795898" y="1.320312" style="font-style: oblique; font-size: 6.3px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + </text> + </g> + </g> + <g id="text_7"> + <!-- $h\ =\ z + \mathrm{RMSNorm}(\,\mathrm{Attn}(z)\,)$ --> + <g style="fill: #3a3a3a" transform="translate(71.316 284.312009)"> + <text> + <tspan x="0" y="-0.15625" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="22.596621" y="-0.15625" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="110.208223" y="-0.15625" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="10.379854" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="29.07416" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">+</tspan> + <tspan x="38.368594" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">R</tspan> + <tspan x="44.622012" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">M</tspan> + <tspan x="52.387149" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">S</tspan> + <tspan x="58.100039" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">N</tspan> + <tspan x="64.832461" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="70.338809" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="74.039004" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="82.806094" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="87.778535" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">A</tspan> + <tspan x="93.935274" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="97.464082" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="100.992891" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="106.696992" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="114.932344" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="119.904785" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + </text> + </g> + </g> + <g id="text_8"> + <!-- $z'\ =\ h + \mathrm{RMSNorm}(\,\mathrm{SwiGLU}(h)\,)$ --> + <g style="fill: #3a3a3a" transform="translate(62.046 315.169152)"> + <text> + <tspan x="0" y="-0.032813" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="23.996572" y="-0.032813" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="127.8196" y="-0.032813" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="5.126074" y="-3.478125" style="font-size: 6.3px; font-family: 'cmsy10'; fill: #3a3a3a">0</tspan> + <tspan x="11.779805" y="-0.032813" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="31.454092" y="-0.032813" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">+</tspan> + <tspan x="40.748526" y="-0.032813" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">R</tspan> + <tspan x="47.001944" y="-0.032813" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">M</tspan> + <tspan x="54.76708" y="-0.032813" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">S</tspan> + <tspan x="60.479971" y="-0.032813" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">N</tspan> + <tspan x="67.212393" y="-0.032813" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="72.71874" y="-0.032813" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="76.418936" y="-0.032813" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="85.186026" y="-0.032813" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="90.158467" y="-0.032813" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">S</tspan> + <tspan x="95.871357" y="-0.032813" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">w</tspan> + <tspan x="103.232197" y="-0.032813" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="105.732686" y="-0.032813" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">G</tspan> + <tspan x="112.706807" y="-0.032813" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">L</tspan> + <tspan x="117.720967" y="-0.032813" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">U</tspan> + <tspan x="124.308369" y="-0.032813" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="133.523701" y="-0.032813" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="138.496143" y="-0.032813" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + </text> + </g> + </g> + <g id="text_9"> + <text style="font-size: 7.5px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #8a8a8a" x="133.056" y="342.869531" transform="rotate(-0 133.056 342.869531)">causal SDPA · QK-RMSNorm · RoPE</text> + </g> + <g id="patch_17"> + <path d="M 133.056 390.360435 +Q 133.056 375.085593 133.056 361.152392 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.2; stroke-linecap: round"/> + <path d="M 130.856 365.552392 +L 133.056 361.152392 +L 135.256 365.552392 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.2; stroke-linecap: round"/> + </g> + <g id="text_10"> + <text style="font-size: 8.5px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="133.056" y="406.916897" transform="rotate(-0 133.056 406.916897)">token embedding</text> + </g> + <g id="text_11"> + <text style="font-size: 7.8px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #8a8a8a" x="152.064" y="447.866598" transform="rotate(-0 152.064 447.866598)">42.75M (TinyStories 4k BPE) / 72.11M (FineWeb-Edu 32k BPE)</text> + </g> + <g id="text_12"> + <text style="font-size: 8.2px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; fill: #3a3a3a" transform="translate(11.705625 466.700156)">Nothing in this graph is modified for training: the trained checkpoint</text> + <text style="font-size: 8.2px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; fill: #3a3a3a" transform="translate(19.486656 475.882362)">is indistinguishable in form from a conventionally trained model.</text> + </g> + <g id="text_13"> + <text style="font-weight: 700; font-size: 10.5px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="446.688" y="21.411629" transform="rotate(-0 446.688 21.411629)">(B) Free phase (= inference)</text> + </g> + <g id="text_14"> + <!-- give each layer a state $z_l$ (3 of $L$ drawn); settle the disagreement energy --> + <g style="fill: #8a8a8a" transform="translate(292.2005 39.366071)"> + <text> + <tspan x="0" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">g</tspan> + <tspan x="5.395508" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="7.75708" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">v</tspan> + <tspan x="12.787354" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="18.016846" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="20.71875" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="25.948242" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="31.156982" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">c</tspan> + <tspan x="35.830322" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">h</tspan> + <tspan x="41.217529" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="43.919434" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">l</tspan> + <tspan x="46.281006" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="51.489746" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">y</tspan> + <tspan x="56.52002" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="61.749512" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="65.244141" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="67.946045" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="73.154785" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="75.856689" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="80.285156" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="83.61792" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="88.82666" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="92.159424" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="97.388916" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="106.451685" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="109.153589" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">(</tspan> + <tspan x="112.469751" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">3</tspan> + <tspan x="117.87771" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="120.579614" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">o</tspan> + <tspan x="125.780054" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">f</tspan> + <tspan x="128.772485" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="136.209985" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="138.91189" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">d</tspan> + <tspan x="144.307397" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="147.802026" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="153.010767" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">w</tspan> + <tspan x="159.962671" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">n</tspan> + <tspan x="165.349878" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">)</tspan> + <tspan x="168.66604" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">;</tspan> + <tspan x="171.52981" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="174.231714" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="178.660181" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="183.889673" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="187.222437" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="190.5552" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">l</tspan> + <tspan x="192.916772" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="198.146265" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="200.848169" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="204.180933" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">h</tspan> + <tspan x="209.56814" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="214.797632" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="217.499536" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">d</tspan> + <tspan x="222.895044" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="225.256616" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="229.685083" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="234.893823" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">g</tspan> + <tspan x="240.289331" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="243.78396" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="249.013452" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="254.242944" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">m</tspan> + <tspan x="262.522974" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="267.752466" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">n</tspan> + <tspan x="273.139673" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="276.472437" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="279.174341" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="284.403833" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">n</tspan> + <tspan x="289.79104" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="295.020532" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="298.515161" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">g</tspan> + <tspan x="303.910669" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">y</tspan> + <tspan x="100.09082" y="-0.15625" style="font-style: oblique; font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">z</tspan> + <tspan x="131.47439" y="-0.15625" style="font-style: oblique; font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">L</tspan> + <tspan x="104.55249" y="1.320312" style="font-style: oblique; font-size: 5.95px; font-family: 'DejaVu Sans'; fill: #8a8a8a">l</tspan> + </text> + </g> + </g> + <g id="text_15"> + <!-- $E(z)\ =\ \sum_l\ \frac{1}{2}\,\|\,z_l - f_l(z_{l-1})\,\|^2$ --> + <g style="fill: #3a3a3a" transform="translate(378.5955 62.505)"> + <text> + <tspan x="0" y="-0.8125" style="font-style: oblique; font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">E</tspan> + <tspan x="10.730713" y="-0.8125" style="font-style: oblique; font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="67.635488" y="-0.8125" style="font-style: oblique; font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="88.37897" y="-0.8125" style="font-style: oblique; font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="98.514785" y="-0.8125" style="font-style: oblique; font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="6.634277" y="-0.8125" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="16.242188" y="-0.8125" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="25.793667" y="-0.8125" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="60.680742" y="-0.8125" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">‖</tspan> + <tspan x="77.535464" y="-0.8125" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">−</tspan> + <tspan x="94.41835" y="-0.8125" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="120.067813" y="-0.8125" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="125.868994" y="-0.8125" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">‖</tspan> + <tspan x="40.046563" y="-0.8125" style="font-size: 10.5px; font-family: 'DejaVu Sans Display'; fill: #3a3a3a">∑</tspan> + <tspan x="44.046563" y="9.992188" style="font-style: oblique; font-size: 7.35px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="73.146963" y="0.992188" style="font-style: oblique; font-size: 7.35px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="92.075503" y="0.992188" style="font-style: oblique; font-size: 7.35px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="104.02626" y="0.992188" style="font-style: oblique; font-size: 7.35px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="52.987202" y="-5.570312" style="font-size: 7.35px; font-family: 'DejaVu Sans'; fill: #3a3a3a">1</tspan> + <tspan x="52.987202" y="3.43125" style="font-size: 7.35px; font-family: 'DejaVu Sans'; fill: #3a3a3a">2</tspan> + <tspan x="107.500283" y="0.992188" style="font-size: 7.35px; font-family: 'DejaVu Sans'; fill: #3a3a3a">−</tspan> + <tspan x="115.090737" y="0.992188" style="font-size: 7.35px; font-family: 'DejaVu Sans'; fill: #3a3a3a">1</tspan> + <tspan x="131.224268" y="-5.023438" style="font-size: 7.35px; font-family: 'DejaVu Sans'; fill: #3a3a3a">2</tspan> + </text> + <rect x="52.987202" y="-3.945312" width="4.676294" height="0.65625"/> + </g> + </g> + <g id="text_16"> + <!-- $z_3 = f_3(z_2)$ --> + <g style="fill: #3a3a3a" transform="translate(367.929 132.769152)"> + <text> + <tspan x="0" y="-0.15625" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="20.026318" y="-0.15625" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="30.960352" y="-0.15625" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="4.724121" y="1.320312" style="font-size: 6.3px; font-family: 'DejaVu Sans'; fill: #3a3a3a">3</tspan> + <tspan x="23.194775" y="1.320312" style="font-size: 6.3px; font-family: 'DejaVu Sans'; fill: #3a3a3a">3</tspan> + <tspan x="35.684473" y="1.320312" style="font-size: 6.3px; font-family: 'DejaVu Sans'; fill: #3a3a3a">2</tspan> + <tspan x="10.731885" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="27.449121" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="39.938818" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + </text> + </g> + </g> + <g id="text_17"> + <text style="font-size: 7.8px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: start; fill: #8a8a8a" x="494.208" y="132.438027" transform="rotate(-0 494.208 132.438027)">term = 0</text> + </g> + <g id="text_18"> + <!-- $z_2 = f_2(z_1)$ --> + <g style="fill: #3a3a3a" transform="translate(367.929 215.054866)"> + <text> + <tspan x="0" y="-0.15625" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="20.026318" y="-0.15625" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="30.960352" y="-0.15625" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="4.724121" y="1.320312" style="font-size: 6.3px; font-family: 'DejaVu Sans'; fill: #3a3a3a">2</tspan> + <tspan x="23.194775" y="1.320312" style="font-size: 6.3px; font-family: 'DejaVu Sans'; fill: #3a3a3a">2</tspan> + <tspan x="35.684473" y="1.320312" style="font-size: 6.3px; font-family: 'DejaVu Sans'; fill: #3a3a3a">1</tspan> + <tspan x="10.731885" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="27.449121" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="39.938818" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + </text> + </g> + </g> + <g id="text_19"> + <text style="font-size: 7.8px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: start; fill: #8a8a8a" x="494.208" y="214.723741" transform="rotate(-0 494.208 214.723741)">term = 0</text> + </g> + <g id="text_20"> + <!-- $z_1 = f_1(\mathrm{emb})$ --> + <g style="fill: #3a3a3a" transform="translate(362.394 297.34058)"> + <text> + <tspan x="0" y="-0.15625" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="20.026318" y="-0.15625" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="4.724121" y="1.320312" style="font-size: 6.3px; font-family: 'DejaVu Sans'; fill: #3a3a3a">1</tspan> + <tspan x="23.194775" y="1.320312" style="font-size: 6.3px; font-family: 'DejaVu Sans'; fill: #3a3a3a">1</tspan> + <tspan x="10.731885" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="27.449121" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="30.960352" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="36.497461" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="45.264551" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">b</tspan> + <tspan x="50.977441" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + </text> + </g> + </g> + <g id="text_21"> + <text style="font-size: 7.8px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: start; fill: #8a8a8a" x="494.208" y="297.009455" transform="rotate(-0 494.208 297.009455)">term = 0</text> + </g> + <g id="patch_18"> + <path d="M 389.664 194.928976 +Q 389.664 171.771648 389.664 149.844158 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.1; stroke-linecap: round"/> + <path d="M 387.464 154.244158 +L 389.664 149.844158 +L 391.864 154.244158 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.1; stroke-linecap: round"/> + </g> + <g id="patch_19"> + <path d="M 389.664 277.21469 +Q 389.664 254.057363 389.664 232.129872 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.1; stroke-linecap: round"/> + <path d="M 387.464 236.529872 +L 389.664 232.129872 +L 391.864 236.529872 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.1; stroke-linecap: round"/> + </g> + <g id="patch_20"> + <path d="M 389.664 351.955413 +Q 389.664 333.943917 389.664 317.162258 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.1; stroke-linecap: round"/> + <path d="M 387.464 321.562258 +L 389.664 317.162258 +L 391.864 321.562258 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.1; stroke-linecap: round"/> + </g> + <g id="text_22"> + <!-- $\mathrm{emb}(x)$ --> + <g style="fill: #3a3a3a" transform="translate(373.464 369.34058)"> + <text> + <tspan x="0" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="5.537109" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="14.304199" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">b</tspan> + <tspan x="20.01709" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="28.854492" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="23.52832" y="-0.15625" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">x</tspan> + </text> + </g> + </g> + <g id="text_23"> + <!-- The minimum is exact: $E=0$, states $\equiv$ forward activations, --> + <g style="fill: #3a3a3a" transform="translate(318.924 416.489304)"> + <text> + <tspan x="0" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">T</tspan> + <tspan x="5.115784" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="10.423767" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="15.576355" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="18.238525" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="26.39679" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="28.723633" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="34.031616" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="36.358459" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="44.516724" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="49.824707" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="57.982971" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="60.645142" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="62.971985" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="67.335327" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="69.997498" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="75.150085" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">x</tspan> + <tspan x="80.106384" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="85.238525" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="89.84314" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="93.126892" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">:</tspan> + <tspan x="95.948547" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="98.610718" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="108.196167" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="116.845154" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">0</tspan> + <tspan x="122.173584" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">,</tspan> + <tspan x="124.835754" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="127.497925" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="130.160095" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="134.523438" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="137.80719" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="142.939331" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="146.223083" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="151.375671" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="155.739014" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="160.032837" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">≡</tspan> + <tspan x="168.681824" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="171.343994" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="174.292419" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="179.416382" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="182.859619" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">w</tspan> + <tspan x="189.70929" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="194.841431" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="198.284668" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="203.60083" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="206.263" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="211.395142" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="215.999756" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="219.283508" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="221.610352" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">v</tspan> + <tspan x="226.56665" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="231.698792" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="234.982544" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="237.309387" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="242.43335" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="247.741333" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="252.104675" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">,</tspan> + <tspan x="101.272888" y="-0.921875" style="font-style: oblique; font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">E</tspan> + </text> + </g> + </g> + <g id="text_24"> + <text style="font-size: 8.4px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="446.688" y="437.060732" transform="rotate(-0 446.688 437.060732)">and one bottom-up pass reaches it.</text> + </g> + <g id="text_25"> + <text style="font-size: 8.2px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #8a8a8a" x="446.688" y="464.434116" transform="rotate(-0 446.688 464.434116)">The free phase adds nothing and changes nothing at inference.</text> + </g> + <g id="text_26"> + <text style="font-weight: 700; font-size: 10.5px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="769.824" y="21.370614" transform="rotate(-0 769.824 21.370614)">(C) Nudged phase (training only)</text> + </g> + <g id="text_27"> + <!-- add the loss at strength $\beta \ll 1$ and settle again: --> + <g style="fill: #8a8a8a" transform="translate(668.844 39.408571)"> + <text> + <tspan x="0" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="5.20874" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">d</tspan> + <tspan x="10.604248" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">d</tspan> + <tspan x="15.999756" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="18.70166" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="22.034424" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">h</tspan> + <tspan x="27.421631" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="32.651123" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="35.353027" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">l</tspan> + <tspan x="37.7146" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">o</tspan> + <tspan x="42.915039" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="47.343506" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="51.771973" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="54.473877" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="59.682617" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="63.015381" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="65.717285" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="70.145752" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="73.478516" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="76.973145" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="82.202637" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">n</tspan> + <tspan x="87.589844" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">g</tspan> + <tspan x="92.985352" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="96.318115" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">h</tspan> + <tspan x="101.705322" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="111.487793" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">≪</tspan> + <tspan x="122.042236" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">1</tspan> + <tspan x="127.450195" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="130.1521" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="135.36084" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">n</tspan> + <tspan x="140.748047" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">d</tspan> + <tspan x="146.143555" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="148.845459" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="153.273926" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="158.503418" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="161.836182" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="165.168945" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">l</tspan> + <tspan x="167.530518" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="172.76001" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="175.461914" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="180.670654" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">g</tspan> + <tspan x="186.066162" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="191.274902" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="193.636475" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">n</tspan> + <tspan x="199.023682" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">:</tspan> + <tspan x="104.407227" y="-0.109375" style="font-style: oblique; font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">β</tspan> + </text> + </g> + </g> + <g id="text_28"> + <!-- $E(z)\ +\ \beta\cdot\mathrm{CE}(\mathrm{logits}(z_3),\,y)$ --> + <g style="fill: #3a3a3a" transform="translate(701.364 65.34)"> + <text> + <tspan x="0" y="-0.578125" style="font-style: oblique; font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">E</tspan> + <tspan x="10.730713" y="-0.578125" style="font-style: oblique; font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="40.046563" y="-0.578125" style="font-style: oblique; font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">β</tspan> + <tspan x="104.846123" y="-0.578125" style="font-style: oblique; font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="126.519155" y="-0.578125" style="font-style: oblique; font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">y</tspan> + <tspan x="6.634277" y="-0.578125" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="16.242188" y="-0.578125" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="25.793667" y="-0.578125" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">+</tspan> + <tspan x="48.793145" y="-0.578125" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">⋅</tspan> + <tspan x="54.176445" y="-0.578125" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">C</tspan> + <tspan x="61.507988" y="-0.578125" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">E</tspan> + <tspan x="68.142266" y="-0.578125" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="72.238701" y="-0.578125" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="75.155938" y="-0.578125" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="81.58001" y="-0.578125" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">g</tspan> + <tspan x="88.245049" y="-0.578125" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="91.162285" y="-0.578125" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="95.279229" y="-0.578125" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="100.749688" y="-0.578125" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="115.334673" y="-0.578125" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="119.431109" y="-0.578125" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">,</tspan> + <tspan x="132.733023" y="-0.578125" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="110.357598" y="1.226562" style="font-size: 7.35px; font-family: 'DejaVu Sans'; fill: #3a3a3a">3</tspan> + </text> + </g> + </g> + <g id="text_29"> + <!-- pull $-\beta\,\nabla\mathrm{CE}$ --> + <g style="fill: #b03a2e" transform="translate(788.832 89.493714)"> + <text> + <tspan x="0" y="-0.109375" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #b03a2e">p</tspan> + <tspan x="5.474854" y="-0.109375" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #b03a2e">u</tspan> + <tspan x="10.941284" y="-0.109375" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #b03a2e">l</tspan> + <tspan x="13.337585" y="-0.109375" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #b03a2e">l</tspan> + <tspan x="15.733887" y="-0.109375" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #b03a2e"> </tspan> + <tspan x="18.475525" y="-0.109375" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #b03a2e">−</tspan> + <tspan x="32.606992" y="-0.109375" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #b03a2e">∇</tspan> + <tspan x="38.376645" y="-0.109375" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #b03a2e">C</tspan> + <tspan x="44.398984" y="-0.109375" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #b03a2e">E</tspan> + <tspan x="25.702332" y="-0.109375" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #b03a2e">β</tspan> + </text> + </g> + </g> + <g id="text_30"> + <text style="font-size: 7.3px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: start; fill: #b03a2e" x="788.832" y="104.871487" transform="rotate(-0 788.832 104.871487)">(the only place the label enters)</text> + </g> + <g id="patch_21"> + <path d="M 778.866838 93.443471 +Q 755.568635 102.856886 733.825358 111.642049 +" style="fill: none; stroke: #b03a2e; stroke-width: 1.5; stroke-linecap: round"/> + <path d="M 738.729106 112.033524 +L 733.825358 111.642049 +L 737.080788 107.953935 +z +" style="fill: #b03a2e; stroke: #b03a2e; stroke-width: 1.5; stroke-linecap: round"/> + </g> + <g id="text_31"> + <!-- $z_3^{\beta} = z_3 + d_3$ --> + <g style="fill: #3a3a3a" transform="translate(687.555 133.750714)"> + <text> + <tspan x="0" y="-0.51875" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="20.440576" y="-0.51875" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="40.466895" y="-0.51875" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="5.126074" y="-5.176563" style="font-style: oblique; font-size: 6.3px; font-family: 'DejaVu Sans'; fill: #3a3a3a">β</tspan> + <tspan x="4.724121" y="1.942187" style="font-size: 6.3px; font-family: 'DejaVu Sans'; fill: #3a3a3a">3</tspan> + <tspan x="25.164697" y="0.957812" style="font-size: 6.3px; font-family: 'DejaVu Sans'; fill: #3a3a3a">3</tspan> + <tspan x="46.179785" y="0.957812" style="font-size: 6.3px; font-family: 'DejaVu Sans'; fill: #3a3a3a">3</tspan> + <tspan x="11.146143" y="-0.51875" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="31.172461" y="-0.51875" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">+</tspan> + </text> + </g> + </g> + <g id="text_32"> + <!-- $z_2^{\beta} = z_2 + d_2$ --> + <g style="fill: #3a3a3a" transform="translate(687.555 216.081429)"> + <text> + <tspan x="0" y="-0.51875" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="20.440576" y="-0.51875" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="40.466895" y="-0.51875" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="5.126074" y="-5.176563" style="font-style: oblique; font-size: 6.3px; font-family: 'DejaVu Sans'; fill: #3a3a3a">β</tspan> + <tspan x="4.724121" y="1.942187" style="font-size: 6.3px; font-family: 'DejaVu Sans'; fill: #3a3a3a">2</tspan> + <tspan x="25.164697" y="0.957812" style="font-size: 6.3px; font-family: 'DejaVu Sans'; fill: #3a3a3a">2</tspan> + <tspan x="46.179785" y="0.957812" style="font-size: 6.3px; font-family: 'DejaVu Sans'; fill: #3a3a3a">2</tspan> + <tspan x="11.146143" y="-0.51875" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="31.172461" y="-0.51875" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">+</tspan> + </text> + </g> + </g> + <g id="text_33"> + <!-- $z_1^{\beta} = z_1 + d_1$ --> + <g style="fill: #3a3a3a" transform="translate(687.555 298.322143)"> + <text> + <tspan x="0" y="-0.60625" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="20.440576" y="-0.60625" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="40.466895" y="-0.60625" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="5.126074" y="-5.176563" style="font-style: oblique; font-size: 6.3px; font-family: 'DejaVu Sans'; fill: #3a3a3a">β</tspan> + <tspan x="4.724121" y="1.854688" style="font-size: 6.3px; font-family: 'DejaVu Sans'; fill: #3a3a3a">1</tspan> + <tspan x="25.164697" y="0.870313" style="font-size: 6.3px; font-family: 'DejaVu Sans'; fill: #3a3a3a">1</tspan> + <tspan x="46.179785" y="0.870313" style="font-size: 6.3px; font-family: 'DejaVu Sans'; fill: #3a3a3a">1</tspan> + <tspan x="11.146143" y="-0.60625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="31.172461" y="-0.60625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">+</tspan> + </text> + </g> + </g> + <g id="patch_22"> + <path d="M 712.8 194.928976 +Q 712.8 171.771648 712.8 149.844158 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.1; stroke-linecap: round"/> + <path d="M 710.6 154.244158 +L 712.8 149.844158 +L 715 154.244158 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.1; stroke-linecap: round"/> + </g> + <g id="patch_23"> + <path d="M 712.8 277.21469 +Q 712.8 254.057363 712.8 232.129872 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.1; stroke-linecap: round"/> + <path d="M 710.6 236.529872 +L 712.8 232.129872 +L 715 236.529872 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.1; stroke-linecap: round"/> + </g> + <g id="patch_24"> + <path d="M 712.8 351.955413 +Q 712.8 333.943917 712.8 317.162258 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.1; stroke-linecap: round"/> + <path d="M 710.6 321.562258 +L 712.8 317.162258 +L 715 321.562258 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.1; stroke-linecap: round"/> + </g> + <g id="text_34"> + <!-- $\mathrm{emb}(x)$ --> + <g style="fill: #3a3a3a" transform="translate(696.6 369.34058)"> + <text> + <tspan x="0" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="5.537109" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="14.304199" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">b</tspan> + <tspan x="20.01709" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="28.854492" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="23.52832" y="-0.15625" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">x</tspan> + </text> + </g> + </g> + <g id="patch_25"> + <path d="M 821.1456 144.498633 +Q 821.1456 169.370812 821.1456 192.56594 +" style="fill: none; stroke: #d95f02; stroke-width: 1.5; stroke-linecap: round"/> + <path d="M 823.3456 188.16594 +L 821.1456 192.56594 +L 818.9456 188.16594 +z +" style="fill: #d95f02; stroke: #d95f02; stroke-width: 1.5; stroke-linecap: round"/> + </g> + <g id="text_35"> + <!-- $d_2 = J_3^{\top} d_3$ --> + <g style="fill: #d95f02" transform="translate(836.352 171.908429)"> + <text> + <tspan x="0" y="-0.146875" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #d95f02">d</tspan> + <tspan x="20.149713" y="-0.146875" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #d95f02">J</tspan> + <tspan x="31.18288" y="-0.146875" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #d95f02">d</tspan> + <tspan x="5.474854" y="1.329687" style="font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #d95f02">2</tspan> + <tspan x="22.693414" y="2.314062" style="font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #d95f02">3</tspan> + <tspan x="24.501306" y="-3.592188" style="font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #d95f02">⊤</tspan> + <tspan x="36.657733" y="1.329687" style="font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #d95f02">3</tspan> + <tspan x="11.242548" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #d95f02">=</tspan> + </text> + </g> + </g> + <g id="patch_26"> + <path d="M 821.1456 226.784347 +Q 821.1456 251.656526 821.1456 274.851654 +" style="fill: none; stroke: #d95f02; stroke-width: 1.5; stroke-linecap: round"/> + <path d="M 823.3456 270.451654 +L 821.1456 274.851654 +L 818.9456 270.451654 +z +" style="fill: #d95f02; stroke: #d95f02; stroke-width: 1.5; stroke-linecap: round"/> + </g> + <g id="text_36"> + <!-- $d_1 = J_2^{\top} d_2$ --> + <g style="fill: #d95f02" transform="translate(836.352 254.237143)"> + <text> + <tspan x="0" y="-0.146875" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #d95f02">d</tspan> + <tspan x="20.149713" y="-0.146875" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #d95f02">J</tspan> + <tspan x="31.18288" y="-0.146875" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #d95f02">d</tspan> + <tspan x="5.474854" y="1.329687" style="font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #d95f02">1</tspan> + <tspan x="22.693414" y="2.314062" style="font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #d95f02">2</tspan> + <tspan x="24.501306" y="-3.592188" style="font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #d95f02">⊤</tspan> + <tspan x="36.657733" y="1.329687" style="font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #d95f02">2</tspan> + <tspan x="11.242548" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #d95f02">=</tspan> + </text> + </g> + </g> + <g id="text_37"> + <!-- The top state is pulled toward lower loss; each layer's mismatch $d_l$ --> + <g style="fill: #3a3a3a" transform="translate(629.544 416.481429)"> + <text> + <tspan x="0" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">T</tspan> + <tspan x="5.115784" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="10.423767" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="15.576355" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="18.238525" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="21.522278" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="26.64624" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="31.962402" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="34.624573" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="38.987915" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="42.271667" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="47.403809" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="50.687561" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="55.840149" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="58.502319" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="60.829163" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="65.192505" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="67.854675" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="73.170837" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="78.478821" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="80.805664" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="83.132507" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="88.285095" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="93.601257" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="96.263428" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="99.54718" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="104.671143" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">w</tspan> + <tspan x="111.520813" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="116.652954" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="120.096191" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="125.412354" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="128.074524" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="130.401367" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="135.52533" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">w</tspan> + <tspan x="142.375" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="147.527588" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="150.970825" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="153.632996" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="155.959839" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="161.083801" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="165.447144" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="169.810486" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">;</tspan> + <tspan x="172.632141" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="175.294312" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="180.446899" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="185.579041" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="190.183655" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="195.491638" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="198.153809" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="200.480652" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="205.612793" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">y</tspan> + <tspan x="210.569092" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="215.72168" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="219.164917" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">'</tspan> + <tspan x="221.467224" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="225.830566" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="228.492737" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="236.651001" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="238.977844" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="243.341187" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="251.499451" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="256.631592" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="259.915344" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="264.519958" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="269.827942" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="272.490112" y="-0.921875" style="font-style: oblique; font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="277.806274" y="0.390625" style="font-style: oblique; font-size: 5.88px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + </text> + </g> + </g> + <g id="text_38"> + <text style="font-size: 8.4px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="769.824" y="437.060732" transform="rotate(-0 769.824 437.060732)">transmits DOWN through the same weights, and the stack re-settles.</text> + </g> + <g id="text_39"> + <!-- ($J^{\top}$ = the transpose read a bidirectional physical device provides) --> + <g style="fill: #8a8a8a" transform="translate(633.991 464.918429)"> + <text> + <tspan x="0" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">(</tspan> + <tspan x="13.711999" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="16.334435" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">=</tspan> + <tspan x="23.247033" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="25.869469" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="29.104211" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">h</tspan> + <tspan x="34.33297" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="39.408654" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="42.03109" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="45.265832" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="48.657677" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="53.713219" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">n</tspan> + <tspan x="58.941979" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="63.240197" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">p</tspan> + <tspan x="68.477013" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">o</tspan> + <tspan x="73.524499" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="77.822716" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="82.8984" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="85.520837" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="88.912682" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="93.988366" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="99.043908" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">d</tspan> + <tspan x="104.280724" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="106.903161" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="111.958703" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="114.581139" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">b</tspan> + <tspan x="119.817956" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="122.11007" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">d</tspan> + <tspan x="127.346886" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="129.639001" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="133.030846" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="138.10653" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">c</tspan> + <tspan x="142.642419" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="145.87716" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="148.169274" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">o</tspan> + <tspan x="153.216759" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">n</tspan> + <tspan x="158.445519" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="163.501061" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">l</tspan> + <tspan x="165.793175" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="168.415612" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">p</tspan> + <tspan x="173.652428" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">h</tspan> + <tspan x="178.881188" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">y</tspan> + <tspan x="183.763512" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="188.06173" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="190.353844" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">c</tspan> + <tspan x="194.889733" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="199.945275" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">l</tspan> + <tspan x="202.237389" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="204.859826" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">d</tspan> + <tspan x="210.096642" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="215.172326" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">v</tspan> + <tspan x="220.05465" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="222.346764" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">c</tspan> + <tspan x="226.882653" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="231.958337" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="234.580773" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">p</tspan> + <tspan x="239.81759" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="243.209435" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">o</tspan> + <tspan x="248.256921" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">v</tspan> + <tspan x="253.139245" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="255.431359" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">d</tspan> + <tspan x="260.668175" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="265.743859" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="270.042077" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">)</tspan> + <tspan x="3.218628" y="-0.010938" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">J</tspan> + <tspan x="7.337572" y="-3.073438" style="font-size: 5.74px; font-family: 'DejaVu Sans'; fill: #8a8a8a">⊤</tspan> + </text> + </g> + </g> + <g id="text_40"> + <text style="font-weight: 700; font-size: 10px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: start; fill: #3a3a3a" x="38.016" y="506.073661" transform="rotate(-0 38.016 506.073661)">(D) The update — a difference measurement between the two settled states</text> + </g> + <g id="text_41"> + <!-- $\hat{g}\ =\ \left[\ \partial_\theta E(z^{\beta})\ -\ \partial_\theta E(z^{0})\ \right]\,/\,\beta$$\qquad\qquad(\partial_\theta E(z^0)\equiv 0\ \mathrm{here,\ since}\ E=0)$ --> + <g style="fill: #3a3a3a" transform="translate(303.3675 532.465714)"> + <text> + <tspan x="6.931885" y="-2.203125" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">̂</tspan> + <tspan x="12.120083" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="33.37299" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">∂</tspan> + <tspan x="50.339616" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="65.430233" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="74.981713" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">−</tspan> + <tspan x="89.234608" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">∂</tspan> + <tspan x="106.201234" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="121.277496" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="134.078689" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">/</tspan> + <tspan x="186.935046" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="191.031482" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">∂</tspan> + <tspan x="207.998108" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="223.074369" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="229.216459" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">≡</tspan> + <tspan x="240.059965" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">0</tspan> + <tspan x="250.149775" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="256.80456" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="263.264521" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="267.581415" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="274.041376" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">,</tspan> + <tspan x="282.834067" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="288.304526" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="291.221762" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="297.876547" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="303.649496" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="322.198779" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="333.042285" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">0</tspan> + <tspan x="339.722704" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="0" y="-0.890625" style="font-style: oblique; font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">g</tspan> + <tspan x="43.705339" y="-0.890625" style="font-style: oblique; font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">E</tspan> + <tspan x="54.436052" y="-0.890625" style="font-style: oblique; font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="99.566957" y="-0.890625" style="font-style: oblique; font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">E</tspan> + <tspan x="110.29767" y="-0.890625" style="font-style: oblique; font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="139.321033" y="-0.890625" style="font-style: oblique; font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">β</tspan> + <tspan x="201.363831" y="-0.890625" style="font-style: oblique; font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">E</tspan> + <tspan x="212.094543" y="-0.890625" style="font-style: oblique; font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="313.518847" y="-0.890625" style="font-style: oblique; font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">E</tspan> + <tspan x="26.372979" y="-0.467187" style="font-size: 9.321329px; font-family: 'STIXSizeOneSym'; fill: #3a3a3a">[</tspan> + <tspan x="128.783321" y="-0.467187" style="font-size: 9.321329px; font-family: 'STIXSizeOneSym'; fill: #3a3a3a">]</tspan> + <tspan x="38.907707" y="0.914062" style="font-style: oblique; font-size: 7.35px; font-family: 'DejaVu Sans'; fill: #3a3a3a">θ</tspan> + <tspan x="60.438803" y="-5.101562" style="font-style: oblique; font-size: 7.35px; font-family: 'DejaVu Sans'; fill: #3a3a3a">β</tspan> + <tspan x="94.769325" y="0.914062" style="font-style: oblique; font-size: 7.35px; font-family: 'DejaVu Sans'; fill: #3a3a3a">θ</tspan> + <tspan x="196.566199" y="0.914062" style="font-style: oblique; font-size: 7.35px; font-family: 'DejaVu Sans'; fill: #3a3a3a">θ</tspan> + <tspan x="116.300421" y="-5.101562" style="font-size: 7.35px; font-family: 'DejaVu Sans'; fill: #3a3a3a">0</tspan> + <tspan x="218.097294" y="-5.101562" style="font-size: 7.35px; font-family: 'DejaVu Sans'; fill: #3a3a3a">0</tspan> + </text> + </g> + </g> + <g id="text_42"> + <!-- per layer: $\Delta\theta_l\ \propto\ \langle\ d_l\ ,\ \partial f_l(z_{l-1})/\partial\theta_l\ \rangle\ /\ \beta$ --> + <g style="fill: #3a3a3a" transform="translate(382.955 554.66)"> + <text> + <tspan x="0" y="-0.328125" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="6.030273" y="-0.328125" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="11.875" y="-0.328125" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="15.780762" y="-0.328125" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="18.800537" y="-0.328125" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="21.439941" y="-0.328125" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="27.261475" y="-0.328125" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">y</tspan> + <tspan x="32.883545" y="-0.328125" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="38.728271" y="-0.328125" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="42.634033" y="-0.328125" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">:</tspan> + <tspan x="45.834717" y="-0.328125" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="48.854492" y="-0.328125" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="51.874268" y="-0.328125" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">Δ</tspan> + <tspan x="71.241839" y="-0.328125" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">∝</tspan> + <tspan x="82.963732" y="-0.328125" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">⟨</tspan> + <tspan x="100.990697" y="-0.328125" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">,</tspan> + <tspan x="108.945988" y="-0.328125" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">∂</tspan> + <tspan x="119.323845" y="-0.328125" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="142.531804" y="-0.328125" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="146.238103" y="-0.328125" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">/</tspan> + <tspan x="149.438786" y="-0.328125" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">∂</tspan> + <tspan x="165.369102" y="-0.328125" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">⟩</tspan> + <tspan x="172.160087" y="-0.328125" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">/</tspan> + <tspan x="58.373047" y="-0.328125" style="font-style: oblique; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">θ</tspan> + <tspan x="89.754717" y="-0.328125" style="font-style: oblique; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="113.858342" y="-0.328125" style="font-style: oblique; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="123.030144" y="-0.328125" style="font-style: oblique; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="154.35114" y="-0.328125" style="font-style: oblique; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">θ</tspan> + <tspan x="178.445456" y="-0.328125" style="font-style: oblique; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">β</tspan> + <tspan x="64.185303" y="1.3125" style="font-style: oblique; font-size: 6.65px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="95.78499" y="1.3125" style="font-style: oblique; font-size: 6.65px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="117.202824" y="1.3125" style="font-style: oblique; font-size: 6.65px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="128.016716" y="1.3125" style="font-style: oblique; font-size: 6.65px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="160.163396" y="1.3125" style="font-style: oblique; font-size: 6.65px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="131.15988" y="1.3125" style="font-size: 6.65px; font-family: 'DejaVu Sans'; fill: #3a3a3a">−</tspan> + <tspan x="138.027434" y="1.3125" style="font-size: 6.65px; font-family: 'DejaVu Sans'; fill: #3a3a3a">1</tspan> + </text> + </g> + </g> + <g id="text_43"> + <text style="font-size: 8.2px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; fill: #3a3a3a" transform="translate(257.54125 566.753059)">Each layer updates from its own boundary mismatch — no global backward graph, no global tape, no loss</text> + <!-- derivatives except the top nudge. $\beta\to 0$ gives the exact gradient; bias is $O(\beta)$; a $\pm\beta$ two-sided read cancels it to $O(\beta^2)$. --> + <g style="fill: #3a3a3a" transform="translate(230.881 577.412803)"> + <text> + <tspan x="0" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="5.236816" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="10.3125" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="13.704346" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="15.99646" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">v</tspan> + <tspan x="20.878784" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="25.934326" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="29.169067" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="31.461182" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">v</tspan> + <tspan x="36.343506" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="41.419189" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="45.717407" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="48.339844" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="53.415527" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">x</tspan> + <tspan x="58.297852" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="62.83374" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="67.909424" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="73.14624" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="76.380981" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="79.003418" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="82.238159" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="87.466919" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="92.542603" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="95.165039" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="98.39978" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="103.447266" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="108.684082" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="111.306519" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="116.535278" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="121.764038" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="127.000854" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">g</tspan> + <tspan x="132.237671" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="137.313354" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">.</tspan> + <tspan x="139.935791" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="142.558228" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="152.052979" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">→</tspan> + <tspan x="160.572876" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">0</tspan> + <tspan x="165.821777" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="168.444214" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">g</tspan> + <tspan x="173.68103" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="175.973145" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">v</tspan> + <tspan x="180.855469" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="185.931152" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="190.22937" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="192.851807" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="196.086548" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="201.315308" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="206.390991" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="209.013428" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="214.089111" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">x</tspan> + <tspan x="218.971436" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="224.026978" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="228.562866" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="231.797607" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="234.420044" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">g</tspan> + <tspan x="239.65686" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="243.048706" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="248.104248" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="253.341064" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="255.633179" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="260.708862" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="265.937622" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="269.172363" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">;</tspan> + <tspan x="271.951904" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="274.574341" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">b</tspan> + <tspan x="279.811157" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="282.103271" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="287.158813" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="291.457031" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="294.079468" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="296.371582" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="300.6698" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="309.785889" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="318.269531" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="321.488159" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">;</tspan> + <tspan x="324.2677" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="326.890137" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="331.945679" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="334.568115" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">±</tspan> + <tspan x="346.745728" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="349.368164" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="352.602905" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">w</tspan> + <tspan x="359.350342" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="364.397827" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">-</tspan> + <tspan x="367.374756" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="371.672974" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="373.965088" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="379.201904" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="384.277588" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="389.514404" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="392.136841" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="395.528687" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="400.60437" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="405.659912" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="410.896729" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="413.519165" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="418.055054" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="423.110596" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="428.339355" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="432.875244" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="437.950928" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="440.243042" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="444.54126" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="447.163696" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="449.455811" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="452.690552" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="455.312988" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="458.547729" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="463.595215" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="472.711304" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="485.690219" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="488.908847" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">.</tspan> + <tspan x="145.180664" y="-0.78125" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">β</tspan> + <tspan x="303.292236" y="-0.78125" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">O</tspan> + <tspan x="313.004517" y="-0.78125" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">β</tspan> + <tspan x="341.480713" y="-0.78125" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">β</tspan> + <tspan x="466.217651" y="-0.78125" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">O</tspan> + <tspan x="475.929932" y="-0.78125" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">β</tspan> + <tspan x="481.797238" y="-3.84375" style="font-size: 5.74px; font-family: 'DejaVu Sans'; fill: #3a3a3a">2</tspan> + </text> + </g> + </g> + <g id="text_44"> + <text style="font-weight: 700; font-size: 10px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: start; fill: #3a3a3a" x="38.016" y="615.06317" transform="rotate(-0 38.016 615.06317)">(E) This is not predictive coding — same energy family, different measurement</text> + </g> + <g id="text_45"> + <text style="font-size: 8.2px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: start; fill: #3a3a3a" x="42.768" y="637.234116" transform="rotate(-0 42.768 637.234116)">· PC (as typically run): ONE settled phase with the target clamped hard; the update uses the raw errors of that single state → finite-clamp</text> + </g> + <g id="text_46"> + <text style="font-size: 8.2px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: start; fill: #3a3a3a" x="42.768" y="653.005545" transform="rotate(-0 42.768 653.005545)"> bias. Its "exact-BP" results require freezing predictions during error transport (fixed-prediction) — backprop re-expressed in local variables.</text> + </g> + <g id="text_47"> + <text style="font-size: 8.2px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: start; fill: #3a3a3a" x="42.768" y="670.859741" transform="rotate(-0 42.768 670.859741)">· EP (here): TWO phases and an infinitesimal nudge; the update is a difference quotient in β → bias is measured and controllable, and the</text> + </g> + <g id="text_48"> + <text style="font-size: 8.2px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: start; fill: #3a3a3a" x="42.768" y="686.605545" transform="rotate(-0 42.768 686.605545)"> settle stays fully self-consistent — which is what physical hardware actually does.</text> + </g> + <g id="text_49"> + <text style="font-size: 8.2px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: start; fill: #3a3a3a" x="42.768" y="701.691259" transform="rotate(-0 42.768 701.691259)">· The free phase is the zero-reference of the measurement: on analog hardware the subtraction cancels state-independent device offsets.</text> + </g> + </g> + </g> + <defs> + <clipPath id="p7fae8feff8"> + <rect x="0" y="0" width="950.4" height="720"/> + </clipPath> + </defs> +</svg> diff --git a/assets/figs/fig_primer_block.emf b/assets/figs/fig_primer_block.emf Binary files differnew file mode 100644 index 0000000..7993cc9 --- /dev/null +++ b/assets/figs/fig_primer_block.emf diff --git a/assets/figs/fig_primer_block.png b/assets/figs/fig_primer_block.png Binary files differnew file mode 100644 index 0000000..b5eb58d --- /dev/null +++ b/assets/figs/fig_primer_block.png diff --git a/assets/figs/fig_primer_block.svg b/assets/figs/fig_primer_block.svg new file mode 100644 index 0000000..7a901a9 --- /dev/null +++ b/assets/figs/fig_primer_block.svg @@ -0,0 +1,1956 @@ +<?xml version="1.0" encoding="utf-8" standalone="no"?> +<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" + "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"> +<svg xmlns:xlink="http://www.w3.org/1999/xlink" width="720pt" height="1022.4pt" viewBox="0 0 720 1022.4" xmlns="http://www.w3.org/2000/svg" version="1.1"> + <metadata> + <rdf:RDF xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:cc="http://creativecommons.org/ns#" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"> + <cc:Work> + <dc:type rdf:resource="http://purl.org/dc/dcmitype/StillImage"/> + <dc:date>2026-07-15T09:29:45.734111</dc:date> + <dc:format>image/svg+xml</dc:format> + <dc:creator> + <cc:Agent> + <dc:title>Matplotlib v3.10.8, https://matplotlib.org/</dc:title> + </cc:Agent> + </dc:creator> + </cc:Work> + </rdf:RDF> + </metadata> + <defs> + <style type="text/css">*{stroke-linejoin: round; stroke-linecap: butt}</style> + </defs> + <g id="figure_1"> + <g id="patch_1"> + <path d="M 0 1022.4 +L 720 1022.4 +L 720 0 +L 0 0 +z +" style="fill: #ffffff"/> + </g> + <g id="axes_1"> + <g id="line2d_1"> + <path d="M 288 83.52 +L 633.6 83.52 +L 633.6 505.44 +" clip-path="url(#pdba9adb895)" style="fill: none; stroke: #555555; stroke-width: 1.4; stroke-linecap: round"/> + </g> + <g id="patch_2"> + <path d="M 182.16 122.04 +L 393.84 122.04 +Q 397.8 122.04 397.8 118.08 +L 397.8 99.36 +Q 397.8 95.4 393.84 95.4 +L 182.16 95.4 +Q 178.2 95.4 178.2 99.36 +L 178.2 118.08 +Q 178.2 122.04 182.16 122.04 +z +" clip-path="url(#pdba9adb895)" style="fill: #eaf1fa; stroke: #2c6fbb; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_3"> + <path d="M 54.36 165.24 +L 176.04 165.24 +Q 180 165.24 180 161.28 +L 180 142.56 +Q 180 138.6 176.04 138.6 +L 54.36 138.6 +Q 50.4 138.6 50.4 142.56 +L 50.4 161.28 +Q 50.4 165.24 54.36 165.24 +z +" clip-path="url(#pdba9adb895)" style="fill: #faf6ee; stroke: #c9b895; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_4"> + <path d="M 227.16 165.24 +L 348.84 165.24 +Q 352.8 165.24 352.8 161.28 +L 352.8 142.56 +Q 352.8 138.6 348.84 138.6 +L 227.16 138.6 +Q 223.2 138.6 223.2 142.56 +L 223.2 161.28 +Q 223.2 165.24 227.16 165.24 +z +" clip-path="url(#pdba9adb895)" style="fill: #faf6ee; stroke: #c9b895; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_5"> + <path d="M 54.36 199.08 +L 176.04 199.08 +Q 180 199.08 180 195.12 +L 180 176.4 +Q 180 172.44 176.04 172.44 +L 54.36 172.44 +Q 50.4 172.44 50.4 176.4 +L 50.4 195.12 +Q 50.4 199.08 54.36 199.08 +z +" clip-path="url(#pdba9adb895)" style="fill: #f4f4f4; stroke: #9a9a9a; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_6"> + <path d="M 227.16 199.08 +L 348.84 199.08 +Q 352.8 199.08 352.8 195.12 +L 352.8 176.4 +Q 352.8 172.44 348.84 172.44 +L 227.16 172.44 +Q 223.2 172.44 223.2 176.4 +L 223.2 195.12 +Q 223.2 199.08 227.16 199.08 +z +" clip-path="url(#pdba9adb895)" style="fill: #f4f4f4; stroke: #9a9a9a; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_7"> + <path d="M 399.96 199.08 +L 521.64 199.08 +Q 525.6 199.08 525.6 195.12 +L 525.6 176.4 +Q 525.6 172.44 521.64 172.44 +L 399.96 172.44 +Q 396 172.44 396 176.4 +L 396 195.12 +Q 396 199.08 399.96 199.08 +z +" clip-path="url(#pdba9adb895)" style="fill: #f4f4f4; stroke: #9a9a9a; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_8"> + <path d="M 54.36 232.92 +L 176.04 232.92 +Q 180 232.92 180 228.96 +L 180 210.24 +Q 180 206.28 176.04 206.28 +L 54.36 206.28 +Q 50.4 206.28 50.4 210.24 +L 50.4 228.96 +Q 50.4 232.92 54.36 232.92 +z +" clip-path="url(#pdba9adb895)" style="fill: #f4f4f4; stroke: #9a9a9a; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_9"> + <path d="M 227.16 232.92 +L 348.84 232.92 +Q 352.8 232.92 352.8 228.96 +L 352.8 210.24 +Q 352.8 206.28 348.84 206.28 +L 227.16 206.28 +Q 223.2 206.28 223.2 210.24 +L 223.2 228.96 +Q 223.2 232.92 227.16 232.92 +z +" clip-path="url(#pdba9adb895)" style="fill: #f4f4f4; stroke: #9a9a9a; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_10"> + <path d="M 174.96 276.12 +L 401.04 276.12 +Q 405 276.12 405 272.16 +L 405 253.44 +Q 405 249.48 401.04 249.48 +L 174.96 249.48 +Q 171 249.48 171 253.44 +L 171 272.16 +Q 171 276.12 174.96 276.12 +z +" clip-path="url(#pdba9adb895)" style="fill: #f1ebf8; stroke: #7b5aa6; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_11"> + <path d="M 174.96 309.96 +L 401.04 309.96 +Q 405 309.96 405 306 +L 405 287.28 +Q 405 283.32 401.04 283.32 +L 174.96 283.32 +Q 171 283.32 171 287.28 +L 171 306 +Q 171 309.96 174.96 309.96 +z +" clip-path="url(#pdba9adb895)" style="fill: #f4f4f4; stroke: #9a9a9a; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_12"> + <path d="M 174.96 343.8 +L 401.04 343.8 +Q 405 343.8 405 339.84 +L 405 321.12 +Q 405 317.16 401.04 317.16 +L 174.96 317.16 +Q 171 317.16 171 321.12 +L 171 339.84 +Q 171 343.8 174.96 343.8 +z +" clip-path="url(#pdba9adb895)" style="fill: #f4f4f4; stroke: #9a9a9a; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_13"> + <path d="M 174.96 377.64 +L 401.04 377.64 +Q 405 377.64 405 373.68 +L 405 354.96 +Q 405 351 401.04 351 +L 174.96 351 +Q 171 351 171 354.96 +L 171 373.68 +Q 171 377.64 174.96 377.64 +z +" clip-path="url(#pdba9adb895)" style="fill: #f1ebf8; stroke: #7b5aa6; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_14"> + <path d="M 174.96 411.48 +L 401.04 411.48 +Q 405 411.48 405 407.52 +L 405 388.8 +Q 405 384.84 401.04 384.84 +L 174.96 384.84 +Q 171 384.84 171 388.8 +L 171 407.52 +Q 171 411.48 174.96 411.48 +z +" clip-path="url(#pdba9adb895)" style="fill: #f4f4f4; stroke: #9a9a9a; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_15"> + <path d="M 174.96 445.32 +L 401.04 445.32 +Q 405 445.32 405 441.36 +L 405 422.64 +Q 405 418.68 401.04 418.68 +L 174.96 418.68 +Q 171 418.68 171 422.64 +L 171 441.36 +Q 171 445.32 174.96 445.32 +z +" clip-path="url(#pdba9adb895)" style="fill: #eaf1fa; stroke: #2c6fbb; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_16"> + <path d="M 174.96 479.16 +L 401.04 479.16 +Q 405 479.16 405 475.2 +L 405 456.48 +Q 405 452.52 401.04 452.52 +L 174.96 452.52 +Q 171 452.52 171 456.48 +L 171 475.2 +Q 171 479.16 174.96 479.16 +z +" clip-path="url(#pdba9adb895)" style="fill: #faf6ee; stroke: #c9b895; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="line2d_2"> + <path d="M 288 540 +L 633.6 540 +L 633.6 754.56 +" clip-path="url(#pdba9adb895)" style="fill: none; stroke: #555555; stroke-width: 1.4; stroke-linecap: round"/> + </g> + <g id="patch_17"> + <path d="M 113.76 586.44 +L 260.64 586.44 +Q 264.6 586.44 264.6 582.48 +L 264.6 563.76 +Q 264.6 559.8 260.64 559.8 +L 113.76 559.8 +Q 109.8 559.8 109.8 563.76 +L 109.8 582.48 +Q 109.8 586.44 113.76 586.44 +z +" clip-path="url(#pdba9adb895)" style="fill: #eaf1fa; stroke: #2c6fbb; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_18"> + <path d="M 315.36 586.44 +L 462.24 586.44 +Q 466.2 586.44 466.2 582.48 +L 466.2 563.76 +Q 466.2 559.8 462.24 559.8 +L 315.36 559.8 +Q 311.4 559.8 311.4 563.76 +L 311.4 582.48 +Q 311.4 586.44 315.36 586.44 +z +" clip-path="url(#pdba9adb895)" style="fill: #eaf1fa; stroke: #2c6fbb; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_19"> + <path d="M 146.16 620.28 +L 228.24 620.28 +Q 232.2 620.28 232.2 616.32 +L 232.2 597.6 +Q 232.2 593.64 228.24 593.64 +L 146.16 593.64 +Q 142.2 593.64 142.2 597.6 +L 142.2 616.32 +Q 142.2 620.28 146.16 620.28 +z +" clip-path="url(#pdba9adb895)" style="fill: #f4f4f4; stroke: #9a9a9a; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_20"> + <path d="M 207.36 658.44 +L 368.64 658.44 +Q 372.6 658.44 372.6 654.48 +L 372.6 635.76 +Q 372.6 631.8 368.64 631.8 +L 207.36 631.8 +Q 203.4 631.8 203.4 635.76 +L 203.4 654.48 +Q 203.4 658.44 207.36 658.44 +z +" clip-path="url(#pdba9adb895)" style="fill: #f4f4f4; stroke: #9a9a9a; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_21"> + <path d="M 200.16 696.6 +L 375.84 696.6 +Q 379.8 696.6 379.8 692.64 +L 379.8 673.92 +Q 379.8 669.96 375.84 669.96 +L 200.16 669.96 +Q 196.2 669.96 196.2 673.92 +L 196.2 692.64 +Q 196.2 696.6 200.16 696.6 +z +" clip-path="url(#pdba9adb895)" style="fill: #eaf1fa; stroke: #2c6fbb; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_22"> + <path d="M 200.16 730.44 +L 375.84 730.44 +Q 379.8 730.44 379.8 726.48 +L 379.8 707.76 +Q 379.8 703.8 375.84 703.8 +L 200.16 703.8 +Q 196.2 703.8 196.2 707.76 +L 196.2 726.48 +Q 196.2 730.44 200.16 730.44 +z +" clip-path="url(#pdba9adb895)" style="fill: #faf6ee; stroke: #c9b895; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_23"> + <path d="M 37.44 846.36 +L 59.04 846.36 +Q 61.56 846.36 61.56 843.84 +L 61.56 833.76 +Q 61.56 831.24 59.04 831.24 +L 37.44 831.24 +Q 34.92 831.24 34.92 833.76 +L 34.92 843.84 +Q 34.92 846.36 37.44 846.36 +z +" clip-path="url(#pdba9adb895)" style="fill: #eaf1fa; stroke: #2c6fbb; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_24"> + <path d="M 37.44 870.84 +L 59.04 870.84 +Q 61.56 870.84 61.56 868.32 +L 61.56 858.24 +Q 61.56 855.72 59.04 855.72 +L 37.44 855.72 +Q 34.92 855.72 34.92 858.24 +L 34.92 868.32 +Q 34.92 870.84 37.44 870.84 +z +" clip-path="url(#pdba9adb895)" style="fill: #f1ebf8; stroke: #7b5aa6; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_25"> + <path d="M 37.44 895.32 +L 59.04 895.32 +Q 61.56 895.32 61.56 892.8 +L 61.56 882.72 +Q 61.56 880.2 59.04 880.2 +L 37.44 880.2 +Q 34.92 880.2 34.92 882.72 +L 34.92 892.8 +Q 34.92 895.32 37.44 895.32 +z +" clip-path="url(#pdba9adb895)" style="fill: #faf6ee; stroke: #c9b895; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_26"> + <path d="M 275.04 895.32 +L 296.64 895.32 +Q 299.16 895.32 299.16 892.8 +L 299.16 882.72 +Q 299.16 880.2 296.64 880.2 +L 275.04 880.2 +Q 272.52 880.2 272.52 882.72 +L 272.52 892.8 +Q 272.52 895.32 275.04 895.32 +z +" clip-path="url(#pdba9adb895)" style="fill: #f4f4f4; stroke: #9a9a9a; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="text_1"> + <text style="font-weight: 700; font-size: 12.5px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="360" y="26.440391" transform="rotate(-0 360 26.440391)">One transformer block, op by op</text> + </g> + <g id="text_2"> + <!-- OLMo2 ordering — RMSNorm after each sublayer, inside the residual. $C{=}512$ $H{=}8$ $h_d{=}64$ $T{=}256$ $h_{ff}{=}1408$ --> + <g style="fill: #8a8a8a" transform="translate(100.664 46.34)"> + <text> + <tspan x="0" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">O</tspan> + <tspan x="6.887207" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">L</tspan> + <tspan x="11.762085" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">M</tspan> + <tspan x="19.311523" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">o</tspan> + <tspan x="24.664917" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">2</tspan> + <tspan x="30.231934" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="33.013306" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">o</tspan> + <tspan x="38.366699" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="41.964111" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">d</tspan> + <tspan x="47.518311" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="52.901611" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="56.499023" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="58.930054" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">n</tspan> + <tspan x="64.475708" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">g</tspan> + <tspan x="70.029907" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="81.561279" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="84.342651" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">R</tspan> + <tspan x="90.422363" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">M</tspan> + <tspan x="97.971802" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">S</tspan> + <tspan x="103.526001" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">N</tspan> + <tspan x="110.071411" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">o</tspan> + <tspan x="115.424805" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="119.022217" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">m</tspan> + <tspan x="127.545776" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="130.327148" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="135.689087" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">f</tspan> + <tspan x="138.769531" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="142.200317" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="147.583618" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="151.18103" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="153.962402" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="159.345703" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="164.707642" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">c</tspan> + <tspan x="169.518433" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">h</tspan> + <tspan x="175.064087" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="177.845459" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="182.404175" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">u</tspan> + <tspan x="187.949829" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">b</tspan> + <tspan x="193.504028" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">l</tspan> + <tspan x="195.935059" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="201.296997" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">y</tspan> + <tspan x="206.47522" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="211.858521" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="215.455933" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">,</tspan> + <tspan x="218.237305" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="221.018677" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="223.449707" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">n</tspan> + <tspan x="228.995361" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="233.554077" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="235.985107" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">d</tspan> + <tspan x="241.539307" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="246.922607" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="249.703979" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="253.134766" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">h</tspan> + <tspan x="258.68042" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="264.063721" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="266.845093" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="270.442505" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="275.825806" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="280.384521" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="282.815552" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">d</tspan> + <tspan x="288.369751" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">u</tspan> + <tspan x="293.915405" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="299.277344" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">l</tspan> + <tspan x="301.708374" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">.</tspan> + <tspan x="304.489746" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="307.271118" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="310.05249" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="312.833862" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="315.615234" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="326.210938" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">=</tspan> + <tspan x="335.247192" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">5</tspan> + <tspan x="340.814209" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">1</tspan> + <tspan x="346.381226" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">2</tspan> + <tspan x="351.948242" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="354.729614" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="357.510986" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="368.57666" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">=</tspan> + <tspan x="377.612915" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">8</tspan> + <tspan x="383.179932" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="385.961304" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="388.742676" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="402.908447" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">=</tspan> + <tspan x="411.944702" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">6</tspan> + <tspan x="417.511719" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">4</tspan> + <tspan x="423.078735" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="425.860107" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="428.641479" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="438.472412" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">=</tspan> + <tspan x="447.508667" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">2</tspan> + <tspan x="453.075684" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">5</tspan> + <tspan x="458.6427" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">6</tspan> + <tspan x="464.209717" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="466.991089" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="469.772461" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="484.362915" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">=</tspan> + <tspan x="493.39917" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">1</tspan> + <tspan x="498.966187" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">4</tspan> + <tspan x="504.533203" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">0</tspan> + <tspan x="510.10022" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">8</tspan> + <tspan x="72.811279" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans Display'; fill: #8a8a8a">—</tspan> + <tspan x="318.396606" y="-0.15625" style="font-style: oblique; font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">C</tspan> + <tspan x="360.292358" y="-0.15625" style="font-style: oblique; font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">H</tspan> + <tspan x="391.524048" y="-0.15625" style="font-style: oblique; font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">h</tspan> + <tspan x="431.422852" y="-0.15625" style="font-style: oblique; font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">T</tspan> + <tspan x="472.553833" y="-0.15625" style="font-style: oblique; font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">h</tspan> + <tspan x="397.069702" y="1.320312" style="font-style: oblique; font-size: 6.16px; font-family: 'DejaVu Sans'; fill: #8a8a8a">d</tspan> + <tspan x="478.099487" y="1.320312" style="font-style: oblique; font-size: 6.16px; font-family: 'DejaVu Sans'; fill: #8a8a8a">f</tspan> + <tspan x="480.255798" y="1.320312" style="font-style: oblique; font-size: 6.16px; font-family: 'DejaVu Sans'; fill: #8a8a8a">f</tspan> + </text> + </g> + </g> + <g id="text_3"> + <text style="font-style: italic; font-size: 9.5px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; fill: #8a8a8a" transform="translate(25.661406 345.993867) rotate(-90)">attention sublayer</text> + </g> + <g id="text_4"> + <!-- input $z$ $(B,T,C)$ --> + <g style="fill: #3a3a3a" transform="translate(246.77 77.4925)"> + <text> + <tspan x="0" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="3.256348" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="10.019531" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="16.819824" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="23.583008" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="28.124268" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="36.418213" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="39.725586" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="43.032959" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="31.431641" y="-0.40625" style="font-style: oblique; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="50.046631" y="-0.40625" style="font-style: oblique; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">B</tspan> + <tspan x="61.43457" y="-0.40625" style="font-style: oblique; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">T</tspan> + <tspan x="72.108154" y="-0.40625" style="font-style: oblique; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">C</tspan> + <tspan x="46.340332" y="-0.40625" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="56.563965" y="-0.40625" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">,</tspan> + <tspan x="67.237549" y="-0.40625" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">,</tspan> + <tspan x="78.741455" y="-0.40625" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + </text> + </g> + </g> + <g id="patch_27"> + <path d="M 633.19823 501.12524 +Q 467.999756 503.280003 304.366397 505.414351 +" style="fill: none; stroke: #555555; stroke-width: 1.4; stroke-linecap: round"/> + <path d="M 308.392142 507.362012 +L 304.366397 505.414351 +L 308.339972 503.362352 +z +" style="fill: #555555; stroke: #555555; stroke-width: 1.4; stroke-linecap: round"/> + </g> + <g id="text_5"> + <text style="font-size: 8px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; fill: #8a8a8a" transform="translate(651.6475 318.636875) rotate(-90)">residual rail</text> + </g> + <g id="patch_28"> + <path d="M 288 85.362188 +Q 288 90.721592 288 94.795257 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 290 90.795257 +L 288 94.795257 +L 286 90.795257 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="text_6"> + <!-- matmul $W_{qkv}: C \to 3C$ --> + <g style="fill: #3a3a3a" transform="translate(241.689 110.655)"> + <text> + <tspan x="0" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="8.401794" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="13.687134" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="17.068909" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="25.470703" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="30.937134" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="33.333435" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="36.075073" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="60.172998" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">:</tspan> + <tspan x="72.461938" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">→</tspan> + <tspan x="81.369104" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">3</tspan> + <tspan x="38.816711" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">W</tspan> + <tspan x="64.759241" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">C</tspan> + <tspan x="86.856592" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">C</tspan> + <tspan x="47.344849" y="1.320312" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">q</tspan> + <tspan x="51.177246" y="1.320312" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">k</tspan> + <tspan x="54.673572" y="1.320312" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">v</tspan> + </text> + </g> + </g> + <g id="text_7"> + <!-- $(B,T,3C)$ — split into $q,k,v$ --> + <g style="fill: #8a8a8a" transform="translate(406.8 110.81)"> + <text> + <tspan x="0" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">(</tspan> + <tspan x="8.205811" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">,</tspan> + <tspan x="16.772766" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">,</tspan> + <tspan x="20.682068" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">3</tspan> + <tspan x="30.857422" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">)</tspan> + <tspan x="33.832214" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="43.880981" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="46.304749" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="50.277344" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">p</tspan> + <tspan x="55.117432" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">l</tspan> + <tspan x="57.235901" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="59.35437" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="62.344055" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="64.767822" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="66.886292" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">n</tspan> + <tspan x="71.718933" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="74.708618" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">o</tspan> + <tspan x="79.373718" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="86.637573" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">,</tspan> + <tspan x="94.962524" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">,</tspan> + <tspan x="2.974792" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">B</tspan> + <tspan x="12.115112" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">T</tspan> + <tspan x="25.533325" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">C</tspan> + <tspan x="81.797485" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">q</tspan> + <tspan x="90.546875" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">k</tspan> + <tspan x="98.871826" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">v</tspan> + <tspan x="36.255981" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans Display'; fill: #8a8a8a">—</tspan> + </text> + </g> + </g> + <g id="text_8"> + <!-- $q$ --> + <g style="fill: #3a3a3a" transform="translate(112.16 137.242109)"> + <text> + <tspan x="0" y="-0.40625" style="font-style: oblique; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">q</tspan> + </text> + </g> + </g> + <g id="text_9"> + <!-- $k$ --> + <g style="fill: #3a3a3a" transform="translate(285.245 137.261406)"> + <text> + <tspan x="0" y="-0.40625" style="font-style: oblique; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">k</tspan> + </text> + </g> + </g> + <g id="text_10"> + <!-- $v$ --> + <g style="fill: #3a3a3a" transform="translate(457.95 137.261406)"> + <text> + <tspan x="0" y="-0.53125" style="font-style: oblique; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">v</tspan> + </text> + </g> + </g> + <g id="patch_29"> + <path d="M 266.000098 120.295986 +Q 194.399445 130.320078 124.072113 140.165904 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 128.310776 141.591996 +L 124.072113 140.165904 +L 127.756184 137.630629 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="patch_30"> + <path d="M 288 120.638672 +Q 288 130.319117 288 138.713824 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 290 134.713824 +L 288 138.713824 +L 286 134.713824 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="patch_31"> + <path d="M 309.999902 120.295986 +Q 381.600555 130.320078 451.927887 140.165904 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 448.243816 137.630629 +L 451.927887 140.165904 +L 447.689224 141.591996 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="text_11"> + <!-- RMSNorm$_q$ --> + <g style="fill: #3a3a3a" transform="translate(91.894 153.812)"> + <text> + <tspan x="0" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">R</tspan> + <tspan x="5.992859" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">M</tspan> + <tspan x="13.434448" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">S</tspan> + <tspan x="18.909302" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">N</tspan> + <tspan x="25.361206" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="30.638123" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="34.184143" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="42.67207" y="1.164062" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">q</tspan> + </text> + </g> + </g> + <g id="text_12"> + <!-- RMSNorm$_k$ --> + <g style="fill: #3a3a3a" transform="translate(264.866 154.293062)"> + <text> + <tspan x="0" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">R</tspan> + <tspan x="5.992859" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">M</tspan> + <tspan x="13.434448" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">S</tspan> + <tspan x="18.909302" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">N</tspan> + <tspan x="25.361206" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="30.638123" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="34.184143" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="42.67207" y="1.164062" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">k</tspan> + </text> + </g> + </g> + <g id="text_13"> + <!-- split heads ($H{=}8$) --> + <g style="fill: #3a3a3a" transform="translate(75.812 188.125)"> + <text> + <tspan x="0" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="4.493591" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="9.968445" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="12.364746" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="14.761047" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="18.142822" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="20.88446" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="26.350891" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="31.657288" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="36.942627" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="42.41748" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="46.911072" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="49.65271" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="61.183594" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="70.090759" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">8</tspan> + <tspan x="75.578247" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="53.017639" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">H</tspan> + </text> + </g> + </g> + <g id="text_14"> + <!-- split heads ($H{=}8$) --> + <g style="fill: #3a3a3a" transform="translate(248.612 188.125)"> + <text> + <tspan x="0" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="4.493591" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="9.968445" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="12.364746" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="14.761047" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="18.142822" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="20.88446" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="26.350891" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="31.657288" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="36.942627" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="42.41748" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="46.911072" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="49.65271" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="61.183594" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="70.090759" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">8</tspan> + <tspan x="75.578247" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="53.017639" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">H</tspan> + </text> + </g> + </g> + <g id="text_15"> + <!-- split heads ($H{=}8$) --> + <g style="fill: #3a3a3a" transform="translate(421.412 188.125)"> + <text> + <tspan x="0" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="4.493591" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="9.968445" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="12.364746" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="14.761047" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="18.142822" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="20.88446" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="26.350891" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="31.657288" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="36.942627" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="42.41748" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="46.911072" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="49.65271" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="61.183594" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="70.090759" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">8</tspan> + <tspan x="75.578247" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="53.017639" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">H</tspan> + </text> + </g> + </g> + <g id="text_16"> + <text style="font-size: 8.6px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="115.2" y="221.973062" transform="rotate(-0 115.2 221.973062)">RoPE</text> + </g> + <g id="text_17"> + <text style="font-size: 8.6px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="288" y="221.973062" transform="rotate(-0 288 221.973062)">RoPE</text> + </g> + <g id="text_18"> + <!-- $\theta = 5{\cdot}10^5$ --> + <g style="fill: #8a8a8a" transform="translate(361.44 221.697125)"> + <text> + <tspan x="0" y="-0.857813" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">θ</tspan> + <tspan x="6.150635" y="-0.857813" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">=</tspan> + <tspan x="14.025085" y="-0.857813" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">5</tspan> + <tspan x="18.876343" y="-0.857813" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">⋅</tspan> + <tspan x="21.30011" y="-0.857813" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">1</tspan> + <tspan x="26.151367" y="-0.857813" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">0</tspan> + <tspan x="31.079187" y="-3.920313" style="font-size: 5.32px; font-family: 'DejaVu Sans'; fill: #8a8a8a">5</tspan> + </text> + </g> + </g> + <g id="patch_32"> + <path d="M 115.2 163.838145 +Q 115.2 168.83972 115.2 172.555557 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 117.2 168.555557 +L 115.2 172.555557 +L 113.2 168.555557 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="patch_33"> + <path d="M 115.2 197.678145 +Q 115.2 202.67972 115.2 206.395557 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 117.2 202.395557 +L 115.2 206.395557 +L 113.2 202.395557 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="patch_34"> + <path d="M 288 163.838145 +Q 288 168.83972 288 172.555557 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 290 168.555557 +L 288 172.555557 +L 286 168.555557 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="patch_35"> + <path d="M 288 197.678145 +Q 288 202.67972 288 206.395557 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 290 202.395557 +L 288 206.395557 +L 286 202.395557 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="patch_36"> + <path d="M 460.8 145.11999 +Q 460.8 159.479155 460.8 172.552581 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 462.8 168.552581 +L 460.8 172.552581 +L 458.8 168.552581 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="patch_37"> + <path d="M 115.590674 231.195344 +Q 165.597693 240.839555 214.342237 250.240288 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 210.793347 247.519005 +L 214.342237 250.240288 +L 210.035876 251.44663 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="patch_38"> + <path d="M 288 231.523418 +Q 288 240.839538 288 248.869919 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 290 244.869919 +L 288 248.869919 +L 286 244.869919 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="patch_39"> + <path d="M 460.44624 197.466987 +Q 410.399379 223.920328 361.489234 249.772834 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 365.960228 249.671794 +L 361.489234 249.772834 +L 364.090999 246.135414 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="text_19"> + <!-- matmul $S = q\,k^{\top} \cdot 1/\sqrt{h_d}$ --> + <g style="fill: #3a3a3a" transform="translate(234.336 266.756)"> + <text> + <tspan x="0" y="-0.969938" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="8.401794" y="-0.969938" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="13.687134" y="-0.969938" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="17.068909" y="-0.969938" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="25.470703" y="-0.969938" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="30.937134" y="-0.969938" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="33.333435" y="-0.969938" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="36.075073" y="-0.969938" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="45.971924" y="-0.969938" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="76.95822" y="-0.969938" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">⋅</tspan> + <tspan x="81.380217" y="-0.969938" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">1</tspan> + <tspan x="86.867705" y="-0.969938" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">/</tspan> + <tspan x="38.816711" y="-0.969938" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">S</tspan> + <tspan x="54.879089" y="-0.969938" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">q</tspan> + <tspan x="61.75427" y="-0.969938" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">k</tspan> + <tspan x="96.926332" y="-0.61875" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="68.596288" y="-4.41525" style="font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">⊤</tspan> + <tspan x="89.773589" y="-1.1715" style="font-size: 5.73953px; font-family: 'STIXSizeOneSym'; fill: #3a3a3a">√</tspan> + <tspan x="102.392763" y="0.857812" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + </text> + <rect x="95.851332" y="-10.925" width="11.694922" height="0.5375"/> + </g> + </g> + <g id="text_20"> + <!-- $(B,H,T,T)$ --> + <g style="fill: #8a8a8a" transform="translate(414 264.897125)"> + <text> + <tspan x="0" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">(</tspan> + <tspan x="8.205811" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">,</tspan> + <tspan x="17.848755" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">,</tspan> + <tspan x="26.41571" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">,</tspan> + <tspan x="34.982666" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">)</tspan> + <tspan x="2.974792" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">B</tspan> + <tspan x="12.115112" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">H</tspan> + <tspan x="21.758057" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">T</tspan> + <tspan x="30.325012" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">T</tspan> + </text> + </g> + </g> + <g id="patch_40"> + <path d="M 288 274.718145 +Q 288 279.71972 288 283.435557 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 290 279.435557 +L 288 283.435557 +L 286 279.435557 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="text_21"> + <text style="font-size: 8.6px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="288" y="299.013062" transform="rotate(-0 288 299.013062)">causal mask</text> + </g> + <g id="patch_41"> + <path d="M 288 308.558145 +Q 288 313.55972 288 317.275557 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 290 313.275557 +L 288 317.275557 +L 286 313.275557 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="text_22"> + <text style="font-size: 8.6px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="288" y="332.853062" transform="rotate(-0 288 332.853062)">softmax over keys</text> + </g> + <g id="patch_42"> + <path d="M 288 342.398145 +Q 288 347.39972 288 351.115557 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 290 347.115557 +L 288 351.115557 +L 286 347.115557 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="text_23"> + <!-- matmul $y = A\,v$ --> + <g style="fill: #3a3a3a" transform="translate(254.632 366.685)"> + <text> + <tspan x="0" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="8.401794" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="13.687134" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="17.068909" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="25.470703" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="30.937134" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="33.333435" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="36.075073" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="45.601318" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="38.816711" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">y</tspan> + <tspan x="54.508484" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">A</tspan> + <tspan x="61.809018" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">v</tspan> + </text> + </g> + </g> + <g id="text_24"> + <!-- $(B,H,T,h_d)$ --> + <g style="fill: #8a8a8a" transform="translate(414 366.417125)"> + <text> + <tspan x="0" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">(</tspan> + <tspan x="8.205811" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">,</tspan> + <tspan x="17.848755" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">,</tspan> + <tspan x="26.41571" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">,</tspan> + <tspan x="38.764465" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">)</tspan> + <tspan x="2.974792" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">B</tspan> + <tspan x="12.115112" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">H</tspan> + <tspan x="21.758057" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">T</tspan> + <tspan x="30.325012" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">h</tspan> + <tspan x="35.157654" y="0.390625" style="font-style: oblique; font-size: 5.32px; font-family: 'DejaVu Sans'; fill: #8a8a8a">d</tspan> + </text> + </g> + </g> + <g id="patch_43"> + <path d="M 288 376.238145 +Q 288 381.23972 288 384.955557 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 290 380.955557 +L 288 384.955557 +L 286 380.955557 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="text_25"> + <text style="font-size: 8.6px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="288" y="400.533062" transform="rotate(-0 288 400.533062)">merge heads</text> + </g> + <g id="text_26"> + <!-- $(B,T,C)$ --> + <g style="fill: #8a8a8a" transform="translate(414 400.257125)"> + <text> + <tspan x="0" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">(</tspan> + <tspan x="8.205811" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">,</tspan> + <tspan x="16.772766" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">,</tspan> + <tspan x="26.006165" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">)</tspan> + <tspan x="2.974792" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">B</tspan> + <tspan x="12.115112" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">T</tspan> + <tspan x="20.682068" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">C</tspan> + </text> + </g> + </g> + <g id="patch_44"> + <path d="M 288 410.078145 +Q 288 415.07972 288 418.795557 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 290 414.795557 +L 288 418.795557 +L 286 414.795557 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="text_27"> + <!-- matmul $W_{proj}: C \to C$ --> + <g style="fill: #3a3a3a" transform="translate(244.011 433.935)"> + <text> + <tspan x="0" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="8.401794" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="13.687134" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="17.068909" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="25.470703" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="30.937134" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="33.333435" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="36.075073" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="60.957166" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">:</tspan> + <tspan x="73.246106" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">→</tspan> + <tspan x="38.816711" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">W</tspan> + <tspan x="65.543408" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">C</tspan> + <tspan x="82.153271" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">C</tspan> + <tspan x="47.344849" y="1.320312" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="51.177246" y="1.320312" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="53.65946" y="1.320312" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="57.353302" y="1.320312" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">j</tspan> + </text> + </g> + </g> + <g id="patch_45"> + <path d="M 288 443.918145 +Q 288 448.91972 288 452.635557 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 290 448.635557 +L 288 452.635557 +L 286 448.635557 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="text_28"> + <!-- RMSNorm$_{attn}$ --> + <g style="fill: #3a3a3a" transform="translate(260.48 468.213062)"> + <text> + <tspan x="0" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">R</tspan> + <tspan x="5.992859" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">M</tspan> + <tspan x="13.434448" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">S</tspan> + <tspan x="18.909302" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">N</tspan> + <tspan x="25.361206" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="30.638123" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="34.184143" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="42.67207" y="1.164062" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="46.371808" y="1.164062" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="48.73905" y="1.164062" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="51.106293" y="1.164062" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + </text> + </g> + </g> + <g id="text_29"> + <text style="font-size: 7.6px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; fill: #3a3a3a" transform="translate(414 463.681956)">◄ OLMo2: norm on the sublayer OUTPUT,</text> + <text style="font-size: 7.6px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; fill: #3a3a3a" transform="translate(414 472.192294)"> not on its input</text> + </g> + <g id="patch_46"> + <path d="M 288 477.757441 +Q 288 484.199288 288 489.355395 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 290 485.355395 +L 288 489.355395 +L 286 485.355395 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="patch_47"> + <path d="M 288 519.12 +C 291.627978 519.12 295.107853 517.678589 297.673221 515.113221 +C 300.238589 512.547853 301.68 509.067978 301.68 505.44 +C 301.68 501.812022 300.238589 498.332147 297.673221 495.766779 +C 295.107853 493.201411 291.627978 491.76 288 491.76 +C 284.372022 491.76 280.892147 493.201411 278.326779 495.766779 +C 275.761411 498.332147 274.32 501.812022 274.32 505.44 +C 274.32 509.067978 275.761411 512.547853 278.326779 515.113221 +C 280.892147 517.678589 284.372022 519.12 288 519.12 +z +" clip-path="url(#pdba9adb895)" style="fill: #ffffff; stroke: #3a3a3a; stroke-width: 1.3; stroke-linejoin: miter"/> + </g> + <g id="text_30"> + <text style="font-weight: 700; font-size: 11px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="288" y="508.475312" transform="rotate(-0 288 508.475312)">+</text> + </g> + <g id="patch_48"> + <path d="M 288 519.517617 +Q 288 528.47975 288 536.156143 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 290 532.156143 +L 288 536.156143 +L 286 532.156143 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="text_31"> + <!-- $z'$ --> + <g style="fill: #3a3a3a" transform="translate(303.84 530.519375)"> + <text> + <tspan x="0" y="-0.25625" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="5.695638" y="-4.084375" style="font-size: 7px; font-family: 'cmsy10'; fill: #3a3a3a">0</tspan> + </text> + </g> + </g> + <g id="text_32"> + <text style="font-style: italic; font-size: 9.5px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; fill: #8a8a8a" transform="translate(25.661406 702.350469) rotate(-90)">SwiGLU sublayer</text> + </g> + <g id="patch_49"> + <path d="M 633.19823 750.24524 +Q 467.999756 752.400003 304.366397 754.534351 +" style="fill: none; stroke: #555555; stroke-width: 1.4; stroke-linecap: round"/> + <path d="M 308.392142 756.482012 +L 304.366397 754.534351 +L 308.339972 752.482352 +z +" style="fill: #555555; stroke: #555555; stroke-width: 1.4; stroke-linecap: round"/> + </g> + <g id="patch_50"> + <path d="M 273.208447 544.40543 +Q 233.999032 552.960211 196.045804 561.240915 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 200.380202 562.342279 +L 196.045804 561.240915 +L 199.527533 558.434216 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="patch_51"> + <path d="M 302.791553 544.40543 +Q 342.000968 552.960211 379.954196 561.240915 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 376.472467 558.434216 +L 379.954196 561.240915 +L 375.619798 562.342279 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="text_33"> + <!-- matmul $W_1: C \to h_{ff}$ --> + <g style="fill: #3a3a3a" transform="translate(145.189 575.493062)"> + <text> + <tspan x="0" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="8.401794" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="13.687134" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="17.068909" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="25.470703" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="30.937134" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="33.333435" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="36.075073" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="53.112543" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">:</tspan> + <tspan x="65.401483" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">→</tspan> + <tspan x="38.816711" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">W</tspan> + <tspan x="57.698785" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">C</tspan> + <tspan x="74.308649" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="47.344849" y="1.320312" style="font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">1</tspan> + <tspan x="79.775079" y="1.320312" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="81.900586" y="1.320312" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + </text> + </g> + </g> + <g id="text_34"> + <!-- matmul $W_3: C \to h_{ff}$ --> + <g style="fill: #3a3a3a" transform="translate(346.789 575.493062)"> + <text> + <tspan x="0" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="8.401794" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="13.687134" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="17.068909" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="25.470703" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="30.937134" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="33.333435" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="36.075073" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="53.112543" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">:</tspan> + <tspan x="65.401483" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">→</tspan> + <tspan x="38.816711" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">W</tspan> + <tspan x="57.698785" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">C</tspan> + <tspan x="74.308649" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="47.344849" y="1.320312" style="font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">3</tspan> + <tspan x="79.775079" y="1.320312" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="81.900586" y="1.320312" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + </text> + </g> + </g> + <g id="text_35"> + <!-- $h_{ff} = 1408$ --> + <g style="fill: #8a8a8a" transform="translate(473.04 575.217125)"> + <text> + <tspan x="0" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">h</tspan> + <tspan x="4.832642" y="0.390625" style="font-style: oblique; font-size: 5.32px; font-family: 'DejaVu Sans'; fill: #8a8a8a">f</tspan> + <tspan x="6.711713" y="0.390625" style="font-style: oblique; font-size: 5.32px; font-family: 'DejaVu Sans'; fill: #8a8a8a">f</tspan> + <tspan x="10.295068" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">=</tspan> + <tspan x="18.169519" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">1</tspan> + <tspan x="23.020776" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">4</tspan> + <tspan x="27.872034" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">0</tspan> + <tspan x="32.723291" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">8</tspan> + </text> + </g> + </g> + <g id="patch_52"> + <path d="M 187.2 585.038145 +Q 187.2 590.03972 187.2 593.755557 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 189.2 589.755557 +L 187.2 593.755557 +L 185.2 589.755557 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="text_36"> + <text style="font-size: 8.6px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="187.2" y="609.333062" transform="rotate(-0 187.2 609.333062)">SiLU</text> + </g> + <g id="patch_53"> + <path d="M 187.58562 618.582832 +Q 214.198105 625.679495 239.568263 632.44487 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 236.218648 629.48175 +L 239.568263 632.44487 +L 235.187998 633.346689 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="patch_54"> + <path d="M 388.500073 584.907935 +Q 361.799167 608.760744 336.057115 631.756977 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 340.372587 630.583646 +L 336.057115 631.756977 +L 337.707733 627.600601 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="text_37"> + <text style="font-size: 8.6px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="288" y="647.493062" transform="rotate(-0 288 647.493062)">⊙ elementwise gate</text> + </g> + <g id="patch_55"> + <path d="M 288 657.038145 +Q 288 662.03972 288 665.755557 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 290 661.755557 +L 288 665.755557 +L 286 661.755557 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="text_38"> + <!-- matmul $W_2: h_{ff} \to C$ --> + <g style="fill: #3a3a3a" transform="translate(245.989 685.653063)"> + <text> + <tspan x="0" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="8.401794" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="13.687134" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="17.068909" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="25.470703" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="30.937134" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="33.333435" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="36.075073" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="53.112543" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">:</tspan> + <tspan x="69.342682" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">→</tspan> + <tspan x="38.816711" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">W</tspan> + <tspan x="57.698785" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="78.249847" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">C</tspan> + <tspan x="47.344849" y="1.320312" style="font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">2</tspan> + <tspan x="63.165216" y="1.320312" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="65.290723" y="1.320312" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + </text> + </g> + </g> + <g id="patch_56"> + <path d="M 288 695.198145 +Q 288 700.19972 288 703.915557 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 290 699.915557 +L 288 703.915557 +L 286 699.915557 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="text_39"> + <!-- RMSNorm$_{ff}$ --> + <g style="fill: #3a3a3a" transform="translate(264.479 719.493062)"> + <text> + <tspan x="0" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">R</tspan> + <tspan x="5.992859" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">M</tspan> + <tspan x="13.434448" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">S</tspan> + <tspan x="18.909302" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">N</tspan> + <tspan x="25.361206" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="30.638123" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="34.184143" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="42.67207" y="1.164062" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="44.797577" y="1.164062" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + </text> + </g> + </g> + <g id="text_40"> + <text style="font-size: 7.6px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: start; fill: #3a3a3a" x="388.8" y="719.217125" transform="rotate(-0 388.8 719.217125)">◄ norm after the sublayer, again</text> + </g> + <g id="patch_57"> + <path d="M 288 729.042187 +Q 288 734.401592 288 738.475257 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 290 734.475257 +L 288 738.475257 +L 286 734.475257 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="patch_58"> + <path d="M 288 768.24 +C 291.627978 768.24 295.107853 766.798589 297.673221 764.233221 +C 300.238589 761.667853 301.68 758.187978 301.68 754.56 +C 301.68 750.932022 300.238589 747.452147 297.673221 744.886779 +C 295.107853 742.321411 291.627978 740.88 288 740.88 +C 284.372022 740.88 280.892147 742.321411 278.326779 744.886779 +C 275.761411 747.452147 274.32 750.932022 274.32 754.56 +C 274.32 758.187978 275.761411 761.667853 278.326779 764.233221 +C 280.892147 766.798589 284.372022 768.24 288 768.24 +z +" clip-path="url(#pdba9adb895)" style="fill: #ffffff; stroke: #3a3a3a; stroke-width: 1.3; stroke-linejoin: miter"/> + </g> + <g id="text_41"> + <text style="font-weight: 700; font-size: 11px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="288" y="757.595312" transform="rotate(-0 288 757.595312)">+</text> + </g> + <g id="patch_59"> + <path d="M 288 768.639902 +Q 288 777.240145 288 784.554649 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 290 780.554649 +L 288 784.554649 +L 286 780.554649 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="text_42"> + <!-- output $z^{\prime\prime}$ — one full $f_l$ done --> + <g style="fill: #3a3a3a" transform="translate(212.8075 799.7475)"> + <text> + <tspan x="0" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="6.526611" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="13.289795" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="17.831055" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="24.631348" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="31.394531" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="35.935791" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="48.60599" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="51.913363" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="64.720736" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="68.028109" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="71.335482" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="77.862093" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="84.625277" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="91.068392" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="94.375765" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="98.508822" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="105.272005" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="108.528353" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="111.784701" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="120.557576" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="123.86495" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="130.665243" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="137.191854" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="143.955037" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="39.243164" y="-0.40625" style="font-style: oblique; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="115.092074" y="-0.40625" style="font-style: oblique; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="44.676351" y="-4.234375" style="font-size: 6.65px; font-family: 'cmsy10'; fill: #3a3a3a">0</tspan> + <tspan x="46.504451" y="-4.234375" style="font-size: 6.65px; font-family: 'cmsy10'; fill: #3a3a3a">0</tspan> + <tspan x="55.220736" y="-0.40625" style="font-size: 9.5px; font-family: 'DejaVu Sans Display'; fill: #3a3a3a">—</tspan> + <tspan x="118.436556" y="1.234375" style="font-style: oblique; font-size: 6.65px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + </text> + </g> + </g> + <g id="text_43"> + <!-- weight × activation matmul — 7 logical ($W_{qkv}$ fuses $W_q,W_k,W_v$); crossbar-mappable on analog hardware --> + <g style="fill: #3a3a3a" transform="translate(68.4 840.645)"> + <text> + <tspan x="0" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">w</tspan> + <tspan x="6.747437" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="11.82312" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="14.115234" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">g</tspan> + <tspan x="19.352051" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="24.580811" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="27.815552" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="30.437988" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">×</tspan> + <tspan x="37.350586" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="39.973022" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="45.028564" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="49.564453" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="52.799194" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="55.091309" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">v</tspan> + <tspan x="59.973633" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="65.029175" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="68.263916" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="70.55603" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="75.603516" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="80.832275" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="83.454712" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="91.491211" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="96.546753" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="99.781494" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="107.817993" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="113.046753" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="115.338867" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="126.211304" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="128.83374" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">7</tspan> + <tspan x="134.082642" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="136.705078" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="138.997192" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="144.044678" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">g</tspan> + <tspan x="149.281494" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="151.573608" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="156.109497" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="161.165039" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="163.457153" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="166.07959" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="188.102026" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="190.724463" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="193.628882" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="198.857642" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="203.155859" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="208.231543" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="212.529761" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="227.194067" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">,</tspan> + <tspan x="243.144214" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">,</tspan> + <tspan x="259.167676" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="262.386304" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">;</tspan> + <tspan x="265.165845" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="267.788281" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="272.32417" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="275.716016" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="280.763501" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="285.061719" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="289.359937" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">b</tspan> + <tspan x="294.596753" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="299.652295" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="303.044141" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">-</tspan> + <tspan x="305.896069" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="313.932568" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="318.98811" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="324.224927" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="329.461743" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="334.517285" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">b</tspan> + <tspan x="339.754102" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="342.046216" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="347.121899" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="349.744336" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="354.791821" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="360.020581" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="362.643018" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="367.69856" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="372.927319" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="377.982861" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="380.274976" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="385.322461" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">g</tspan> + <tspan x="390.559277" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="393.181714" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="398.410474" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="403.466016" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="406.857861" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="412.094678" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">w</tspan> + <tspan x="418.842114" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="423.897656" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="427.289502" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="117.961304" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans Display'; fill: #3a3a3a">—</tspan> + <tspan x="169.298218" y="-0.921875" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">W</tspan> + <tspan x="215.152197" y="-0.921875" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">W</tspan> + <tspan x="231.423804" y="-0.921875" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">W</tspan> + <tspan x="247.37395" y="-0.921875" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">W</tspan> + <tspan x="177.455566" y="0.390625" style="font-style: oblique; font-size: 5.74px; font-family: 'DejaVu Sans'; fill: #3a3a3a">q</tspan> + <tspan x="181.121338" y="0.390625" style="font-style: oblique; font-size: 5.74px; font-family: 'DejaVu Sans'; fill: #3a3a3a">k</tspan> + <tspan x="184.465649" y="0.390625" style="font-style: oblique; font-size: 5.74px; font-family: 'DejaVu Sans'; fill: #3a3a3a">v</tspan> + <tspan x="223.309546" y="0.390625" style="font-style: oblique; font-size: 5.74px; font-family: 'DejaVu Sans'; fill: #3a3a3a">q</tspan> + <tspan x="239.581152" y="0.390625" style="font-style: oblique; font-size: 5.74px; font-family: 'DejaVu Sans'; fill: #3a3a3a">k</tspan> + <tspan x="255.531299" y="0.390625" style="font-style: oblique; font-size: 5.74px; font-family: 'DejaVu Sans'; fill: #3a3a3a">v</tspan> + </text> + </g> + </g> + <g id="text_44"> + <!-- activation × activation matmul — 2 ($q\,k^{\top}$ and $A\,v$); computed on the fly: the non-crossbar part of attention --> + <g style="fill: #3a3a3a" transform="translate(68.4 866.027)"> + <text> + <tspan x="0" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="5.055542" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="9.591431" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="12.826172" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="15.118286" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">v</tspan> + <tspan x="20.00061" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="25.056152" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="28.290894" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="30.583008" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="35.630493" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="40.859253" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="43.481689" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">×</tspan> + <tspan x="50.394287" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="53.016724" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="58.072266" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="62.608154" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="65.842896" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="68.13501" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">v</tspan> + <tspan x="73.017334" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="78.072876" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="81.307617" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="83.599731" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="88.647217" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="93.875977" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="96.498413" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="104.534912" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="109.590454" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="112.825195" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="120.861694" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="126.090454" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="128.382568" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="139.255005" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="141.877441" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">2</tspan> + <tspan x="147.126343" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="149.748779" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="172.41652" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="175.038957" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="180.094499" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="185.323258" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="190.560075" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="205.047956" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="208.266584" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">;</tspan> + <tspan x="211.046125" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="213.668561" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="218.20445" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="223.251935" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="231.288434" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="236.525251" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="241.75401" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="244.988751" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="250.064435" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="255.301251" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="257.923688" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="262.971173" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="268.199933" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="270.82237" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="274.057111" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="279.285871" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="284.361554" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="286.983991" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="289.88841" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="292.180524" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">y</tspan> + <tspan x="297.062848" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">:</tspan> + <tspan x="299.842389" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="302.464826" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="305.699567" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="310.928327" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="316.00401" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="318.626447" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="323.855207" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="328.902692" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="334.131452" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">-</tspan> + <tspan x="337.10838" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="341.644269" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="345.036115" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="350.0836" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="354.381818" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="358.680036" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">b</tspan> + <tspan x="363.916852" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="368.972394" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="372.36424" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="374.986676" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="380.223493" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="385.279035" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="388.67088" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="391.905622" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="394.528058" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="399.575543" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="402.479962" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="405.102399" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="410.157941" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="413.392682" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="416.627423" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="421.703107" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="426.931867" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="430.166608" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="432.458722" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="437.506208" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="131.005005" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans Display'; fill: #3a3a3a">—</tspan> + <tspan x="152.967407" y="-0.010938" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">q</tspan> + <tspan x="159.543667" y="-0.010938" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">k</tspan> + <tspan x="193.182511" y="-0.010938" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">A</tspan> + <tspan x="200.165631" y="-0.010938" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">v</tspan> + <tspan x="166.042094" y="-3.073438" style="font-size: 5.74px; font-family: 'DejaVu Sans'; fill: #3a3a3a">⊤</tspan> + </text> + </g> + </g> + <g id="text_45"> + <text style="font-size: 8.2px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: start; fill: #3a3a3a" x="68.4" y="890.022687" transform="rotate(-0 68.4 890.022687)">RMSNorm (learned gain)</text> + </g> + <g id="text_46"> + <text style="font-size: 8.2px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: start; fill: #3a3a3a" x="306" y="890.022687" transform="rotate(-0 306 890.022687)">elementwise / reshape</text> + </g> + <g id="text_47"> + <!-- No biases anywhere in the network. The $1/\sqrt{h_d}$ scale is folded into $S$; mask and softmax run along the key axis. --> + <g style="fill: #3a3a3a" transform="translate(36 921.052)"> + <text> + <tspan x="0" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">N</tspan> + <tspan x="6.171387" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="11.218872" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="13.841309" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">b</tspan> + <tspan x="19.078125" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="21.370239" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="26.425781" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="30.723999" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="35.799683" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="40.0979" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="42.720337" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="47.775879" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="53.004639" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">y</tspan> + <tspan x="57.886963" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">w</tspan> + <tspan x="64.634399" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="69.863159" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="74.938843" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="78.330688" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="83.406372" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="86.028809" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="88.320923" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="93.549683" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="96.172119" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="99.40686" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="104.63562" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="109.711304" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="112.33374" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="117.5625" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="122.638184" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="125.872925" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">w</tspan> + <tspan x="132.620361" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="137.667847" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="141.059692" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">k</tspan> + <tspan x="145.83728" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">.</tspan> + <tspan x="148.459717" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="151.082153" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="153.70459" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">T</tspan> + <tspan x="158.744019" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="163.972778" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="169.048462" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="171.670898" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">1</tspan> + <tspan x="176.9198" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">/</tspan> + <tspan x="196.808255" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="199.430692" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="203.728909" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="208.264798" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="213.32034" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="215.612454" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="220.688138" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="223.310574" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="225.602689" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="229.900906" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="232.523343" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="235.427762" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="240.475247" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="242.767361" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="248.004178" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="253.079861" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="258.316678" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="260.939114" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="263.231229" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="268.459988" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="271.69473" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="276.742215" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="284.601468" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">;</tspan> + <tspan x="287.381009" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="290.003445" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="298.039944" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="303.095486" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="307.393704" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">k</tspan> + <tspan x="312.171292" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="314.793729" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="319.849271" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="325.07803" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="330.314847" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="332.937283" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="337.235501" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="342.282986" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="345.187405" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="348.422147" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="356.458646" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="361.514188" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">x</tspan> + <tspan x="366.396512" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="369.018948" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="372.410794" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="377.639554" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="382.868314" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="385.49075" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="390.546292" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="392.838406" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="397.885892" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="403.114651" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">g</tspan> + <tspan x="408.351468" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="410.973904" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="414.208646" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="419.437405" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="424.513089" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="427.135526" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">k</tspan> + <tspan x="431.913113" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="436.988797" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">y</tspan> + <tspan x="441.871121" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="444.493558" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="449.5491" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">x</tspan> + <tspan x="454.431424" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="456.723538" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="461.021756" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">.</tspan> + <tspan x="179.699341" y="-1.1955" style="font-size: 5.564038px; font-family: 'STIXSizeOneSym'; fill: #3a3a3a">√</tspan> + <tspan x="186.669974" y="-0.409375" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="279.364651" y="-0.818938" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">S</tspan> + <tspan x="191.898734" y="0.903125" style="font-style: oblique; font-size: 5.74px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + </text> + <rect x="185.644974" y="-10.975" width="11.163281" height="0.5125"/> + </g> + </g> + <g id="text_48"> + <!-- $h_{ff} = 1408 \approx 8C/3$, rounded up to a multiple of 64. QK-norm acts on the full width $C$, before the head split. --> + <g style="fill: #3a3a3a" transform="translate(36 938.975)"> + <text> + <tspan x="0" y="-0.921875" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="56.012598" y="-0.921875" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">C</tspan> + <tspan x="337.75332" y="-0.921875" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">C</tspan> + <tspan x="5.22876" y="0.390625" style="font-style: oblique; font-size: 5.74px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="7.261853" y="0.390625" style="font-style: oblique; font-size: 5.74px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="11.120996" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="19.640894" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">1</tspan> + <tspan x="24.889795" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">4</tspan> + <tspan x="30.138696" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">0</tspan> + <tspan x="35.387598" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">8</tspan> + <tspan x="42.243799" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">≈</tspan> + <tspan x="50.763696" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">8</tspan> + <tspan x="61.773096" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">/</tspan> + <tspan x="64.552637" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">3</tspan> + <tspan x="69.801538" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">,</tspan> + <tspan x="72.423975" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="75.046411" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="78.438257" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="83.485742" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="88.714502" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="93.943262" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="99.180078" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="104.255762" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="109.492578" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="112.115015" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="117.343774" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="122.580591" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="125.203027" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="128.437769" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="133.485254" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="136.10769" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="141.163232" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="143.785669" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="151.822168" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="157.050928" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="159.343042" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="162.577783" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="164.869897" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="170.106714" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="172.398828" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="177.474512" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="180.096948" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="185.144434" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="188.048853" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="190.671289" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">6</tspan> + <tspan x="195.92019" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">4</tspan> + <tspan x="201.169092" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">.</tspan> + <tspan x="203.791528" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="206.413965" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="209.036401" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">Q</tspan> + <tspan x="215.530054" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">K</tspan> + <tspan x="220.940088" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">-</tspan> + <tspan x="223.917017" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="229.145776" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="234.193262" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="237.585107" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="245.621606" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="248.244043" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="253.299585" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="257.835474" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="261.070215" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="265.368433" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="267.990869" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="273.038354" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="278.267114" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="280.889551" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="284.124292" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="289.353052" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="294.428735" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="297.051172" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="299.955591" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="305.184351" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="307.476465" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="309.768579" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="312.391016" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">w</tspan> + <tspan x="319.138452" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="321.430566" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="326.667383" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="329.902124" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="335.130884" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="343.513818" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">,</tspan> + <tspan x="346.136255" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="348.758691" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">b</tspan> + <tspan x="353.995508" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="359.071191" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="361.97561" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="367.023096" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="370.414941" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="375.490625" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="378.113062" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="381.347803" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="386.576563" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="391.652246" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="394.274683" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="399.503442" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="404.579126" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="409.634668" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="414.871484" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="417.493921" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="421.792139" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="427.028955" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="429.321069" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="431.613184" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="434.847925" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">.</tspan> + </text> + </g> + </g> + <g id="text_49"> + <!-- The boxed pipeline, input $z$ to output $z^{\prime\prime}$, is exactly $f_l$ on the companion page; the state $z_l$ lives on the residual rail. --> + <g style="fill: #3a3a3a" transform="translate(36 958.497)"> + <text> + <tspan x="0" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">T</tspan> + <tspan x="5.039429" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="10.268188" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="15.343872" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="17.966309" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">b</tspan> + <tspan x="23.203125" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="28.25061" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">x</tspan> + <tspan x="33.132935" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="38.208618" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="43.445435" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="46.067871" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="51.304688" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="53.596802" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="58.833618" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="63.909302" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="66.201416" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="68.49353" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="73.72229" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="78.797974" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">,</tspan> + <tspan x="81.42041" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="84.042847" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="86.334961" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="91.563721" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="96.800537" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="102.029297" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="105.264038" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="112.216919" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="114.839355" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="118.074097" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="123.121582" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="125.744019" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="130.791504" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="136.020264" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="139.255005" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="144.491821" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="149.720581" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="152.955322" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="163.635313" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">,</tspan> + <tspan x="166.257749" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="168.880186" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="171.1723" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="175.470518" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="178.092955" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="183.168638" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">x</tspan> + <tspan x="188.050962" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="193.106504" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="197.642393" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="200.877134" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="203.169248" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">y</tspan> + <tspan x="208.051573" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="215.401658" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="218.024095" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="223.07158" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="228.30034" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="230.922776" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="234.157517" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="239.386277" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="244.461961" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="247.084397" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="251.620286" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="256.667771" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="264.70427" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="269.941087" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="274.996629" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="280.225389" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="282.517503" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="287.564988" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="292.793748" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="295.416184" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="300.653001" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="305.708543" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">g</tspan> + <tspan x="310.945359" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="316.021043" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">;</tspan> + <tspan x="318.800584" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="321.42302" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="324.657762" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="329.886521" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="334.962205" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="337.584642" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="341.882859" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="345.117601" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="350.173142" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="353.407884" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="358.483567" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="367.259678" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="369.882115" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="372.174229" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="374.466343" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">v</tspan> + <tspan x="379.348667" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="384.424351" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="388.722569" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="391.345005" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="396.392491" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="401.62125" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="404.243687" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="407.478428" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="412.707188" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="417.782872" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="420.405308" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="423.797154" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="428.872837" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="433.171055" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="435.463169" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="440.699986" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="445.928746" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="450.984288" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="453.276402" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="455.898838" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="459.290684" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="464.346226" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="466.63834" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="468.930455" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">.</tspan> + <tspan x="107.886475" y="-0.732812" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="155.577759" y="-0.732812" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="210.674009" y="-0.732812" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="361.106004" y="-0.732812" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="160.265495" y="-3.795312" style="font-size: 5.74px; font-family: 'cmsy10'; fill: #3a3a3a">0</tspan> + <tspan x="161.841029" y="-3.795312" style="font-size: 5.74px; font-family: 'cmsy10'; fill: #3a3a3a">0</tspan> + <tspan x="213.578428" y="0.579688" style="font-style: oblique; font-size: 5.74px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="365.436448" y="0.579688" style="font-style: oblique; font-size: 5.74px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + </text> + </g> + </g> + </g> + </g> + <defs> + <clipPath id="pdba9adb895"> + <rect x="0" y="0" width="720" height="1022.4"/> + </clipPath> + </defs> +</svg> diff --git a/assets/figs/fig_primer_phases.emf b/assets/figs/fig_primer_phases.emf Binary files differnew file mode 100644 index 0000000..c28c459 --- /dev/null +++ b/assets/figs/fig_primer_phases.emf diff --git a/assets/figs/fig_primer_phases.png b/assets/figs/fig_primer_phases.png Binary files differnew file mode 100644 index 0000000..a6db577 --- /dev/null +++ b/assets/figs/fig_primer_phases.png diff --git a/assets/figs/fig_primer_phases.svg b/assets/figs/fig_primer_phases.svg new file mode 100644 index 0000000..da9ed6a --- /dev/null +++ b/assets/figs/fig_primer_phases.svg @@ -0,0 +1,925 @@ +<?xml version="1.0" encoding="utf-8" standalone="no"?> +<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" + "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"> +<svg xmlns:xlink="http://www.w3.org/1999/xlink" width="878.4pt" height="792pt" viewBox="0 0 878.4 792" xmlns="http://www.w3.org/2000/svg" version="1.1"> + <metadata> + <rdf:RDF xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:cc="http://creativecommons.org/ns#" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"> + <cc:Work> + <dc:type rdf:resource="http://purl.org/dc/dcmitype/StillImage"/> + <dc:date>2026-07-15T09:44:53.517354</dc:date> + <dc:format>image/svg+xml</dc:format> + <dc:creator> + <cc:Agent> + <dc:title>Matplotlib v3.10.8, https://matplotlib.org/</dc:title> + </cc:Agent> + </dc:creator> + </cc:Work> + </rdf:RDF> + </metadata> + <defs> + <style type="text/css">*{stroke-linejoin: round; stroke-linecap: butt}</style> + </defs> + <g id="figure_1"> + <g id="patch_1"> + <path d="M 0 792 +L 878.4 792 +L 878.4 0 +L 0 0 +z +" style="fill: #ffffff"/> + </g> + <g id="axes_1"> + <g id="patch_2"> + <path d="M 77.7384 199.779775 +L 256.0536 199.779775 +Q 261.324 199.779775 261.324 194.440449 +L 261.324 161.514607 +Q 261.324 156.175281 256.0536 156.175281 +L 77.7384 156.175281 +Q 72.468 156.175281 72.468 161.514607 +L 72.468 194.440449 +Q 72.468 199.779775 77.7384 199.779775 +z +" clip-path="url(#p90a79555a2)" style="fill: #eaf1fa; stroke: #2c6fbb; stroke-width: 1.2; stroke-linejoin: miter"/> + </g> + <g id="patch_3"> + <path d="M 77.7384 306.566292 +L 256.0536 306.566292 +Q 261.324 306.566292 261.324 301.226966 +L 261.324 268.301124 +Q 261.324 262.961798 256.0536 262.961798 +L 77.7384 262.961798 +Q 72.468 262.961798 72.468 268.301124 +L 72.468 301.226966 +Q 72.468 306.566292 77.7384 306.566292 +z +" clip-path="url(#p90a79555a2)" style="fill: #eaf1fa; stroke: #2c6fbb; stroke-width: 1.2; stroke-linejoin: miter"/> + </g> + <g id="patch_4"> + <path d="M 77.7384 413.352809 +L 256.0536 413.352809 +Q 261.324 413.352809 261.324 408.013483 +L 261.324 375.08764 +Q 261.324 369.748315 256.0536 369.748315 +L 77.7384 369.748315 +Q 72.468 369.748315 72.468 375.08764 +L 72.468 408.013483 +Q 72.468 413.352809 77.7384 413.352809 +z +" clip-path="url(#p90a79555a2)" style="fill: #eaf1fa; stroke: #2c6fbb; stroke-width: 1.2; stroke-linejoin: miter"/> + </g> + <g id="patch_5"> + <path d="M 473.0184 199.779775 +L 651.3336 199.779775 +Q 656.604 199.779775 656.604 194.440449 +L 656.604 161.514607 +Q 656.604 156.175281 651.3336 156.175281 +L 473.0184 156.175281 +Q 467.748 156.175281 467.748 161.514607 +L 467.748 194.440449 +Q 467.748 199.779775 473.0184 199.779775 +z +" clip-path="url(#p90a79555a2)" style="fill: #fdeee2; stroke: #d95f02; stroke-width: 1.2; stroke-linejoin: miter"/> + </g> + <g id="patch_6"> + <path d="M 473.0184 306.566292 +L 651.3336 306.566292 +Q 656.604 306.566292 656.604 301.226966 +L 656.604 268.301124 +Q 656.604 262.961798 651.3336 262.961798 +L 473.0184 262.961798 +Q 467.748 262.961798 467.748 268.301124 +L 467.748 301.226966 +Q 467.748 306.566292 473.0184 306.566292 +z +" clip-path="url(#p90a79555a2)" style="fill: #fdeee2; stroke: #d95f02; stroke-width: 1.2; stroke-linejoin: miter"/> + </g> + <g id="patch_7"> + <path d="M 473.0184 413.352809 +L 651.3336 413.352809 +Q 656.604 413.352809 656.604 408.013483 +L 656.604 375.08764 +Q 656.604 369.748315 651.3336 369.748315 +L 473.0184 369.748315 +Q 467.748 369.748315 467.748 375.08764 +L 467.748 408.013483 +Q 467.748 413.352809 473.0184 413.352809 +z +" clip-path="url(#p90a79555a2)" style="fill: #fdeee2; stroke: #d95f02; stroke-width: 1.2; stroke-linejoin: miter"/> + </g> + <g id="patch_8"> + <path d="M 20.6424 771.977528 +L 857.7576 771.977528 +Q 863.028 771.977528 863.028 766.638202 +L 863.028 639.38427 +Q 863.028 634.044944 857.7576 634.044944 +L 20.6424 634.044944 +Q 15.372 634.044944 15.372 639.38427 +L 15.372 766.638202 +Q 15.372 771.977528 20.6424 771.977528 +z +" clip-path="url(#p90a79555a2)" style="fill: #faf6ee; stroke: #c9b895; stroke-width: 1.2; stroke-linejoin: miter"/> + </g> + <g id="text_1"> + <text style="font-weight: 700; font-size: 11.5px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="219.6" y="27.200248" transform="rotate(-0 219.6 27.200248)">Free phase (= inference)</text> + </g> + <g id="text_2"> + <!-- give each layer a state $z_l$ (3 of $L$ drawn); settle the disagreement energy --> + <g style="fill: #8a8a8a" transform="translate(59.66 52.253708)"> + <text> + <tspan x="0" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">g</tspan> + <tspan x="5.554199" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="7.985229" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">v</tspan> + <tspan x="13.163452" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="18.546753" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="21.328125" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="26.711426" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="32.073364" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">c</tspan> + <tspan x="36.884155" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">h</tspan> + <tspan x="42.42981" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="45.211182" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">l</tspan> + <tspan x="47.642212" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="53.00415" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">y</tspan> + <tspan x="58.182373" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="63.565674" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="67.163086" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="69.944458" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="75.306396" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="78.087769" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="82.646484" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="86.077271" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="91.439209" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="94.869995" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="100.253296" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="109.575378" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="112.35675" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">(</tspan> + <tspan x="115.770447" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">3</tspan> + <tspan x="121.337463" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="124.118835" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">o</tspan> + <tspan x="129.472229" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">f</tspan> + <tspan x="132.552673" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="140.208923" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="142.990295" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">d</tspan> + <tspan x="148.544495" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="152.141907" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="157.503845" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">w</tspan> + <tspan x="164.660217" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">n</tspan> + <tspan x="170.205872" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">)</tspan> + <tspan x="173.619568" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">;</tspan> + <tspan x="176.567566" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="179.348938" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="183.907654" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="189.290955" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="192.721741" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="196.152527" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">l</tspan> + <tspan x="198.583557" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="203.966858" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="206.74823" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="210.179016" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">h</tspan> + <tspan x="215.72467" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="221.107971" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="223.889343" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">d</tspan> + <tspan x="229.443542" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="231.874573" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="236.433289" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="241.795227" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">g</tspan> + <tspan x="247.349426" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="250.946838" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="256.330139" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="261.71344" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">m</tspan> + <tspan x="270.237" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="275.6203" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">n</tspan> + <tspan x="281.165955" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="284.596741" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="287.378113" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="292.761414" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">n</tspan> + <tspan x="298.307068" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="303.690369" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="307.287781" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">g</tspan> + <tspan x="312.84198" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">y</tspan> + <tspan x="103.034668" y="-0.15625" style="font-style: oblique; font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">z</tspan> + <tspan x="135.334045" y="-0.15625" style="font-style: oblique; font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">L</tspan> + <tspan x="107.627563" y="1.320312" style="font-style: oblique; font-size: 6.16px; font-family: 'DejaVu Sans'; fill: #8a8a8a">l</tspan> + </text> + </g> + </g> + <g id="text_3"> + <!-- $E(z)\ =\ \sum_l\ \frac{1}{2}\,\|\,z_l - f_l(z_{l-1})\,\|^2$ --> + <g style="fill: #3a3a3a" transform="translate(145.0225 85.544213)"> + <text> + <tspan x="0" y="-0.796875" style="font-style: oblique; font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">E</tspan> + <tspan x="11.752686" y="-0.796875" style="font-style: oblique; font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="74.075475" y="-0.796875" style="font-style: oblique; font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="96.793224" y="-0.796875" style="font-style: oblique; font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="107.893053" y="-0.796875" style="font-style: oblique; font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="7.266113" y="-0.796875" style="font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="17.789062" y="-0.796875" style="font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="28.250207" y="-0.796875" style="font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="66.458373" y="-0.796875" style="font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">‖</tspan> + <tspan x="84.917004" y="-0.796875" style="font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">−</tspan> + <tspan x="103.406481" y="-0.796875" style="font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="131.497448" y="-0.796875" style="font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="137.851123" y="-0.796875" style="font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">‖</tspan> + <tspan x="43.860521" y="-0.796875" style="font-size: 11.5px; font-family: 'DejaVu Sans Display'; fill: #3a3a3a">∑</tspan> + <tspan x="47.860521" y="11.0125" style="font-style: oblique; font-size: 8.05px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="80.111852" y="1.171875" style="font-style: oblique; font-size: 8.05px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="100.841808" y="1.171875" style="font-style: oblique; font-size: 8.05px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="113.92943" y="1.171875" style="font-style: oblique; font-size: 8.05px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="58.032115" y="-5.992188" style="font-size: 8.05px; font-family: 'DejaVu Sans'; fill: #3a3a3a">1</tspan> + <tspan x="58.032115" y="3.835938" style="font-size: 8.05px; font-family: 'DejaVu Sans'; fill: #3a3a3a">2</tspan> + <tspan x="117.734313" y="1.171875" style="font-size: 8.05px; font-family: 'DejaVu Sans'; fill: #3a3a3a">−</tspan> + <tspan x="126.047668" y="1.171875" style="font-size: 8.05px; font-family: 'DejaVu Sans'; fill: #3a3a3a">1</tspan> + <tspan x="143.715967" y="-5.390625" style="font-size: 8.05px; font-family: 'DejaVu Sans'; fill: #3a3a3a">2</tspan> + </text> + <rect x="58.032115" y="-4.117188" width="5.121655" height="0.71875"/> + </g> + </g> + <g id="text_4"> + <!-- $z_3 = f_3(z_2)$ --> + <g style="fill: #3a3a3a" transform="translate(142.746 180.736903)"> + <text> + <tspan x="0" y="-0.40625" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="22.251465" y="-0.40625" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="34.400391" y="-0.40625" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="5.249023" y="1.234375" style="font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">3</tspan> + <tspan x="25.771973" y="1.234375" style="font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">3</tspan> + <tspan x="39.649414" y="1.234375" style="font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">2</tspan> + <tspan x="11.924316" y="-0.40625" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="30.499023" y="-0.40625" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="44.376465" y="-0.40625" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + </text> + </g> + </g> + <g id="text_5"> + <text style="font-size: 8.4px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: start; fill: #8a8a8a" x="274.0608" y="180.295403" transform="rotate(-0 274.0608 180.295403)">term = 0</text> + </g> + <g id="text_6"> + <!-- $z_2 = f_2(z_1)$ --> + <g style="fill: #3a3a3a" transform="translate(142.746 287.52342)"> + <text> + <tspan x="0" y="-0.40625" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="22.251465" y="-0.40625" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="34.400391" y="-0.40625" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="5.249023" y="1.234375" style="font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">2</tspan> + <tspan x="25.771973" y="1.234375" style="font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">2</tspan> + <tspan x="39.649414" y="1.234375" style="font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">1</tspan> + <tspan x="11.924316" y="-0.40625" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="30.499023" y="-0.40625" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="44.376465" y="-0.40625" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + </text> + </g> + </g> + <g id="text_7"> + <text style="font-size: 8.4px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: start; fill: #8a8a8a" x="274.0608" y="287.08192" transform="rotate(-0 274.0608 287.08192)">term = 0</text> + </g> + <g id="text_8"> + <!-- $z_1 = f_1(\mathrm{emb})$ --> + <g style="fill: #3a3a3a" transform="translate(136.596 394.309937)"> + <text> + <tspan x="0" y="-0.40625" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="22.251465" y="-0.40625" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="5.249023" y="1.234375" style="font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">1</tspan> + <tspan x="25.771973" y="1.234375" style="font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">1</tspan> + <tspan x="11.924316" y="-0.40625" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="30.499023" y="-0.40625" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="34.400391" y="-0.40625" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="40.552734" y="-0.40625" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="50.293945" y="-0.40625" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">b</tspan> + <tspan x="56.641602" y="-0.40625" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + </text> + </g> + </g> + <g id="text_9"> + <text style="font-size: 8.4px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: start; fill: #8a8a8a" x="274.0608" y="393.868437" transform="rotate(-0 274.0608 393.868437)">term = 0</text> + </g> + <g id="patch_9"> + <path d="M 166.896 260.239287 +Q 166.896 231.817088 166.896 204.680629 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 164.696 209.080629 +L 166.896 204.680629 +L 169.096 209.080629 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="patch_10"> + <path d="M 166.896 367.025804 +Q 166.896 338.603605 166.896 311.467146 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 164.696 315.867146 +L 166.896 311.467146 +L 169.096 315.867146 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="patch_11"> + <path d="M 166.896 467.579034 +Q 166.896 443.164529 166.896 420.035763 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 164.696 424.435763 +L 166.896 420.035763 +L 169.096 424.435763 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="text_10"> + <!-- $\mathrm{emb}(x)$ --> + <g style="fill: #3a3a3a" transform="translate(148.896 490.417802)"> + <text> + <tspan x="0" y="-0.40625" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="6.152344" y="-0.40625" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="15.893555" y="-0.40625" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">b</tspan> + <tspan x="22.241211" y="-0.40625" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="32.060547" y="-0.40625" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="26.142578" y="-0.40625" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">x</tspan> + </text> + </g> + </g> + <g id="text_11"> + <!-- The minimum is exact: $E=0$, states $\equiv$ forward activations, --> + <g style="fill: #3a3a3a" transform="translate(82.71 547.094673)"> + <text> + <tspan x="0" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">T</tspan> + <tspan x="5.497559" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="11.20166" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="16.73877" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="19.599609" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="28.366699" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="30.867188" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="36.571289" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="39.071777" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="47.838867" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="53.542969" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="62.310059" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="65.170898" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="67.671387" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="72.360352" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="75.221191" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="80.758301" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">x</tspan> + <tspan x="86.084473" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="91.599609" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="96.547852" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="100.07666" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">:</tspan> + <tspan x="103.108887" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="105.969727" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="116.270508" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="125.564941" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">0</tspan> + <tspan x="131.291016" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">,</tspan> + <tspan x="134.151855" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="137.012695" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="139.873535" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="144.5625" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="148.091309" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="153.606445" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="157.135254" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="162.672363" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="167.361328" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="171.975586" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">≡</tspan> + <tspan x="181.27002" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="184.130859" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="187.299316" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="192.805664" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="196.505859" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">w</tspan> + <tspan x="203.866699" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="209.381836" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="213.082031" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="218.794922" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="221.655762" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="227.170898" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="232.119141" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="235.647949" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="238.148438" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">v</tspan> + <tspan x="243.474609" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="248.989746" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="252.518555" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="255.019043" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="260.525391" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="266.229492" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="270.918457" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">,</tspan> + <tspan x="108.830566" y="-0.15625" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">E</tspan> + </text> + </g> + </g> + <g id="text_12"> + <text style="font-size: 9px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="219.6" y="573.791303" transform="rotate(-0 219.6 573.791303)">and one bottom-up pass reaches it.</text> + </g> + <g id="text_13"> + <text style="font-size: 8.6px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #8a8a8a" x="219.6" y="605.716883" transform="rotate(-0 219.6 605.716883)">The free phase adds nothing and changes nothing at inference.</text> + </g> + <g id="text_14"> + <text style="font-weight: 700; font-size: 11.5px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="614.88" y="27.155326" transform="rotate(-0 614.88 27.155326)">Nudged phase (training only)</text> + </g> + <g id="text_15"> + <!-- add the loss at strength $\beta \ll 1$ and settle again: --> + <g style="fill: #8a8a8a" transform="translate(510.336 52.297708)"> + <text> + <tspan x="0" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="5.361938" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">d</tspan> + <tspan x="10.916138" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">d</tspan> + <tspan x="16.470337" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="19.251709" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="22.682495" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">h</tspan> + <tspan x="28.228149" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="33.61145" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="36.392822" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">l</tspan> + <tspan x="38.823853" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">o</tspan> + <tspan x="44.177246" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="48.735962" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="53.294678" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="56.07605" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="61.437988" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="64.868774" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="67.650146" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="72.208862" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="75.639648" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="79.237061" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="84.620361" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">n</tspan> + <tspan x="90.166016" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">g</tspan> + <tspan x="95.720215" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="99.151001" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">h</tspan> + <tspan x="104.696655" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="114.766846" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">≪</tspan> + <tspan x="125.631714" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">1</tspan> + <tspan x="131.19873" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="133.980103" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="139.342041" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">n</tspan> + <tspan x="144.887695" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">d</tspan> + <tspan x="150.441895" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="153.223267" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="157.781982" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="163.165283" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="166.596069" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="170.026855" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">l</tspan> + <tspan x="172.457886" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="177.841187" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="180.622559" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="185.984497" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">g</tspan> + <tspan x="191.538696" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="196.900635" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="199.331665" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">n</tspan> + <tspan x="204.877319" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">:</tspan> + <tspan x="107.478027" y="-0.109375" style="font-style: oblique; font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">β</tspan> + </text> + </g> + </g> + <g id="text_16"> + <!-- $E(z)\ +\ \beta\cdot\mathrm{CE}(\mathrm{logits}(z_3),\,y)$ --> + <g style="fill: #3a3a3a" transform="translate(539.9 88.649213)"> + <text> + <tspan x="0" y="-0.8125" style="font-style: oblique; font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">E</tspan> + <tspan x="11.752686" y="-0.8125" style="font-style: oblique; font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="43.860521" y="-0.8125" style="font-style: oblique; font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">β</tspan> + <tspan x="114.831468" y="-0.8125" style="font-style: oblique; font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="138.567297" y="-0.8125" style="font-style: oblique; font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">y</tspan> + <tspan x="7.266113" y="-0.8125" style="font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="17.789062" y="-0.8125" style="font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="28.250207" y="-0.8125" style="font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">+</tspan> + <tspan x="53.440111" y="-0.8125" style="font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">⋅</tspan> + <tspan x="59.336107" y="-0.8125" style="font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">C</tspan> + <tspan x="67.365892" y="-0.8125" style="font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">E</tspan> + <tspan x="74.632005" y="-0.8125" style="font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="79.118578" y="-0.8125" style="font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="82.313646" y="-0.8125" style="font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="89.349535" y="-0.8125" style="font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">g</tspan> + <tspan x="96.649339" y="-0.8125" style="font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="99.844408" y="-0.8125" style="font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="104.353441" y="-0.8125" style="font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="110.344896" y="-0.8125" style="font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="126.317626" y="-0.8125" style="font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="130.804198" y="-0.8125" style="font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">,</tspan> + <tspan x="145.372961" y="-0.8125" style="font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="120.867845" y="1.15625" style="font-size: 8.05px; font-family: 'DejaVu Sans'; fill: #3a3a3a">3</tspan> + </text> + </g> + </g> + <g id="text_17"> + <!-- pull $-\beta\,\nabla\mathrm{CE}$ --> + <g style="fill: #b03a2e" transform="translate(641.232 120.097169)"> + <text> + <tspan x="0" y="-0.109375" style="font-size: 9.4px; font-family: 'DejaVu Sans'; fill: #b03a2e">p</tspan> + <tspan x="5.950928" y="-0.109375" style="font-size: 9.4px; font-family: 'DejaVu Sans'; fill: #b03a2e">u</tspan> + <tspan x="11.8927" y="-0.109375" style="font-size: 9.4px; font-family: 'DejaVu Sans'; fill: #b03a2e">l</tspan> + <tspan x="14.497375" y="-0.109375" style="font-size: 9.4px; font-family: 'DejaVu Sans'; fill: #b03a2e">l</tspan> + <tspan x="17.102051" y="-0.109375" style="font-size: 9.4px; font-family: 'DejaVu Sans'; fill: #b03a2e"> </tspan> + <tspan x="20.082092" y="-0.109375" style="font-size: 9.4px; font-family: 'DejaVu Sans'; fill: #b03a2e">−</tspan> + <tspan x="35.442383" y="-0.109375" style="font-size: 9.4px; font-family: 'DejaVu Sans'; fill: #b03a2e">∇</tspan> + <tspan x="41.713745" y="-0.109375" style="font-size: 9.4px; font-family: 'DejaVu Sans'; fill: #b03a2e">C</tspan> + <tspan x="48.259766" y="-0.109375" style="font-size: 9.4px; font-family: 'DejaVu Sans'; fill: #b03a2e">E</tspan> + <tspan x="27.937317" y="-0.109375" style="font-style: oblique; font-size: 9.4px; font-family: 'DejaVu Sans'; fill: #b03a2e">β</tspan> + </text> + </g> + </g> + <g id="text_18"> + <text style="font-size: 7.8px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: start; fill: #b03a2e" x="641.232" y="140.974784" transform="rotate(-0 641.232 140.974784)">(the only place the label enters)</text> + </g> + <g id="patch_12"> + <path d="M 632.014537 125.721668 +Q 608.292978 139.266857 586.124861 151.925019 +" style="fill: none; stroke: #b03a2e; stroke-width: 1.6; stroke-linecap: round"/> + <path d="M 591.036723 151.653702 +L 586.124861 151.925019 +L 588.854924 147.83274 +z +" style="fill: #b03a2e; stroke: #b03a2e; stroke-width: 1.6; stroke-linecap: round"/> + </g> + <g id="text_19"> + <!-- $z_3^{\beta} = z_3 + d_3$ --> + <g style="fill: #3a3a3a" transform="translate(534.126 181.827528)"> + <text> + <tspan x="0" y="-0.475" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="22.711751" y="-0.475" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="44.963216" y="-0.475" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="5.695638" y="-5.640625" style="font-style: oblique; font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">β</tspan> + <tspan x="5.249023" y="2.259375" style="font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">3</tspan> + <tspan x="27.960775" y="1.165625" style="font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">3</tspan> + <tspan x="51.310872" y="1.165625" style="font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">3</tspan> + <tspan x="12.384603" y="-0.475" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="34.636068" y="-0.475" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">+</tspan> + </text> + </g> + </g> + <g id="text_20"> + <!-- $z_2^{\beta} = z_2 + d_2$ --> + <g style="fill: #3a3a3a" transform="translate(534.126 288.664045)"> + <text> + <tspan x="0" y="-0.475" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="22.711751" y="-0.475" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="44.963216" y="-0.475" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="5.695638" y="-5.640625" style="font-style: oblique; font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">β</tspan> + <tspan x="5.249023" y="2.259375" style="font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">2</tspan> + <tspan x="27.960775" y="1.165625" style="font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">2</tspan> + <tspan x="51.310872" y="1.165625" style="font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">2</tspan> + <tspan x="12.384603" y="-0.475" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="34.636068" y="-0.475" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">+</tspan> + </text> + </g> + </g> + <g id="text_21"> + <!-- $z_1^{\beta} = z_1 + d_1$ --> + <g style="fill: #3a3a3a" transform="translate(534.126 395.400562)"> + <text> + <tspan x="0" y="-0.5625" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="22.711751" y="-0.5625" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="44.963216" y="-0.5625" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="5.695638" y="-5.640625" style="font-style: oblique; font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">β</tspan> + <tspan x="5.249023" y="2.171875" style="font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">1</tspan> + <tspan x="27.960775" y="1.078125" style="font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">1</tspan> + <tspan x="51.310872" y="1.078125" style="font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">1</tspan> + <tspan x="12.384603" y="-0.5625" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="34.636068" y="-0.5625" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">+</tspan> + </text> + </g> + </g> + <g id="patch_13"> + <path d="M 562.176 260.239287 +Q 562.176 231.817088 562.176 204.680629 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 559.976 209.080629 +L 562.176 204.680629 +L 564.376 209.080629 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="patch_14"> + <path d="M 562.176 367.025804 +Q 562.176 338.603605 562.176 311.467146 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 559.976 315.867146 +L 562.176 311.467146 +L 564.376 315.867146 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="patch_15"> + <path d="M 562.176 467.579034 +Q 562.176 443.164529 562.176 420.035763 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 559.976 424.435763 +L 562.176 420.035763 +L 564.376 424.435763 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="text_22"> + <!-- $\mathrm{emb}(x)$ --> + <g style="fill: #3a3a3a" transform="translate(544.176 490.417802)"> + <text> + <tspan x="0" y="-0.40625" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="6.152344" y="-0.40625" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="15.893555" y="-0.40625" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">b</tspan> + <tspan x="22.241211" y="-0.40625" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="32.060547" y="-0.40625" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="26.142578" y="-0.40625" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">x</tspan> + </text> + </g> + </g> + <g id="patch_16"> + <path d="M 671.0976 196.271932 +Q 671.0976 227.809281 671.0976 257.557775 +" style="fill: none; stroke: #d95f02; stroke-width: 1.6; stroke-linecap: round"/> + <path d="M 673.2976 253.157775 +L 671.0976 257.557775 +L 668.8976 253.157775 +z +" style="fill: #d95f02; stroke: #d95f02; stroke-width: 1.6; stroke-linecap: round"/> + </g> + <g id="text_23"> + <!-- $d_2 = J_3^{\top} d_3$ --> + <g style="fill: #d95f02" transform="translate(686.9088 230.584236)"> + <text> + <tspan x="0" y="-0.146875" style="font-style: oblique; font-size: 9.4px; font-family: 'DejaVu Sans'; fill: #d95f02">d</tspan> + <tspan x="21.880463" y="-0.146875" style="font-style: oblique; font-size: 9.4px; font-family: 'DejaVu Sans'; fill: #d95f02">J</tspan> + <tspan x="33.79671" y="-0.146875" style="font-style: oblique; font-size: 9.4px; font-family: 'DejaVu Sans'; fill: #d95f02">d</tspan> + <tspan x="5.950928" y="1.329687" style="font-size: 6.58px; font-family: 'DejaVu Sans'; fill: #d95f02">2</tspan> + <tspan x="24.645355" y="2.314062" style="font-size: 6.58px; font-family: 'DejaVu Sans'; fill: #d95f02">3</tspan> + <tspan x="26.55553" y="-3.592188" style="font-size: 6.58px; font-family: 'DejaVu Sans'; fill: #d95f02">⊤</tspan> + <tspan x="39.747638" y="1.329687" style="font-size: 6.58px; font-family: 'DejaVu Sans'; fill: #d95f02">3</tspan> + <tspan x="12.198761" y="-0.146875" style="font-size: 9.4px; font-family: 'DejaVu Sans'; fill: #d95f02">=</tspan> + </text> + </g> + </g> + <g id="patch_17"> + <path d="M 671.0976 303.058449 +Q 671.0976 334.595797 671.0976 364.344291 +" style="fill: none; stroke: #d95f02; stroke-width: 1.6; stroke-linecap: round"/> + <path d="M 673.2976 359.944291 +L 671.0976 364.344291 +L 668.8976 359.944291 +z +" style="fill: #d95f02; stroke: #d95f02; stroke-width: 1.6; stroke-linecap: round"/> + </g> + <g id="text_24"> + <!-- $d_1 = J_2^{\top} d_2$ --> + <g style="fill: #d95f02" transform="translate(686.9088 337.417753)"> + <text> + <tspan x="0" y="-0.146875" style="font-style: oblique; font-size: 9.4px; font-family: 'DejaVu Sans'; fill: #d95f02">d</tspan> + <tspan x="21.880463" y="-0.146875" style="font-style: oblique; font-size: 9.4px; font-family: 'DejaVu Sans'; fill: #d95f02">J</tspan> + <tspan x="33.79671" y="-0.146875" style="font-style: oblique; font-size: 9.4px; font-family: 'DejaVu Sans'; fill: #d95f02">d</tspan> + <tspan x="5.950928" y="1.329687" style="font-size: 6.58px; font-family: 'DejaVu Sans'; fill: #d95f02">1</tspan> + <tspan x="24.645355" y="2.314062" style="font-size: 6.58px; font-family: 'DejaVu Sans'; fill: #d95f02">2</tspan> + <tspan x="26.55553" y="-3.592188" style="font-size: 6.58px; font-family: 'DejaVu Sans'; fill: #d95f02">⊤</tspan> + <tspan x="39.747638" y="1.329687" style="font-size: 6.58px; font-family: 'DejaVu Sans'; fill: #d95f02">2</tspan> + <tspan x="12.198761" y="-0.146875" style="font-size: 9.4px; font-family: 'DejaVu Sans'; fill: #d95f02">=</tspan> + </text> + </g> + </g> + <g id="text_25"> + <!-- The top state is pulled toward lower loss; each layer's mismatch $d_l$ --> + <g style="fill: #3a3a3a" transform="translate(464.58 547.086236)"> + <text> + <tspan x="0" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">T</tspan> + <tspan x="5.497559" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="11.20166" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="16.73877" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="19.599609" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="23.128418" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="28.634766" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="34.347656" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="37.208496" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="41.897461" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="45.42627" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="50.941406" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="54.470215" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="60.007324" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="62.868164" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="65.368652" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="70.057617" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="72.918457" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="78.631348" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="84.335449" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="86.835938" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="89.336426" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="94.873535" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="100.586426" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="103.447266" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="106.976074" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="112.482422" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">w</tspan> + <tspan x="119.843262" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="125.358398" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="129.058594" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="134.771484" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="137.632324" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="140.132812" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="145.63916" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">w</tspan> + <tspan x="153" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="158.537109" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="162.237305" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="165.098145" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="167.598633" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="173.10498" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="177.793945" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="182.48291" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">;</tspan> + <tspan x="185.515137" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="188.375977" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="193.913086" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="199.428223" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="204.376465" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="210.080566" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="212.941406" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="215.441895" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="220.957031" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">y</tspan> + <tspan x="226.283203" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="231.820312" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="235.520508" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">'</tspan> + <tspan x="237.994629" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="242.683594" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="245.544434" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="254.311523" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="256.812012" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="261.500977" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="270.268066" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="275.783203" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="279.312012" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="284.260254" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="289.964355" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="292.825195" y="-0.15625" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="298.538086" y="1.320312" style="font-style: oblique; font-size: 6.3px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + </text> + </g> + </g> + <g id="text_26"> + <text style="font-size: 9px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="614.88" y="573.791303" transform="rotate(-0 614.88 573.791303)">transmits DOWN through the same weights, and the stack re-settles.</text> + </g> + <g id="text_27"> + <!-- ($J^{\top}$ = the transpose read a bidirectional physical device provides) --> + <g style="fill: #8a8a8a" transform="translate(472.421 606.22482)"> + <text> + <tspan x="0" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">(</tspan> + <tspan x="14.398096" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="17.139734" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">=</tspan> + <tspan x="24.366541" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="27.108179" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="30.489954" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">h</tspan> + <tspan x="35.956384" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="41.262781" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="44.004419" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="47.386194" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="50.932214" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="56.217554" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">n</tspan> + <tspan x="61.683984" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="66.177576" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">p</tspan> + <tspan x="71.652429" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">o</tspan> + <tspan x="76.929346" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="81.422937" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="86.729333" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="89.470972" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="93.016992" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="98.323389" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="103.608728" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">d</tspan> + <tspan x="109.083582" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="111.82522" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="117.110559" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="119.852197" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">b</tspan> + <tspan x="125.327051" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="127.723352" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">d</tspan> + <tspan x="133.198206" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="135.594507" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="139.140527" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="144.446924" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">c</tspan> + <tspan x="149.188989" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="152.570764" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="154.967065" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">o</tspan> + <tspan x="160.243982" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">n</tspan> + <tspan x="165.710413" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="170.995752" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">l</tspan> + <tspan x="173.392053" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="176.133691" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">p</tspan> + <tspan x="181.608545" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">h</tspan> + <tspan x="187.074976" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">y</tspan> + <tspan x="192.179224" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="196.672815" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="199.069116" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">c</tspan> + <tspan x="203.811182" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="209.096521" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">l</tspan> + <tspan x="211.492822" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="214.23446" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">d</tspan> + <tspan x="219.709314" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="225.01571" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">v</tspan> + <tspan x="230.119958" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="232.51626" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">c</tspan> + <tspan x="237.258325" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="242.564722" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="245.30636" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">p</tspan> + <tspan x="250.781213" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="254.327234" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">o</tspan> + <tspan x="259.60415" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">v</tspan> + <tspan x="264.708398" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="267.1047" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">d</tspan> + <tspan x="272.579553" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="277.88595" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="282.379541" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">)</tspan> + <tspan x="3.364929" y="-0.146875" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">J</tspan> + <tspan x="7.716522" y="-3.592188" style="font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #8a8a8a">⊤</tspan> + </text> + </g> + </g> + <g id="text_28"> + <text style="font-weight: 700; font-size: 11px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: start; fill: #3a3a3a" x="35.136" y="662.442054" transform="rotate(-0 35.136 662.442054)">The update — a difference measurement between the two settled states</text> + </g> + <g id="text_29"> + <!-- $\hat{g}\ =\ \left[\ \partial_\theta E(z^{\beta})\ -\ \partial_\theta E(z^{0})\ \right]\,/\,\beta$$\qquad\qquad(\partial_\theta E(z^0)\equiv 0\ \mathrm{here,\ since}\ E=0)$ --> + <g style="fill: #3a3a3a" transform="translate(259.185 697.182472)"> + <text> + <tspan x="7.214355" y="-1.970313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">̂</tspan> + <tspan x="12.69723" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="34.934829" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">∂</tspan> + <tspan x="52.690054" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="68.461555" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="78.467867" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">−</tspan> + <tspan x="93.399471" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">∂</tspan> + <tspan x="111.154696" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="126.911158" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="140.294581" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">/</tspan> + <tspan x="195.667907" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="199.959411" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">∂</tspan> + <tspan x="217.714636" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="233.471097" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="239.905668" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">≡</tspan> + <tspan x="251.265531" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">0</tspan> + <tspan x="261.835808" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="268.807487" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="275.575066" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="280.097526" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="286.865105" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">,</tspan> + <tspan x="296.076495" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="301.807452" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="304.863604" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="311.835284" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="317.883135" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="337.315717" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="348.67558" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">0</tspan> + <tspan x="355.674115" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="0" y="-0.595313" style="font-style: oblique; font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">g</tspan> + <tspan x="45.739859" y="-0.595313" style="font-style: oblique; font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">E</tspan> + <tspan x="56.981558" y="-0.595313" style="font-style: oblique; font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="104.204501" y="-0.595313" style="font-style: oblique; font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">E</tspan> + <tspan x="115.4462" y="-0.595313" style="font-style: oblique; font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="145.78656" y="-0.595313" style="font-style: oblique; font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">β</tspan> + <tspan x="210.76444" y="-0.595313" style="font-style: oblique; font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">E</tspan> + <tspan x="222.00614" y="-0.595313" style="font-style: oblique; font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="328.222455" y="-0.595313" style="font-style: oblique; font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">E</tspan> + <tspan x="27.628835" y="-0.34375" style="font-size: 9.765202px; font-family: 'STIXSizeOneSym'; fill: #3a3a3a">[</tspan> + <tspan x="134.774403" y="-0.34375" style="font-size: 9.765202px; font-family: 'STIXSizeOneSym'; fill: #3a3a3a">]</tspan> + <tspan x="40.728091" y="1.209375" style="font-style: oblique; font-size: 7.7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">θ</tspan> + <tspan x="63.24676" y="-4.80625" style="font-style: oblique; font-size: 7.7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">β</tspan> + <tspan x="99.192733" y="1.209375" style="font-style: oblique; font-size: 7.7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">θ</tspan> + <tspan x="205.752673" y="1.209375" style="font-style: oblique; font-size: 7.7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">θ</tspan> + <tspan x="121.711402" y="-4.80625" style="font-size: 7.7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">0</tspan> + <tspan x="228.271342" y="-4.80625" style="font-size: 7.7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">0</tspan> + </text> + </g> + </g> + <g id="text_30"> + <!-- per layer: $\Delta\theta_l\ \propto\ \langle\ d_l\ ,\ \partial f_l(z_{l-1})/\partial\theta_l\ \rangle\ /\ \beta$ --> + <g style="fill: #3a3a3a" transform="translate(342.1 728.948315)"> + <text> + <tspan x="0" y="-0.328125" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="6.347656" y="-0.328125" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="12.5" y="-0.328125" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="16.611328" y="-0.328125" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="19.790039" y="-0.328125" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="22.568359" y="-0.328125" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="28.696289" y="-0.328125" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">y</tspan> + <tspan x="34.614258" y="-0.328125" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="40.766602" y="-0.328125" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="44.87793" y="-0.328125" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">:</tspan> + <tspan x="48.24707" y="-0.328125" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="51.425781" y="-0.328125" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="54.604492" y="-0.328125" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">Δ</tspan> + <tspan x="74.977018" y="-0.328125" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">∝</tspan> + <tspan x="87.315853" y="-0.328125" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">⟨</tspan> + <tspan x="106.277214" y="-0.328125" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">,</tspan> + <tspan x="114.651205" y="-0.328125" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">∂</tspan> + <tspan x="125.560873" y="-0.328125" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="149.975912" y="-0.328125" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="153.877279" y="-0.328125" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">/</tspan> + <tspan x="157.24642" y="-0.328125" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">∂</tspan> + <tspan x="174.000782" y="-0.328125" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">⟩</tspan> + <tspan x="181.149187" y="-0.328125" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">/</tspan> + <tspan x="61.445312" y="-0.328125" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">θ</tspan> + <tspan x="94.464258" y="-0.328125" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="119.822103" y="-0.328125" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="129.46224" y="-0.328125" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="162.417318" y="-0.328125" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">θ</tspan> + <tspan x="187.765365" y="-0.328125" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">β</tspan> + <tspan x="67.563477" y="1.3125" style="font-style: oblique; font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="100.811914" y="1.3125" style="font-style: oblique; font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="123.342611" y="1.3125" style="font-style: oblique; font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="134.711263" y="1.3125" style="font-style: oblique; font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="168.535482" y="1.3125" style="font-style: oblique; font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="138.019857" y="1.3125" style="font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">−</tspan> + <tspan x="145.248861" y="1.3125" style="font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">1</tspan> + </text> + </g> + </g> + <g id="text_31"> + <text style="font-size: 8.6px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="439.2" y="755.218006" transform="rotate(-0 439.2 755.218006)">Each layer updates from its own boundary mismatch — no global backward graph, no global tape, no loss derivatives except the top nudge.</text> + </g> + </g> + </g> + <defs> + <clipPath id="p90a79555a2"> + <rect x="0" y="0" width="878.4" height="792"/> + </clipPath> + </defs> +</svg> diff --git a/assets/frozen_vs_adaptive.png b/assets/figs/frozen_vs_adaptive.png Binary files differindex e45e77b..e45e77b 100644 --- a/assets/frozen_vs_adaptive.png +++ b/assets/figs/frozen_vs_adaptive.png diff --git a/assets/hw/CLOCKLESS_ANALOG_MVP_PLAN.html b/assets/hw/CLOCKLESS_ANALOG_MVP_PLAN.html new file mode 100644 index 0000000..a2e1e07 --- /dev/null +++ b/assets/hw/CLOCKLESS_ANALOG_MVP_PLAN.html @@ -0,0 +1,509 @@ +<!DOCTYPE html><html><head><meta charset='utf-8'><style> +body { font-family: 'Helvetica Neue', Arial, sans-serif; font-size: 10.5pt; line-height: 1.38; + max-width: 46em; margin: 0 auto; color: #111; } +h1 { font-size: 15pt; border-bottom: 2px solid #333; padding-bottom: 4px; margin-top: 0.4em;} +h2 { font-size: 12pt; margin-top: 1.1em; border-bottom: 1px solid #bbb; padding-bottom: 2px; } +h3 { font-size: 11pt; margin-top: 1em; } +table { border-collapse: collapse; margin: 0.6em 0; font-size: 9.5pt; } +th, td { border: 1px solid #999; padding: 3px 7px; text-align: left; } +th { background: #eee; } +code, pre { font-family: Menlo, Consolas, monospace; font-size: 9pt; background: #f4f4f4; } +pre { padding: 7px 10px; border: 1px solid #ddd; overflow-x: hidden; white-space: pre-wrap; } +blockquote { border-left: 3px solid #888; margin-left: 0; padding-left: 12px; color: #333; } +li { margin: 2px 0; } +@page { margin: 16mm 15mm; } +</style></head><body><h1>Clockless analogue MVP for physical EP research</h1> +<p><strong>Prepared:</strong> 2026-07-11<br /> +<strong>Starting point:</strong> <code>COMPONENT_HW_MAP.md</code><br /> +<strong>Scope:</strong> academic prototype, not a product demonstrator; analogue learning core; no processor, ADC, DAC, FPGA, sample-and-hold bank, phase clock, or digital optimizer in the learning loop.</p> +<hr /> +<h2>1. Executive decision</h2> +<p>The current <code>$5k–$20k</code> Demo-0 is not an MVP. It is an architecture demonstrator that tries to validate, at once, an SRAM-CIM module, bidirectional transpose reads, RoPE mixers, QK/RMS normalization, analogue softmax, SwiGLU multipliers, analogue state handling, ADC/DAC boundaries, a phase sequencer, write programming, and digital supervisory logic. A failure would be difficult to attribute to one mechanism.</p> +<p>The lower-risk research question is narrower and more publishable:</p> +<blockquote> +<p>Can a reciprocal nonlinear analogue network continuously learn by a local equilibrium contrast, with the free and nudged states physically present at the same time, without a clock or processor in the learning loop?</p> +</blockquote> +<p>Build that first. The recommended core is a <strong>twin-equilibrium analogue learning tile</strong>:</p> +<ul> +<li>two small physical replicas of the same reciprocal nonlinear resistor network;</li> +<li>corresponding trainable edges share one analogue weight capacitor;</li> +<li>both replicas run continuously, so there are no sequential free/nudged phases;</li> +<li>an output OTA supplies a true EP-style current nudge; a jumper also provides a voltage-clamped Coupled Learning control condition;</li> +<li>each edge updates its own capacitor from a local contrast;</li> +<li>manual boundary-condition selection and a manual learn/freeze switch replace an FPGA;</li> +<li>a scope or DMM may observe the circuit but is not part of the feedback or learning path.</li> +</ul> +<p>Recommended first populated board: <strong>8 sign-update edges plus one exact multiplier reference channel</strong>, approximately <strong>$170–$300</strong> excluding instruments, tax, and shipping. This is about <strong>17×–118× below</strong> the current <code>$5k–$20k</code> board class. A one-edge calibration rig is approximately <strong>$70–$130</strong>.</p> +<hr /> +<h2>2. What “completely analogue and clockless” can honestly mean</h2> +<p>A defensible claim is:</p> +<blockquote> +<p>The state evolution, nudge, local learning rule, and weight storage are continuous-time analogue processes. No periodic control signal, processor, converter, sampled state memory, or digitally computed parameter update participates in learning.</p> +</blockquote> +<p>Allowed outside the claim:</p> +<ul> +<li>a bench supply;</li> +<li>an oscilloscope or DMM used only for observation;</li> +<li>a human changing input/target switches;</li> +<li>a static mechanical learn/freeze switch;</li> +<li>offline analysis after an experiment.</li> +</ul> +<p>A full language-model demonstration cannot honestly be entirely analogue and clockless at the token interface: symbol lookup, presentation of a sequence, cross-entropy labels, and token sampling are discrete operations. The academic MVP should therefore validate the <strong>physical learning primitive</strong>, then add a very small reciprocal attention cell as a second experiment.</p> +<hr /> +<h2>3. Core topology</h2> +<pre><code class="language-text"> analogue input x analogue target y* + │ │ + ┌────────────┴────────────┐ │ + │ │ │ + ▼ ▼ │ + FREE reciprocal network NUDGED reciprocal network + natural RC settling natural RC settling + output y0 output yβ + nudge current + │ ▲ │ + │ │ OTA/error transconductor + │ └──────────────┘ + │ + for every trainable edge e: + + edge voltage Δv0,e edge voltage Δvβ,e + │ │ + └──── local contrast cell ─────┘ + │ + charge/discharge current + │ + shared weight Cw,e + │ + ┌────────────┴────────────┐ + ▼ ▼ + MOSFET edge in free net MOSFET edge in nudged net +</code></pre> +<p>The two replicas must be laid out symmetrically and use matched transistor pairs where practical. The shared capacitor ensures both edge copies always use the same learned weight.</p> +<h3>3.1 State dynamics</h3> +<p>The node voltages are the states. Resistors/MOSFET conductances and node capacitances produce the relaxation automatically. Add small capacitors only where needed to set a reproducible pole and suppress oscillation; do not build a capacitor-plus-OTA integrator for every abstract model state.</p> +<p>Design target, not a theorem:</p> +<pre><code>τ_{weight}/τ_state ≥ 10^2, preferably 10^3. +</code></pre> +<p>The state should equilibrate much faster than the weight capacitors move. This time-scale separation replaces a settle detector and phase sequencer.</p> +<h3>3.2 True EP nudge and the CL control</h3> +<p>Populate both modes on the same board.</p> +<p><strong>EP mode — current/force nudge.</strong> For voltage outputs, compute the output error and inject a proportional current at the nudged output. A practical small-nudge implementation is</p> +<pre><code>I_β = -g_β (y_0-y^*) +</code></pre> +<p>with an LM13700-class OTA or a discrete transconductor. The polarity is chosen to push the nudged output toward the target. This crosses voltage error with current nudge and is the mode to use for the formal EP/gradient-flow claim.</p> +<p><strong>CL mode — voltage constraint.</strong> Buffer a weighted voltage between the free output and the target and impose it on the second replica. This is easier and reproduces the demonstrated clockless Coupled Learning architecture, but it should not be called exact EP.</p> +<p>A two-position switch should select EP-current or CL-voltage nudge. That comparison is itself a useful experiment.</p> +<h3>3.3 Exact local contrast</h3> +<p>For a conductance-like parameter, the local energy derivative is proportional to the squared voltage drop. Use</p> +<pre><code>C_{w,e} dV/dt_{w,e} + = s_e k≤ft[(\Delta v_{β,e})^2-(\Delta v_{0,e})^2], +</code></pre> +<p>where <code>s_e = ±1</code> accounts for whether increasing capacitor voltage increases or decreases effective conductance.</p> +<p>Do not square twice. Use</p> +<pre><code>a^2-b^2=(a-b)(a+b), +</code></pre> +<p>so one four-quadrant multiplier can implement an exact reference channel. The established laboratory circuit used an AD633 with op-amp conditioning and a local capacitor.</p> +<h3>3.4 Low-cost sign contrast</h3> +<p>The exact multiplier dominates cost. A much cheaper local rule is</p> +<pre><code>C_{w,e} dV/dt_{w,e} + = s_e I_0 sgn≤ft(|\Delta v_{β,e}|-|\Delta v_{0,e}|) +</code></pre> +<p>outside a deadband <code>δ</code>, with zero current inside the deadband. Implement it with absolute-value/rectifier stages, a comparator with hysteresis, and two matched charge/discharge current sources.</p> +<p>This rule has precedent in simulated memristor EP hardware because it removes the analogue multiplier. The <strong>continuous capacitor implementation proposed here is an engineering adaptation</strong>, not an already demonstrated result. That is a legitimate research contribution, but it must be labeled correctly.</p> +<h3>3.5 Freeze and retention</h3> +<p>Use a mechanical toggle or relay to disconnect update current from every weight capacitor. Avoid a clocked switch matrix. Use film capacitors initially; characterize leakage and dielectric absorption. Volatile analogue weights are acceptable for an academic demonstrator, but not for a storage product.</p> +<hr /> +<h2>4. Recommended hardware ladder</h2> +<h3>Rung A — one-edge metrology tile</h3> +<p>Purpose: validate the physics before assembling a network.</p> +<p>Populate:</p> +<ul> +<li>one free/nudged MOSFET pair;</li> +<li>one shared weight capacitor;</li> +<li>one exact AD633 contrast path;</li> +<li>one sign-only contrast path selectable by jumper;</li> +<li>one EP-current nudge OTA;</li> +<li>one CL-voltage nudge path;</li> +<li>test points for all four edge terminals, multiplier inputs/output, capacitor current, and weight voltage.</li> +</ul> +<p>Estimated cost: <strong>$70–$130</strong>.</p> +<p>Required measurements:</p> +<ol> +<li>transfer surface <code>I_update(Δv0, Δvβ)</code>;</li> +<li>exact-channel gain and offset;</li> +<li>sign-channel boundary, hysteresis, and deadband;</li> +<li>zero-contrast drift at <code>Δv0 = Δvβ</code>;</li> +<li>weight retention with learning frozen;</li> +<li>temperature drift;</li> +<li>nudge linearity versus <code>β</code>;</li> +<li>time-scale ratio between node settling and weight motion.</li> +</ol> +<p>Do not build the multi-edge board until this tile gives a stable null at zero contrast.</p> +<h3>Rung B — two modular four-edge cards</h3> +<p>Build a four-edge card and populate two cards for eight trainable edges. Each card contains:</p> +<ul> +<li>four twin MOSFET edges;</li> +<li>four sign-update cells;</li> +<li>four weight capacitors;</li> +<li>local trim/deadband points;</li> +<li>a shared learn/freeze bus that is static, not clocked;</li> +<li>edge terminals on headers so the network topology is patchable.</li> +</ul> +<p>Recommended population: eight sign channels plus <strong>one parallel exact AD633 channel on a selected edge</strong>. This gives a continuously measured exact-versus-sign comparison without buying eight multipliers.</p> +<p>Estimated cost: <strong>$170–$300</strong>.</p> +<p>Initial task:</p> +<ul> +<li>one-input nonlinear regression with 4–8 manually selected static examples;</li> +<li>then a two-input task after topology simulation and one-edge characterization.</li> +</ul> +<p>Do not promise XOR at eight edges. The published nonlinear clockless network used 32 twin edges for XOR. Design the card so eight identical four-edge modules can be stacked later.</p> +<h3>Rung C — 32-edge replication-class network</h3> +<p>Use eight four-edge cards and the sign-update rule. This approaches the scale of the published nonlinear demonstration while avoiding 32 AD633 multipliers.</p> +<p>Estimated cost: <strong>$450–$900</strong>.</p> +<p>This is still below the low end of the original plan by approximately <strong>5.5×–44×</strong> and is large enough for a serious robustness and nonlinear-learning study.</p> +<h3>Rung D — reciprocal attention microcell</h3> +<p>Only after Rungs A–C work, add a transformer-adjacent cell:</p> +<ul> +<li>two tokens;</li> +<li>one head;</li> +<li>scalar or two-dimensional state;</li> +<li>tied key/value or another explicitly energy-based reciprocal construction;</li> +<li>hardwired causal connectivity;</li> +<li>no embedding memory, RoPE, RMSNorm, QK norm, SwiGLU, ADC, LM head, or token sampler.</li> +</ul> +<p>For two alternatives, softmax reduces to a logistic function of a score difference, so a differential pair can replace a general <code>N</code>-way entropic-resistor array. This is an <strong>attention-shaped energy cell</strong>, not an OLMo2 block.</p> +<p>Estimated total including the learning core: <strong>$300–$700</strong>.</p> +<hr /> +<h2>5. Costed options</h2> +<p>Budgetary single-quantity catalogue prices checked on 2026-07-11; prices exclude tax, shipping, instruments, assembly labor, and rework.</p> +<table> +<thead> +<tr> +<th>Item</th> +<th style="text-align: right;">Planning role</th> +<th style="text-align: right;">Unit price used</th> +<th style="text-align: right;">Planning quantity, 8-edge board</th> +<th style="text-align: right;">Extended</th> +</tr> +</thead> +<tbody> +<tr> +<td>ALD1106PBL</td> +<td style="text-align: right;">matched N-MOS array; two twin edges/package</td> +<td style="text-align: right;">$9.29</td> +<td style="text-align: right;">4</td> +<td style="text-align: right;">$37.16</td> +</tr> +<tr> +<td>AD633ANZ</td> +<td style="text-align: right;">exact four-quadrant contrast multiplier</td> +<td style="text-align: right;">$21.48</td> +<td style="text-align: right;">0, 1, or 8</td> +<td style="text-align: right;">$0 / $21.48 / $171.84</td> +</tr> +<tr> +<td>TLV274IPWR</td> +<td style="text-align: right;">quad rail-to-rail op amp</td> +<td style="text-align: right;">$1.43</td> +<td style="text-align: right;">6–8</td> +<td style="text-align: right;">$8.58–$11.44</td> +</tr> +<tr> +<td>LM13700</td> +<td style="text-align: right;">dual OTA; output nudge/current sources</td> +<td style="text-align: right;">$1.70</td> +<td style="text-align: right;">1–2</td> +<td style="text-align: right;">$1.70–$3.40</td> +</tr> +<tr> +<td>LM339-class comparator</td> +<td style="text-align: right;">sign/deadband channels</td> +<td style="text-align: right;">about $0.67</td> +<td style="text-align: right;">2–4</td> +<td style="text-align: right;">about $1.34–$2.68</td> +</tr> +<tr> +<td>CD4066-class switch</td> +<td style="text-align: right;">optional static freeze/routing</td> +<td style="text-align: right;">about $0.73–$0.97</td> +<td style="text-align: right;">2–4</td> +<td style="text-align: right;">about $1.46–$3.88</td> +</tr> +<tr> +<td>Film capacitors</td> +<td style="text-align: right;">weight storage</td> +<td style="text-align: right;">about $0.63 at 1 µF</td> +<td style="text-align: right;">8–12</td> +<td style="text-align: right;">about $5–$8</td> +</tr> +<tr> +<td>Diodes, resistors, trims</td> +<td style="text-align: right;">rectifiers, limits, biasing</td> +<td style="text-align: right;">—</td> +<td style="text-align: right;">lot</td> +<td style="text-align: right;">$15–$40</td> +</tr> +<tr> +<td>PCB/protoboard, headers, test points</td> +<td style="text-align: right;">physical implementation</td> +<td style="text-align: right;">—</td> +<td style="text-align: right;">lot</td> +<td style="text-align: right;">$35–$120</td> +</tr> +<tr> +<td>Power rails, protection, spare parts</td> +<td style="text-align: right;">laboratory overhead</td> +<td style="text-align: right;">—</td> +<td style="text-align: right;">lot</td> +<td style="text-align: right;">$30–$90</td> +</tr> +</tbody> +</table> +<p>Resulting envelopes:</p> +<table> +<thead> +<tr> +<th>Variant</th> +<th style="text-align: right;">Estimated build cost</th> +<th>Recommendation</th> +</tr> +</thead> +<tbody> +<tr> +<td>One-edge exact/sign calibration tile</td> +<td style="text-align: right;">$70–$130</td> +<td>Build first</td> +</tr> +<tr> +<td>8-edge sign-only</td> +<td style="text-align: right;">$140–$250</td> +<td>Cheapest useful network</td> +</tr> +<tr> +<td>8-edge sign + one exact reference channel</td> +<td style="text-align: right;">$170–$300</td> +<td><strong>Recommended MVP</strong></td> +</tr> +<tr> +<td>8-edge all-exact AD633</td> +<td style="text-align: right;">$300–$500</td> +<td>Only after reference channel works</td> +</tr> +<tr> +<td>32-edge sign-update network</td> +<td style="text-align: right;">$450–$900</td> +<td>Replication-class nonlinear demo</td> +</tr> +<tr> +<td>Reciprocal two-token attention add-on</td> +<td style="text-align: right;">+$100–$250</td> +<td>Phase 2 only</td> +</tr> +</tbody> +</table> +<h3>Aggressive transistor substitution</h3> +<p>A CD4007UBE costs about <code>$0.89</code> and contains a CMOS dual complementary pair plus inverter. It can be explored on the one-edge tile, but it is not the main-board recommendation: matching, body connections, and device operating region become the dominant uncertainty. Saving roughly <code>$30</code> of matched-transistor cost on an eight-edge board is not worth sacrificing the experiment’s interpretability.</p> +<hr /> +<h2>6. Delete list for the original plan</h2> +<p>For the academic MVP, delete these entirely:</p> +<ul> +<li>SRAM-CIM, Mythic, HERMES, or other accelerator evaluation board;</li> +<li>FPGA and phase sequencer;</li> +<li>DAC and ADC arrays;</li> +<li>token embedding lookup;</li> +<li>RoPE DDS and mixers;</li> +<li>full causal softmax array;</li> +<li>QK normalization;</li> +<li>RMSNorm banks;</li> +<li>SwiGLU multiplier array;</li> +<li>digital cross-entropy head;</li> +<li>Adam/Muon or any digital optimizer;</li> +<li>crossbar write-programming machinery;</li> +<li>sampled free-phase memories;</li> +<li>digital settle/retry logic;</li> +<li>GPU gradient telemetry.</li> +</ul> +<p>Keep or replace as follows:</p> +<table> +<thead> +<tr> +<th>Original function</th> +<th>MVP replacement</th> +</tr> +</thead> +<tbody> +<tr> +<td>MVM/crossbar</td> +<td>patchable reciprocal nonlinear resistor network</td> +</tr> +<tr> +<td>state integrators</td> +<td>natural RC/KCL state dynamics</td> +</tr> +<tr> +<td>sequential free/nudged phases</td> +<td>two continuously operating physical replicas</td> +</tr> +<tr> +<td>DAC nudge</td> +<td>OTA current injection</td> +</tr> +<tr> +<td>digital phase memory</td> +<td>simultaneous physical state comparison</td> +</tr> +<tr> +<td>outer-product update/programming</td> +<td>local capacitor charge/discharge</td> +</tr> +<tr> +<td>FPGA control</td> +<td>manual boundary selection and learn/freeze</td> +</tr> +<tr> +<td>ADC telemetry</td> +<td>buffered scope/DMM observation outside loop</td> +</tr> +</tbody> +</table> +<hr /> +<h2>7. Experimental matrix</h2> +<p>The cleanest paper is not merely “it learned.” It should isolate the choices that make the clockless implementation possible.</p> +<h3>Factor 1 — nudge type</h3> +<ul> +<li>EP: voltage error, current nudge;</li> +<li>CL: voltage error, voltage constraint.</li> +</ul> +<h3>Factor 2 — local update</h3> +<ul> +<li>exact difference of squares;</li> +<li>sign-only magnitude comparison;</li> +<li>sign-only with deadband/hysteresis.</li> +</ul> +<h3>Factor 3 — device quality</h3> +<ul> +<li>matched ALD1106 edges;</li> +<li>one-edge CD4007 substitution;</li> +<li>deliberate mismatch or trim-offset injection.</li> +</ul> +<h3>Factor 4 — nudge magnitude</h3> +<ul> +<li>small nudge;</li> +<li>moderate/overclamped nudge;</li> +<li>measured bias-versus-SNR curve.</li> +</ul> +<h3>Factor 5 — operating conditions</h3> +<ul> +<li>temperature;</li> +<li>supply variation;</li> +<li>capacitor leakage/hold time;</li> +<li>damaged or disconnected edge;</li> +<li>topology size.</li> +</ul> +<p>Proposed acceptance criteria, to be declared before network training:</p> +<ol> +<li>no periodic control signal in the learning loop;</li> +<li>no processor, ADC, DAC, or stored free-phase sample used for an update;</li> +<li>exact reference channel has a stable zero-contrast null;</li> +<li>sign channel agrees with the exact channel on update direction in at least 95% of the specified operating grid;</li> +<li>frozen weights remain within the chosen tolerance over the measurement interval;</li> +<li>loss decreases consistently across multiple resets, with all failures reported;</li> +<li>EP-current and CL-voltage modes are labeled separately.</li> +</ol> +<hr /> +<h2>8. Six-week execution plan</h2> +<h3>Week 1 — SPICE and one-edge schematic</h3> +<ul> +<li>simulate the MOSFET edge over the intended node/gate voltage range;</li> +<li>choose <code>Cw</code>, update current, and state capacitance for at least 100× time-scale separation;</li> +<li>simulate exact and sign contrast transfer surfaces;</li> +<li>simulate OTA current nudge and output compliance;</li> +<li>add hard rails on weight voltage.</li> +</ul> +<h3>Week 2 — one-edge breadboard</h3> +<ul> +<li>build only the exact path first;</li> +<li>measure offsets and null drift;</li> +<li>add sign path and deadband;</li> +<li>add manual freeze;</li> +<li>decide whether breadboard leakage is tolerable or proceed directly to PCB.</li> +</ul> +<h3>Week 3 — one-edge characterization</h3> +<ul> +<li>automate only measurement if desired; do not put automation in the learning loop;</li> +<li>produce measured update-vector fields;</li> +<li>identify the safe voltage region;</li> +<li>lock component values for the modular card.</li> +</ul> +<h3>Week 4 — first four-edge card</h3> +<ul> +<li>symmetrical placement;</li> +<li>dense test points;</li> +<li>patchable node topology;</li> +<li>one exact reference edge routed in parallel with its sign cell.</li> +</ul> +<h3>Week 5 — eight-edge network</h3> +<ul> +<li>build a topology selected by SPICE;</li> +<li>train one static example, then 4–8 manually selected regression points;</li> +<li>compare exact-reference and sign decisions continuously.</li> +</ul> +<h3>Week 6 — robustness and paper figures</h3> +<ul> +<li>CL versus EP;</li> +<li>exact versus sign;</li> +<li>mismatch and temperature;</li> +<li>retention and drift;</li> +<li>energy/power by function;</li> +<li>scaling estimate to 32 edges and the reciprocal attention cell.</li> +</ul> +<hr /> +<h2>9. Claims this MVP can and cannot support</h2> +<h3>It can support</h3> +<ul> +<li>a continuous-time analogue physical-learning core;</li> +<li>clockless simultaneous free/nudged operation;</li> +<li>local analogue weight storage and updates;</li> +<li>processor-free learning after boundary conditions are applied;</li> +<li>measured exact-versus-sign update behavior;</li> +<li>measured EP-current-versus-CL-voltage behavior;</li> +<li>nonlinear supervised learning in a small reciprocal network;</li> +<li>a credible scaling BOM.</li> +</ul> +<h3>It cannot support</h3> +<ul> +<li>a full analogue OLMo2 block;</li> +<li>ordinary independent-Q/K/V attention on a passive reciprocal network;</li> +<li>end-to-end clockless language modeling;</li> +<li>nonvolatile weights;</li> +<li>a claim that the sign-only capacitor rule is exact EP;</li> +<li>a claim that voltage-clamped Coupled Learning is identical to EP;</li> +<li>production energy/area numbers derived from a breadboard.</li> +</ul> +<p>A passive reciprocal network avoids the adjoint/transposed-Jacobian problem by construction, but it also excludes normal non-reciprocal attention. The transformer connection must therefore use tied, energy-based attention or be deferred to an active-adjoint phase.</p> +<hr /> +<h2>10. Why this is the better academic MVP</h2> +<p>It tests one scientific claim at a time. Every expensive item in the original plan exists to preserve a nearly complete transformer block. That is appropriate after the physical learning primitive is established, not before it.</p> +<p>The proposed sequence produces publishable intermediate results even if the final nonlinear task fails:</p> +<ul> +<li>measured clockless local-gradient circuit;</li> +<li>exact-versus-sign learning rule;</li> +<li>EP-versus-CL physical comparison;</li> +<li>mismatch-induced bias and its suppression;</li> +<li>analogue weight retention and drift;</li> +<li>modular scaling law;</li> +<li>reciprocal attention microcell.</li> +</ul> +<p>The main recommendation is therefore:</p> +<blockquote> +<p>Build the <code>$70–$130</code> one-edge tile, then the <code>$170–$300</code> eight-edge hybrid board. Do not purchase a CIM evaluation module, FPGA, DAC/ADC bank, or softmax hardware for the MVP.</p> +</blockquote> +<hr /> +<h2>11. Primary references used</h2> +<ol> +<li>Sam Dillavou et al., <strong>“Machine Learning Without a Processor: Emergent Learning in a Nonlinear Electronic Metamaterial,”</strong> arXiv:2311.00537v2 / PNAS (2024). Demonstrated a fully analogue, clockless nonlinear twin-resistor network; 32 twin edges; local capacitive learning; XOR and nonlinear regression. Its implemented rule is Coupled Learning.</li> +<li>Sam Dillavou et al., <strong>“Understanding and Embracing Imperfection in Physical Learning Networks,”</strong> arXiv:2505.22887v2 (2026 revision). Characterizes bias, limit cycles, and system-agnostic suppression of analogue learning imperfections.</li> +<li>Joshua A. McGinnis, Xinbo Li, and Yoichiro Mori, <strong>“Coercivity and Local Convergence of Physical Learning in Linear Circuits,”</strong> arXiv:2606.15443 (2026). Distinguishes EP’s force/current nudge from CL’s voltage constraint and analyzes their gradient-flow structure.</li> +<li>Seung-Hyun Oh et al., <strong>“Memristor Crossbar Circuits Implementing Equilibrium Propagation for On-Device Learning,”</strong> <em>Micromachines</em> 14(7):1367 (2023). Proposes simultaneous free/nudged networks and a sign-only fixed-step update to remove analogue multipliers; results are simulation-oriented for memristor programming.</li> +<li>I. M. Elfadel and J. L. Wyatt, <strong>“The ‘Softmax’ Nonlinearity: Derivation Using Statistical Mechanics and Useful Properties as a Multiterminal Analog Circuit Element,”</strong> NeurIPS 1993. Establishes reciprocal/passive circuit properties of an entropic softmax element.</li> +<li><strong>“How to Train Your Resistive Network: Generalized Equilibrium Propagation and Analytical Learning,”</strong> arXiv:2602.03546 (2026). Gives a single-network projector-based route for linear resistive networks; useful as a future replica-reduction direction, not the first nonlinear build.</li> +</ol></body></html>
\ No newline at end of file diff --git a/assets/hw/COLLABORATOR_BRIEF.html b/assets/hw/COLLABORATOR_BRIEF.html new file mode 100644 index 0000000..48df78c --- /dev/null +++ b/assets/hw/COLLABORATOR_BRIEF.html @@ -0,0 +1,98 @@ +<!DOCTYPE html><html><head><meta charset='utf-8'><style> +body { font-family: 'Helvetica Neue', Arial, sans-serif; font-size: 10.5pt; line-height: 1.38; + max-width: 46em; margin: 0 auto; color: #111; } +h1 { font-size: 15pt; border-bottom: 2px solid #333; padding-bottom: 4px; margin-top: 0.4em;} +h2 { font-size: 12pt; margin-top: 1.1em; border-bottom: 1px solid #bbb; padding-bottom: 2px; } +h3 { font-size: 11pt; margin-top: 1em; } +table { border-collapse: collapse; margin: 0.6em 0; font-size: 9.5pt; } +th, td { border: 1px solid #999; padding: 3px 7px; text-align: left; } +th { background: #eee; } +code, pre { font-family: Menlo, Consolas, monospace; font-size: 9pt; background: #f4f4f4; } +pre { padding: 7px 10px; border: 1px solid #ddd; overflow-x: hidden; white-space: pre-wrap; } +blockquote { border-left: 3px solid #888; margin-left: 0; padding-left: 12px; color: #333; } +li { margin: 2px 0; } +@page { margin: 16mm 15mm; } +</style></head><body><h1>Backprop-free training of transformers — results and a staged hardware program</h1> +<p><strong>One-page brief for hardware-side collaborators · rev. 2026-07-12 · Yuren Hao (UIUC)</strong></p> +<h2>The idea in three sentences</h2> +<p>We train <strong>standard multi-layer transformers</strong> with <strong>Equilibrium Propagation</strong> on a layered energy: +training consists of two relaxation phases and a <strong>local</strong> contrast update per weight — no +backpropagation anywhere — and inference is an ordinary forward pass. On GPU this now works at +language-model scale with essentially no quality gap to backprop. The hardware program starts with +the cheapest object that can validate the physical learning rule — a <strong>clockless twin-network +analog tile (~$300)</strong> — and climbs rung by rung to an in-memory-compute transformer block.</p> +<h2>GPU-scale results (2026-07, measured)</h2> +<ul> +<li><strong>A 12-layer, 42.7M-parameter transformer LM trained for a full epoch (59k steps, 361M tokens) + with no backpropagation in the training loop; it generates coherent text.</strong> To our knowledge the + first transformer language model trained fully this way.</li> +<li><strong>Gap to a tuned, same-architecture backprop control: 0.05 nats</strong> (at 4k steps: statistically + indistinguishable, 3 seeds/arm). Prior backprop-free attempts at scale all report qualitative gaps.</li> +<li>EP step = <strong>3.2× backprop FLOPs</strong> (measured); mixed-precision training validated; the two known + EP-specific instabilities are mechanistically diagnosed and closed (an estimator-SNR floor with a + β-schedule law; a relaxation-contractivity crossing eliminated by norm placement).</li> +<li><strong>Every trained operation chosen analog-implementable:</strong> crossbar MVM, divisive normalization, + fixed I/Q rotations (position code), translinear gated MLP, subthreshold-exponential softmax, + two-phase relaxation for the learning rule.</li> +</ul> +<h2>Measured fault tolerances (fault injection at the trained model)</h2> +<table> +<thead> +<tr> +<th>fault</th> +<th>free</th> +<th>marginal</th> +<th>dead</th> +</tr> +</thead> +<tbody> +<tr> +<td>weight precision</td> +<td><strong>8-bit</strong> (ΔCE +0.004)</td> +<td>6-bit (+0.05)</td> +<td>4-bit</td> +</tr> +<tr> +<td>forward state noise</td> +<td><strong>1%</strong></td> +<td>—</td> +<td>—</td> +</tr> +<tr> +<td>error-channel (nudge) noise</td> +<td><strong>10% relative</strong></td> +<td>30%</td> +<td>—</td> +</tr> +<tr> +<td>divider mismatch / gate gain / phase error</td> +<td>3% / 10% / 0.03 rad</td> +<td>10% / — / 0.1 rad</td> +<td>—</td> +</tr> +</tbody> +</table> +<p>Under every non-fatal fault the learning signal tracks the <em>faulted</em> network (gradient cosine ≈ 0.97 +invariant): <strong>the rule co-adapts to the device.</strong> The only hard spec is ~7-bit effective weights.</p> +<h2>The hardware ladder (each rung publishable alone)</h2> +<ol> +<li><strong>One-edge metrology tile ($70–130):</strong> twin MOSFET edge, shared weight capacitor, exact + (difference-of-squares) and sign-only local update channels, OTA current nudge — no processor, + converter, clock, or sampled memory in the learning loop.</li> +<li><strong>8-edge twin network ($170–300):</strong> nonlinear regression; EP current-nudge vs Coupled-Learning + voltage-clamp on one board; exact-vs-sign update comparison; measured bias-vs-nudge-magnitude + curve (the same β-SNR law we measured in simulation).</li> +<li><strong>32-edge network ($450–900):</strong> replication-class nonlinear tasks, robustness study.</li> +<li><strong>Reciprocal attention microcell (+$100–250):</strong> two tokens, one head, energy-based attention.</li> +<li><strong>CIM transformer block (partner phase):</strong> analog MVM + in-situ two-phase EP weight update — + the piece no shipping analog-AI chip has (all are inference-only or on-chip-backprop).</li> +<li><strong>North star: a few-M-parameter TinyStories LM trained on analog hardware.</strong></li> +</ol> +<h2>What we bring / what we ask</h2> +<p><strong>Bring:</strong> the trained models and recipe, the estimator theory (β-SNR law, stability walls), the +measured tolerance ledger, SPICE-first costed build plan, and parts funding (rungs 1–3 are <$1k). +<strong>Ask (rungs 1–3):</strong> bench access, analog-design mentorship, and/or a student who enjoys discrete +analog — six-week plan, instruments = a scope and a DMM. <strong>Ask (rung 5):</strong> a CIM/mixed-signal +partnership where the substrate expertise is yours and the learning rule is ours.</p> +<p><em>(Detail: CLOCKLESS_ANALOG_MVP_PLAN.md — full BOM, schedule, acceptance criteria, claim limits; +COMPONENT_HW_MAP.md — per-operation analog mapping + tolerance status.)</em></p></body></html>
\ No newline at end of file diff --git a/assets/hw/hw_mvp_explainer.html b/assets/hw/hw_mvp_explainer.html new file mode 100644 index 0000000..4114648 --- /dev/null +++ b/assets/hw/hw_mvp_explainer.html @@ -0,0 +1,220 @@ +<!DOCTYPE html> +<html lang="zh"> +<head> +<meta charset="utf-8"> +<title>无时钟模拟学习板 — 结构图解（给不懂硬件的人）</title> +<style> + body { font-family: -apple-system, "PingFang SC", "Microsoft YaHei", sans-serif; margin: 0; + background: #fafbfd; color: #222; line-height: 1.65; } + .wrap { max-width: 1080px; margin: 0 auto; padding: 28px 20px 80px; } + h1 { color: #13294B; font-size: 26px; border-bottom: 3px solid #E84A27; padding-bottom: 8px; } + h2 { color: #13294B; font-size: 20px; margin-top: 2.2em; } + .lead { font-size: 16.5px; color: #444; } + .lead b { color: #E84A27; } + svg { width: 100%; height: auto; background: white; border: 1px solid #dde3ec; border-radius: 10px; } + .hint { text-align: center; color: #888; font-size: 13px; margin-top: 6px; } + [data-tip] { cursor: help; } + [data-tip]:hover { filter: brightness(1.15) drop-shadow(0 0 4px rgba(232,74,39,.45)); } + #tip { position: fixed; display: none; max-width: 340px; background: #13294B; color: #fff; + padding: 10px 13px; border-radius: 8px; font-size: 13.5px; line-height: 1.5; z-index: 10; + box-shadow: 0 4px 18px rgba(0,0,0,.25); pointer-events: none; } + #tip b { color: #FFB894; } + table { border-collapse: collapse; width: 100%; font-size: 14.5px; } + th, td { border: 1px solid #ccd4e0; padding: 8px 12px; text-align: left; } + th { background: #13294B; color: white; } + tr:nth-child(even) { background: #f0f3f8; } + .steps li { margin: 10px 0; } + .steps b { color: #13294B; } + .chip { display: inline-block; background: #eef1f7; border: 1px solid #ccd4e0; border-radius: 14px; + padding: 3px 12px; margin: 3px 4px; font-size: 13.5px; } + .chip b { color: #E84A27; } + .note { background: #fff7f3; border-left: 4px solid #E84A27; padding: 10px 16px; border-radius: 0 8px 8px 0; + font-size: 14.5px; } +</style> +</head> +<body> +<div class="wrap"> + +<h1>无时钟模拟学习板：它长什么样、为什么能学习</h1> +<p class="lead"> +一句话：这块板子上<b>同时活着两份一模一样的电路</b>——一份自由地"回答问题"（FREE），一份的输出被轻轻拉向正确答案（NUDGED）。 +每个"权重"是一颗电容，它不停地比较自己在两份电路里的受力差异、朝着缩小差异的方向缓慢充放电。 +<b>没有处理器、没有时钟、没有数模转换——物理本身在做训练。</b> +把鼠标悬停在图里任何东西上看它是什么。 +</p> + +<svg viewBox="0 0 1200 620" xmlns="http://www.w3.org/2000/svg"> + <defs> + <marker id="arr" markerWidth="9" markerHeight="9" refX="7" refY="3" orient="auto"> + <path d="M0,0 L7,3 L0,6 Z" fill="#2E8B57"/> + </marker> + </defs> + + <!-- ================== FREE replica ================== --> + <text x="235" y="46" text-anchor="middle" font-size="21" font-weight="bold" fill="#13294B">FREE 网络（自由态）</text> + <text x="235" y="70" text-anchor="middle" font-size="13" fill="#888">自然弛豫，输出 = 网络自己的回答 ŷ</text> + <g data-tip="<b>FREE 网络</b>：真实的物理电路。输入电压加上后，电流按电阻自然流动，节点电压在微秒级"稳定下来"（settle）——这个稳定过程就是 ML 里的前向传播，不需要任何计算指令。"> + <rect x="60" y="85" width="350" height="420" rx="14" fill="#f4f6fa" stroke="#13294B" stroke-width="2"/> + </g> + <!-- nodes --> + <g data-tip="<b>输入节点</b>：一个电压源把输入 x 设成固定电压（比如 0.3V 代表某个特征值）。两份网络收到完全相同的输入。"> + <circle cx="120" cy="170" r="22" fill="white" stroke="#13294B" stroke-width="2.5"/> + <text x="120" y="176" text-anchor="middle" font-size="15" fill="#13294B" font-style="italic">x₁</text> + <circle cx="120" cy="420" r="22" fill="white" stroke="#13294B" stroke-width="2.5"/> + <text x="120" y="426" text-anchor="middle" font-size="15" fill="#13294B" font-style="italic">x₂</text> + </g> + <g data-tip="<b>隐藏节点</b>：中间节点的电压就是 ML 里的"隐层激活值"。它不被任何人直接设定——是电流平衡自己找到的值。"> + <circle cx="245" cy="295" r="22" fill="white" stroke="#13294B" stroke-width="2.5"/> + <text x="245" y="301" text-anchor="middle" font-size="15" fill="#13294B" font-style="italic">h</text> + </g> + <g data-tip="<b>输出节点 ŷ</b>：FREE 网络稳定后，这里的电压就是网络的预测。用示波器可以读它，但读数不参与任何学习回路。"> + <circle cx="360" cy="295" r="22" fill="white" stroke="#13294B" stroke-width="2.5"/> + <text x="360" y="301" text-anchor="middle" font-size="15" fill="#13294B" font-style="italic">ŷ</text> + </g> + <!-- edges with MOSFET boxes --> + <g data-tip="<b>可调边（一个 MOSFET 晶体管）</b>：连接两个节点的"电阻"，导电能力由旁边绿色电容的电压控制——<b>这就是一个权重</b>。板上每条这样的边配一个孪生兄弟（右边网络里位置相同的边），两兄弟永远共享同一颗权重电容。"> + <line x1="140" y1="182" x2="228" y2="285" stroke="#13294B" stroke-width="2.5"/> + <rect x="168" y="216" width="30" height="18" fill="#dfe7f3" stroke="#13294B" stroke-width="1.5"/> + <line x1="140" y1="408" x2="228" y2="305" stroke="#13294B" stroke-width="2.5"/> + <rect x="168" y="342" width="30" height="18" fill="#dfe7f3" stroke="#13294B" stroke-width="1.5"/> + <line x1="267" y1="295" x2="338" y2="295" stroke="#13294B" stroke-width="2.5"/> + <rect x="287" y="286" width="30" height="18" fill="#dfe7f3" stroke="#13294B" stroke-width="1.5"/> + <line x1="139" y1="152" x2="341" y2="282" stroke="#13294B" stroke-width="2.5"/> + <rect x="225" y="196" width="30" height="18" fill="#dfe7f3" stroke="#13294B" stroke-width="1.5"/> + </g> + + <!-- ================== NUDGED replica ================== --> + <text x="875" y="46" text-anchor="middle" font-size="21" font-weight="bold" fill="#E84A27">NUDGED 网络（微推态）</text> + <text x="875" y="70" text-anchor="middle" font-size="13" fill="#888">同样的电路 + 输出被轻拉向目标 y*</text> + <g data-tip="<b>NUDGED 网络</b>：FREE 的物理复制品（同样的布线、配对的晶体管），唯一区别：输出端多了一个小电流，把它往正确答案方向轻推。整个网络因此微微变形——每条边感受到的电压差都和 FREE 侧略有不同，<b>这个差异里藏着梯度</b>。"> + <rect x="700" y="85" width="350" height="420" rx="14" fill="#fff6f2" stroke="#E84A27" stroke-width="2"/> + </g> + <g> + <circle cx="760" cy="170" r="22" fill="white" stroke="#E84A27" stroke-width="2.5"/> + <text x="760" y="176" text-anchor="middle" font-size="15" fill="#E84A27" font-style="italic">x₁</text> + <circle cx="760" cy="420" r="22" fill="white" stroke="#E84A27" stroke-width="2.5"/> + <text x="760" y="426" text-anchor="middle" font-size="15" fill="#E84A27" font-style="italic">x₂</text> + <circle cx="885" cy="295" r="22" fill="white" stroke="#E84A27" stroke-width="2.5"/> + <text x="885" y="301" text-anchor="middle" font-size="15" fill="#E84A27" font-style="italic">h</text> + </g> + <g data-tip="<b>被推的输出 ŷβ</b>：因为 OTA 的小电流，这里的电压介于"网络自己的回答"和"正确答案"之间——偏向正确答案一点点（推力强度 = β）。"> + <circle cx="1000" cy="295" r="22" fill="white" stroke="#E84A27" stroke-width="3"/> + <text x="1000" y="301" text-anchor="middle" font-size="14" fill="#E84A27" font-style="italic">ŷβ</text> + </g> + <g data-tip="<b>可调边（孪生 MOSFET）</b>：和左边网络位置对应的边共享同一颗权重电容——所以两份网络的"权重"永远相同，差别只来自输出端的推力。"> + <line x1="780" y1="182" x2="868" y2="285" stroke="#E84A27" stroke-width="2.5"/> + <rect x="808" y="216" width="30" height="18" fill="#fde4da" stroke="#E84A27" stroke-width="1.5"/> + <line x1="780" y1="408" x2="868" y2="305" stroke="#E84A27" stroke-width="2.5"/> + <rect x="808" y="342" width="30" height="18" fill="#fde4da" stroke="#E84A27" stroke-width="1.5"/> + <line x1="907" y1="295" x2="978" y2="295" stroke="#E84A27" stroke-width="2.5"/> + <rect x="927" y="286" width="30" height="18" fill="#fde4da" stroke="#E84A27" stroke-width="1.5"/> + <line x1="779" y1="152" x2="981" y2="282" stroke="#E84A27" stroke-width="2.5"/> + <rect x="865" y="196" width="30" height="18" fill="#fde4da" stroke="#E84A27" stroke-width="1.5"/> + </g> + <!-- target + OTA --> + <g data-tip="<b>目标 y*</b>：正确答案，也用一个电压表示。只有 NUDGED 网络"知道"它——通过 OTA 的推力间接感受到。"> + <circle cx="1130" cy="295" r="22" fill="#eaf7ef" stroke="#2E8B57" stroke-width="2.5"/> + <text x="1130" y="301" text-anchor="middle" font-size="15" fill="#2E8B57" font-style="italic">y*</text> + </g> + <g data-tip="<b>OTA（跨导放大器，约 $1.7 一颗）</b>：持续测量"输出与目标的差距"，按差距大小注入一股小电流把输出往目标推。推力系数就是 EP 里的 β。它相当于告诉网络："往这边一点点"——但从不告诉任何一条边该怎么改。"> + <path d="M 1060,272 L 1060,318 L 1096,295 Z" fill="#eaf7ef" stroke="#2E8B57" stroke-width="2"/> + <line x1="1022" y1="295" x2="1058" y2="295" stroke="#2E8B57" stroke-width="2" marker-end="url(#arr)"/> + <text x="1078" y="345" text-anchor="middle" font-size="12.5" fill="#2E8B57">电流微推 (β)</text> + </g> + + <!-- ================== weight caps + contrast cells ================== --> + <text x="555" y="115" text-anchor="middle" font-size="15" font-weight="bold" fill="#2E8B57">共享权重 + 学习单元（每条边一套）</text> + <g data-tip="<b>对比单元 + 权重电容（紫框）</b>：每条边配一套。它同时量到这条边在 FREE 侧的电压差 Δv 和在 NUDGED 侧的电压差 Δvβ，比较两者（精确版算 Δvβ²−Δv²，便宜版只取正负号），把成比例的电流灌进绿色电容。<b>电容电压 = 权重</b>，它同时控制左右两个孪生 MOSFET 的导电性。这就是一次"梯度更新"——纯模拟、纯局部、连续发生。"> + <g id="cells"></g> + </g> + + <!-- learn/freeze switch --> + <g data-tip="<b>学习/冻结开关（机械拨杆）</b>：断开所有电容的充电电流 = 冻结权重（相当于 model.eval()）。故意用机械开关而不是芯片——保住"学习回路里没有任何数字器件"的主张。"> + <rect x="505" y="530" width="100" height="34" rx="8" fill="#eef1f7" stroke="#555" stroke-width="1.5"/> + <text x="555" y="552" text-anchor="middle" font-size="12.5" fill="#333">learn / freeze</text> + </g> + <!-- scope --> + <g data-tip="<b>示波器（只看不摸）</b>：观察节点电压用，不在任何反馈或学习回路里。这是"完全模拟"主张允许的唯一仪器。"> + <rect x="1075" y="500" width="90" height="60" rx="8" fill="white" stroke="#888" stroke-width="1.5"/> + <path d="M 1085,540 q 12,-24 24,0 t 24,0 t 24,0" fill="none" stroke="#2E8B57" stroke-width="2"/> + <text x="1120" y="516" text-anchor="middle" font-size="11" fill="#888">示波器</text> + </g> +</svg> +<p class="hint">悬停任何元素查看解释 · 紫框 = 学习单元（对比 + 电容）· 绿 = 权重/目标相关 · 蓝 = FREE · 橙 = NUDGED</p> + +<h2>ML ↔ 电路 词典</h2> +<table> + <tr><th>机器学习里的</th><th>板子上的</th><th>备注</th></tr> + <tr><td>激活值 z</td><td>节点电压</td><td>没人"计算"它——电流平衡自己找到它</td></tr> + <tr><td>权重 w</td><td>一颗电容的电压（控制 MOSFET 导电性）</td><td>非易失升级版 = FeFET（Zhu 组的方向）</td></tr> + <tr><td>前向传播</td><td>RC 自然弛豫（~微秒）</td><td>免费，且天然并行</td></tr> + <tr><td>损失的输出梯度</td><td>OTA 电流 = g·(ŷ − y*)</td><td>只在输出端，任何学习规则都需要这个</td></tr> + <tr><td>反向传播</td><td><b>不存在</b></td><td>被"双生网络 + 局部对比"替代</td></tr> + <tr><td>每个权重的梯度</td><td>该边两侧压降的平方差 Δvβ² − Δv²</td><td>纯局部量——只需要摸自己这条边</td></tr> + <tr><td>optimizer.step()</td><td>电容充/放电（~毫秒-秒）</td><td>学习率 = 充电电流大小</td></tr> + <tr><td>训练循环 / 时钟</td><td><b>不存在</b></td><td>两态同时物理存在 + 快慢时间尺度分离</td></tr> +</table> + +<h2>一次"学习"是怎么发生的（其实是连续的，这里人为分步）</h2> +<ol class="steps"> + <li><b>摆题：</b>手动开关把输入 x 设到两份网络上，把目标 y* 设到 OTA 上。</li> + <li><b>瞬间稳定（微秒级）：</b>两份网络各自弛豫。FREE 的输出是网络自己的回答；NUDGED 的输出被 OTA 拉向正确答案一点点。</li> + <li><b>形变传播：</b>输出端被拉动后，NUDGED 网络<b>全身</b>的电压分布都微微变形——离输出越远变形越小，但每条边都感受得到。这就是"credit assignment 由物理完成"。</li> + <li><b>逐边对比（连续进行）：</b>每条边的学习单元比较自己在两侧的压降，差异越大充电越快。方向恰好是能量下降的方向——数学上等价于梯度下降。</li> + <li><b>慢速累积（毫秒-秒级）：</b>权重电容比节点慢 100–1000 倍，所以在权重看来网络永远是"稳定的"——<b>这个时间尺度差就是时钟的替代品</b>。</li> + <li><b>换题：</b>手动换下一组输入/目标。重复。loss 就这样降下去（Penn 的 32 边版本已经在物理上学会了 XOR 和非线性回归）。</li> +</ol> + +<h2>造价阶梯</h2> +<p> +<span class="chip">① 单边标定件 <b>$70–130</b>（先证明物理）</span> +<span class="chip">② 8 边网络 <b>$170–300</b> ← 推荐 MVP</span> +<span class="chip">③ 32 边 <b>$450–900</b>（复现 Penn 规模）</span> +<span class="chip">④ 注意力微单元 <b>+$100–250</b></span> +<span class="chip">⑤ CIM transformer block（合作方阶段）</span> +</p> + +<div class="note"> +<b>诚实边界</b>（也是给合作者的说法）：这块板子验证的是<b>物理学习原语</b>——无时钟双生 EP、局部模拟更新、真电流 nudge vs 电压钳制的对比—— +<b>不是</b> transformer 本身。它和我们 GPU 侧共享同一条定律：nudge 幅度 vs 噪声的信噪比曲线（我们在 fp32 里测过的 β 定律，这块板会在物理里再测一遍）。 +</div> + +</div> +<div id="tip"></div> +<script> +// draw 4 contrast-cell/cap pairs + dashed pairing lines +const svg = document.querySelector('svg'); +const cells = document.getElementById('cells'); +const pairs = [ + {y: 205, fL: [183, 225], fR: [808, 225]}, + {y: 285, fL: [240, 205], fR: [880, 205]}, + {y: 365, fL: [302, 295], fR: [942, 295]}, + {y: 445, fL: [183, 351], fR: [808, 351]}, +]; +let inner = ''; +pairs.forEach((p, i) => { + inner += ` + <rect x="480" y="${p.y-32}" width="150" height="64" rx="10" fill="#faf7ff" stroke="#7B4FB6" stroke-width="2"/> + <text x="555" y="${p.y-12}" text-anchor="middle" font-size="11.5" fill="#7B4FB6">对比: Δv²β − Δv²</text> + <line x1="543" y1="${p.y+2}" x2="543" y2="${p.y+20}" stroke="#2E8B57" stroke-width="2"/> + <line x1="537" y1="${p.y+6}" x2="549" y2="${p.y+6}" stroke="#2E8B57" stroke-width="3"/> + <line x1="537" y1="${p.y+13}" x2="549" y2="${p.y+13}" stroke="#2E8B57" stroke-width="3"/> + <text x="585" y="${p.y+16}" text-anchor="middle" font-size="11" fill="#2E8B57">C_w${i+1}</text> + <line x1="${p.fL[0]+15}" y1="${p.fL[1]+9}" x2="480" y2="${p.y}" stroke="#7B4FB6" stroke-width="1.3" stroke-dasharray="5,4" opacity="0.75"/> + <line x1="${p.fR[0]+15}" y1="${p.fR[1]+9}" x2="630" y2="${p.y}" stroke="#7B4FB6" stroke-width="1.3" stroke-dasharray="5,4" opacity="0.75"/>`; +}); +cells.innerHTML = inner; + +const tip = document.getElementById('tip'); +document.querySelectorAll('[data-tip]').forEach(el => { + el.addEventListener('mousemove', e => { + tip.innerHTML = el.getAttribute('data-tip'); + tip.style.display = 'block'; + tip.style.left = Math.min(e.clientX + 16, window.innerWidth - 360) + 'px'; + tip.style.top = (e.clientY + 18) + 'px'; + }); + el.addEventListener('mouseleave', () => tip.style.display = 'none'); +}); +</script> +</body> +</html> diff --git a/assets/hw/hw_outreach_pack.zip b/assets/hw/hw_outreach_pack.zip Binary files differnew file mode 100644 index 0000000..4737d6c --- /dev/null +++ b/assets/hw/hw_outreach_pack.zip diff --git a/assets/ept_method_intro.tex b/assets/misc/ept_method_intro.tex index 31d458d..31d458d 100644 --- a/assets/ept_method_intro.tex +++ b/assets/misc/ept_method_intro.tex diff --git a/assets/misc/fw72m_api_proof.png b/assets/misc/fw72m_api_proof.png Binary files differnew file mode 100644 index 0000000..e3f594b --- /dev/null +++ b/assets/misc/fw72m_api_proof.png |
