diff options
Diffstat (limited to 'assets/figs')
| -rw-r--r-- | assets/figs/fig_ep_phases.png | bin | 0 -> 159757 bytes | |||
| -rw-r--r-- | assets/figs/fig_p3_schedule.png | bin | 0 -> 148284 bytes | |||
| -rw-r--r-- | assets/figs/fig_p4_blowup.png | bin | 0 -> 171798 bytes | |||
| -rw-r--r-- | assets/figs/fig_p5_loop.png | bin | 0 -> 130618 bytes | |||
| -rw-r--r-- | assets/figs/fig_p6_window.png | bin | 0 -> 160498 bytes | |||
| -rw-r--r-- | assets/figs/fig_primer_arch.emf | bin | 0 -> 539020 bytes | |||
| -rw-r--r-- | assets/figs/fig_primer_arch.png | bin | 0 -> 389335 bytes | |||
| -rw-r--r-- | assets/figs/fig_primer_arch.svg | 1327 | ||||
| -rw-r--r-- | assets/figs/fig_primer_block.emf | bin | 0 -> 826816 bytes | |||
| -rw-r--r-- | assets/figs/fig_primer_block.png | bin | 0 -> 266630 bytes | |||
| -rw-r--r-- | assets/figs/fig_primer_block.svg | 1956 | ||||
| -rw-r--r-- | assets/figs/fig_primer_phases.emf | bin | 0 -> 358368 bytes | |||
| -rw-r--r-- | assets/figs/fig_primer_phases.png | bin | 0 -> 204700 bytes | |||
| -rw-r--r-- | assets/figs/fig_primer_phases.svg | 925 | ||||
| -rw-r--r-- | assets/figs/frozen_vs_adaptive.png | bin | 0 -> 217748 bytes |
15 files changed, 4208 insertions, 0 deletions
diff --git a/assets/figs/fig_ep_phases.png b/assets/figs/fig_ep_phases.png Binary files differnew file mode 100644 index 0000000..e787f58 --- /dev/null +++ b/assets/figs/fig_ep_phases.png diff --git a/assets/figs/fig_p3_schedule.png b/assets/figs/fig_p3_schedule.png Binary files differnew file mode 100644 index 0000000..c4ad8b3 --- /dev/null +++ b/assets/figs/fig_p3_schedule.png diff --git a/assets/figs/fig_p4_blowup.png b/assets/figs/fig_p4_blowup.png Binary files differnew file mode 100644 index 0000000..53b9d3c --- /dev/null +++ b/assets/figs/fig_p4_blowup.png diff --git a/assets/figs/fig_p5_loop.png b/assets/figs/fig_p5_loop.png Binary files differnew file mode 100644 index 0000000..18964ad --- /dev/null +++ b/assets/figs/fig_p5_loop.png diff --git a/assets/figs/fig_p6_window.png b/assets/figs/fig_p6_window.png Binary files differnew file mode 100644 index 0000000..9519e39 --- /dev/null +++ b/assets/figs/fig_p6_window.png diff --git a/assets/figs/fig_primer_arch.emf b/assets/figs/fig_primer_arch.emf Binary files differnew file mode 100644 index 0000000..87ba265 --- /dev/null +++ b/assets/figs/fig_primer_arch.emf diff --git a/assets/figs/fig_primer_arch.png b/assets/figs/fig_primer_arch.png Binary files differnew file mode 100644 index 0000000..a7e7adf --- /dev/null +++ b/assets/figs/fig_primer_arch.png diff --git a/assets/figs/fig_primer_arch.svg b/assets/figs/fig_primer_arch.svg new file mode 100644 index 0000000..24e64f1 --- /dev/null +++ b/assets/figs/fig_primer_arch.svg @@ -0,0 +1,1327 @@ +<?xml version="1.0" encoding="utf-8" standalone="no"?> +<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" + "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"> +<svg xmlns:xlink="http://www.w3.org/1999/xlink" width="950.4pt" height="720pt" viewBox="0 0 950.4 720" xmlns="http://www.w3.org/2000/svg" version="1.1"> + <metadata> + <rdf:RDF xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:cc="http://creativecommons.org/ns#" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"> + <cc:Work> + <dc:type rdf:resource="http://purl.org/dc/dcmitype/StillImage"/> + <dc:date>2026-07-15T09:29:43.786622</dc:date> + <dc:format>image/svg+xml</dc:format> + <dc:creator> + <cc:Agent> + <dc:title>Matplotlib v3.10.8, https://matplotlib.org/</dc:title> + </cc:Agent> + </dc:creator> + </cc:Work> + </rdf:RDF> + </metadata> + <defs> + <style type="text/css">*{stroke-linejoin: round; stroke-linecap: butt}</style> + </defs> + <g id="figure_1"> + <g id="patch_1"> + <path d="M 0 720 +L 950.4 720 +L 950.4 0 +L 0 0 +z +" style="fill: #ffffff"/> + </g> + <g id="axes_1"> + <g id="patch_2"> + <path d="M 41.3424 141.6 +L 224.7696 141.6 +Q 230.472 141.6 230.472 137.485714 +L 230.472 114.857143 +Q 230.472 110.742857 224.7696 110.742857 +L 41.3424 110.742857 +Q 35.64 110.742857 35.64 114.857143 +L 35.64 137.485714 +Q 35.64 141.6 41.3424 141.6 +z +" clip-path="url(#p7fae8feff8)" style="fill: #f4f4f4; stroke: #9a9a9a; stroke-linejoin: miter"/> + </g> + <g id="patch_3"> + <path d="M 41.3424 193.714286 +L 224.7696 193.714286 +Q 230.472 193.714286 230.472 189.6 +L 230.472 166.971429 +Q 230.472 162.857143 224.7696 162.857143 +L 41.3424 162.857143 +Q 35.64 162.857143 35.64 166.971429 +L 35.64 189.6 +Q 35.64 193.714286 41.3424 193.714286 +z +" clip-path="url(#p7fae8feff8)" style="fill: #f4f4f4; stroke: #9a9a9a; stroke-linejoin: miter"/> + </g> + <g id="patch_4"> + <path d="M 17.5824 358.285714 +L 248.5296 358.285714 +Q 254.232 358.285714 254.232 354.171429 +L 254.232 231.428571 +Q 254.232 227.314286 248.5296 227.314286 +L 17.5824 227.314286 +Q 11.88 227.314286 11.88 231.428571 +L 11.88 354.171429 +Q 11.88 358.285714 17.5824 358.285714 +z +" clip-path="url(#p7fae8feff8)" style="fill: #eeeeee; stroke: #777777; stroke-width: 1.3; stroke-linejoin: miter"/> + </g> + <g id="patch_5"> + <path d="M 41.3424 420 +L 224.7696 420 +Q 230.472 420 230.472 415.885714 +L 230.472 393.257143 +Q 230.472 389.142857 224.7696 389.142857 +L 41.3424 389.142857 +Q 35.64 389.142857 35.64 393.257143 +L 35.64 415.885714 +Q 35.64 420 41.3424 420 +z +" clip-path="url(#p7fae8feff8)" style="fill: #f4f4f4; stroke: #9a9a9a; stroke-linejoin: miter"/> + </g> + <g id="patch_6"> + <path d="M 302.7024 145.714286 +L 476.6256 145.714286 +Q 482.328 145.714286 482.328 141.6 +L 482.328 118.971429 +Q 482.328 114.857143 476.6256 114.857143 +L 302.7024 114.857143 +Q 297 114.857143 297 118.971429 +L 297 141.6 +Q 297 145.714286 302.7024 145.714286 +z +" clip-path="url(#p7fae8feff8)" style="fill: #eaf1fa; stroke: #2c6fbb; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_7"> + <path d="M 302.7024 228 +L 476.6256 228 +Q 482.328 228 482.328 223.885714 +L 482.328 201.257143 +Q 482.328 197.142857 476.6256 197.142857 +L 302.7024 197.142857 +Q 297 197.142857 297 201.257143 +L 297 223.885714 +Q 297 228 302.7024 228 +z +" clip-path="url(#p7fae8feff8)" style="fill: #eaf1fa; stroke: #2c6fbb; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_8"> + <path d="M 302.7024 310.285714 +L 476.6256 310.285714 +Q 482.328 310.285714 482.328 306.171429 +L 482.328 283.542857 +Q 482.328 279.428571 476.6256 279.428571 +L 302.7024 279.428571 +Q 297 279.428571 297 283.542857 +L 297 306.171429 +Q 297 310.285714 302.7024 310.285714 +z +" clip-path="url(#p7fae8feff8)" style="fill: #eaf1fa; stroke: #2c6fbb; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_9"> + <path d="M 625.8384 145.714286 +L 799.7616 145.714286 +Q 805.464 145.714286 805.464 141.6 +L 805.464 118.971429 +Q 805.464 114.857143 799.7616 114.857143 +L 625.8384 114.857143 +Q 620.136 114.857143 620.136 118.971429 +L 620.136 141.6 +Q 620.136 145.714286 625.8384 145.714286 +z +" clip-path="url(#p7fae8feff8)" style="fill: #fdeee2; stroke: #d95f02; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_10"> + <path d="M 625.8384 228 +L 799.7616 228 +Q 805.464 228 805.464 223.885714 +L 805.464 201.257143 +Q 805.464 197.142857 799.7616 197.142857 +L 625.8384 197.142857 +Q 620.136 197.142857 620.136 201.257143 +L 620.136 223.885714 +Q 620.136 228 625.8384 228 +z +" clip-path="url(#p7fae8feff8)" style="fill: #fdeee2; stroke: #d95f02; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_11"> + <path d="M 625.8384 310.285714 +L 799.7616 310.285714 +Q 805.464 310.285714 805.464 306.171429 +L 805.464 283.542857 +Q 805.464 279.428571 799.7616 279.428571 +L 625.8384 279.428571 +Q 620.136 279.428571 620.136 283.542857 +L 620.136 306.171429 +Q 620.136 310.285714 625.8384 310.285714 +z +" clip-path="url(#p7fae8feff8)" style="fill: #fdeee2; stroke: #d95f02; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_12"> + <path d="M 22.3344 581.142857 +L 928.0656 581.142857 +Q 933.768 581.142857 933.768 577.028571 +L 933.768 489.257143 +Q 933.768 485.142857 928.0656 485.142857 +L 22.3344 485.142857 +Q 16.632 485.142857 16.632 489.257143 +L 16.632 577.028571 +Q 16.632 581.142857 22.3344 581.142857 +z +" clip-path="url(#p7fae8feff8)" style="fill: #faf6ee; stroke: #c9b895; stroke-width: 1.2; stroke-linejoin: miter"/> + </g> + <g id="patch_13"> + <path d="M 22.3344 708 +L 928.0656 708 +Q 933.768 708 933.768 703.885714 +L 933.768 597.6 +Q 933.768 593.485714 928.0656 593.485714 +L 22.3344 593.485714 +Q 16.632 593.485714 16.632 597.6 +L 16.632 703.885714 +Q 16.632 708 22.3344 708 +z +" clip-path="url(#p7fae8feff8)" style="fill: #f7f7f7; stroke: #9a9a9a; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="text_1"> + <text style="font-weight: 700; font-size: 10.5px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="152.064" y="21.370614" transform="rotate(-0 152.064 21.370614)">(A) The model — a stock OLMo2-style decoder</text> + </g> + <g id="text_2"> + <text style="font-size: 8.5px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #8a8a8a" x="152.064" y="39.37404" transform="rotate(-0 152.064 39.37404)">Inference is one ordinary forward pass.</text> + </g> + <g id="text_3"> + <text style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="133.056" y="81.44144" transform="rotate(-0 133.056 81.44144)">logits</text> + </g> + <g id="patch_14"> + <path d="M 133.056 111.95625 +Q 133.056 101.485232 133.056 92.355855 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.2; stroke-linecap: round"/> + <path d="M 130.856 96.755855 +L 133.056 92.355855 +L 135.256 96.755855 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.2; stroke-linecap: round"/> + </g> + <g id="text_4"> + <!-- linear readout $W_{\mathrm{out}}$ (untied) --> + <g style="fill: #3a3a3a" transform="translate(72.9185 128.516897)"> + <text> + <tspan x="0" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="2.361572" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="4.723145" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="10.110352" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="15.339844" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="20.548584" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="24.043213" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="26.745117" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="30.239746" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="35.469238" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="40.677979" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="46.073486" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="51.273926" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="56.661133" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="59.993896" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="81.090723" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="83.792627" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="86.494531" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="89.810693" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="95.1979" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="100.585107" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="103.917871" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="106.279443" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="111.508936" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="116.904443" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="62.695801" y="-0.15625" style="font-style: oblique; font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">W</tspan> + <tspan x="71.100342" y="1.320312" style="font-size: 5.95px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="74.740649" y="1.320312" style="font-size: 5.95px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="78.511694" y="1.320312" style="font-size: 5.95px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + </text> + </g> + </g> + <g id="patch_15"> + <path d="M 133.056 164.070536 +Q 133.056 153.599518 133.056 144.470141 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.2; stroke-linecap: round"/> + <path d="M 130.856 148.870141 +L 133.056 144.470141 +L 135.256 148.870141 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.2; stroke-linecap: round"/> + </g> + <g id="text_5"> + <text style="font-size: 8.5px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="133.056" y="180.631183" transform="rotate(-0 133.056 180.631183)">final RMSNorm</text> + </g> + <g id="patch_16"> + <path d="M 133.056 228.530525 +Q 133.056 211.886221 133.056 196.583559 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.2; stroke-linecap: round"/> + <path d="M 130.856 200.983559 +L 133.056 196.583559 +L 135.256 200.983559 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.2; stroke-linecap: round"/> + </g> + <g id="text_6"> + <!-- transformer block ($=f_l$) $\times\,12$ --> + <g style="fill: #3a3a3a" transform="translate(57.141 253.454866)"> + <text> + <tspan x="0" y="-0.15625" style="font-weight: 700; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="4.302246" y="-0.15625" style="font-weight: 700; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="8.740723" y="-0.15625" style="font-weight: 700; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="14.813965" y="-0.15625" style="font-weight: 700; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="21.221191" y="-0.15625" style="font-weight: 700; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="26.578125" y="-0.15625" style="font-weight: 700; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="30.493652" y="-0.15625" style="font-weight: 700; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="36.676758" y="-0.15625" style="font-weight: 700; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="41.115234" y="-0.15625" style="font-weight: 700; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="50.493164" y="-0.15625" style="font-weight: 700; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="56.597168" y="-0.15625" style="font-weight: 700; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="61.035645" y="-0.15625" style="font-weight: 700; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="64.168945" y="-0.15625" style="font-weight: 700; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">b</tspan> + <tspan x="70.611328" y="-0.15625" style="font-weight: 700; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="73.696289" y="-0.15625" style="font-weight: 700; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="79.879395" y="-0.15625" style="font-weight: 700; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="85.214355" y="-0.15625" style="font-weight: 700; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">k</tspan> + <tspan x="91.199707" y="-0.15625" style="font-weight: 700; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="94.333008" y="-0.15625" style="font-weight: 700; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="97.466309" y="-0.15625" style="font-weight: 700; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="117.792334" y="-0.15625" style="font-weight: 700; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="121.905615" y="-0.15625" style="font-weight: 700; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="125.038916" y="-0.15625" style="font-weight: 700; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="128.172217" y="-0.15625" style="font-weight: 700; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="103.333008" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="131.305518" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">×</tspan> + <tspan x="140.307744" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">1</tspan> + <tspan x="146.033818" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">2</tspan> + <tspan x="112.627441" y="-0.15625" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="115.795898" y="1.320312" style="font-style: oblique; font-size: 6.3px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + </text> + </g> + </g> + <g id="text_7"> + <!-- $h\ =\ z + \mathrm{RMSNorm}(\,\mathrm{Attn}(z)\,)$ --> + <g style="fill: #3a3a3a" transform="translate(71.316 284.312009)"> + <text> + <tspan x="0" y="-0.15625" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="22.596621" y="-0.15625" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="110.208223" y="-0.15625" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="10.379854" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="29.07416" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">+</tspan> + <tspan x="38.368594" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">R</tspan> + <tspan x="44.622012" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">M</tspan> + <tspan x="52.387149" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">S</tspan> + <tspan x="58.100039" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">N</tspan> + <tspan x="64.832461" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="70.338809" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="74.039004" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="82.806094" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="87.778535" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">A</tspan> + <tspan x="93.935274" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="97.464082" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="100.992891" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="106.696992" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="114.932344" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="119.904785" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + </text> + </g> + </g> + <g id="text_8"> + <!-- $z'\ =\ h + \mathrm{RMSNorm}(\,\mathrm{SwiGLU}(h)\,)$ --> + <g style="fill: #3a3a3a" transform="translate(62.046 315.169152)"> + <text> + <tspan x="0" y="-0.032813" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="23.996572" y="-0.032813" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="127.8196" y="-0.032813" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="5.126074" y="-3.478125" style="font-size: 6.3px; font-family: 'cmsy10'; fill: #3a3a3a">0</tspan> + <tspan x="11.779805" y="-0.032813" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="31.454092" y="-0.032813" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">+</tspan> + <tspan x="40.748526" y="-0.032813" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">R</tspan> + <tspan x="47.001944" y="-0.032813" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">M</tspan> + <tspan x="54.76708" y="-0.032813" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">S</tspan> + <tspan x="60.479971" y="-0.032813" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">N</tspan> + <tspan x="67.212393" y="-0.032813" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="72.71874" y="-0.032813" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="76.418936" y="-0.032813" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="85.186026" y="-0.032813" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="90.158467" y="-0.032813" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">S</tspan> + <tspan x="95.871357" y="-0.032813" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">w</tspan> + <tspan x="103.232197" y="-0.032813" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="105.732686" y="-0.032813" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">G</tspan> + <tspan x="112.706807" y="-0.032813" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">L</tspan> + <tspan x="117.720967" y="-0.032813" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">U</tspan> + <tspan x="124.308369" y="-0.032813" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="133.523701" y="-0.032813" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="138.496143" y="-0.032813" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + </text> + </g> + </g> + <g id="text_9"> + <text style="font-size: 7.5px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #8a8a8a" x="133.056" y="342.869531" transform="rotate(-0 133.056 342.869531)">causal SDPA · QK-RMSNorm · RoPE</text> + </g> + <g id="patch_17"> + <path d="M 133.056 390.360435 +Q 133.056 375.085593 133.056 361.152392 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.2; stroke-linecap: round"/> + <path d="M 130.856 365.552392 +L 133.056 361.152392 +L 135.256 365.552392 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.2; stroke-linecap: round"/> + </g> + <g id="text_10"> + <text style="font-size: 8.5px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="133.056" y="406.916897" transform="rotate(-0 133.056 406.916897)">token embedding</text> + </g> + <g id="text_11"> + <text style="font-size: 7.8px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #8a8a8a" x="152.064" y="447.866598" transform="rotate(-0 152.064 447.866598)">42.75M (TinyStories 4k BPE) / 72.11M (FineWeb-Edu 32k BPE)</text> + </g> + <g id="text_12"> + <text style="font-size: 8.2px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; fill: #3a3a3a" transform="translate(11.705625 466.700156)">Nothing in this graph is modified for training: the trained checkpoint</text> + <text style="font-size: 8.2px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; fill: #3a3a3a" transform="translate(19.486656 475.882362)">is indistinguishable in form from a conventionally trained model.</text> + </g> + <g id="text_13"> + <text style="font-weight: 700; font-size: 10.5px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="446.688" y="21.411629" transform="rotate(-0 446.688 21.411629)">(B) Free phase (= inference)</text> + </g> + <g id="text_14"> + <!-- give each layer a state $z_l$ (3 of $L$ drawn); settle the disagreement energy --> + <g style="fill: #8a8a8a" transform="translate(292.2005 39.366071)"> + <text> + <tspan x="0" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">g</tspan> + <tspan x="5.395508" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="7.75708" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">v</tspan> + <tspan x="12.787354" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="18.016846" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="20.71875" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="25.948242" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="31.156982" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">c</tspan> + <tspan x="35.830322" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">h</tspan> + <tspan x="41.217529" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="43.919434" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">l</tspan> + <tspan x="46.281006" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="51.489746" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">y</tspan> + <tspan x="56.52002" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="61.749512" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="65.244141" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="67.946045" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="73.154785" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="75.856689" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="80.285156" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="83.61792" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="88.82666" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="92.159424" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="97.388916" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="106.451685" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="109.153589" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">(</tspan> + <tspan x="112.469751" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">3</tspan> + <tspan x="117.87771" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="120.579614" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">o</tspan> + <tspan x="125.780054" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">f</tspan> + <tspan x="128.772485" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="136.209985" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="138.91189" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">d</tspan> + <tspan x="144.307397" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="147.802026" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="153.010767" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">w</tspan> + <tspan x="159.962671" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">n</tspan> + <tspan x="165.349878" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">)</tspan> + <tspan x="168.66604" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">;</tspan> + <tspan x="171.52981" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="174.231714" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="178.660181" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="183.889673" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="187.222437" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="190.5552" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">l</tspan> + <tspan x="192.916772" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="198.146265" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="200.848169" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="204.180933" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">h</tspan> + <tspan x="209.56814" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="214.797632" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="217.499536" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">d</tspan> + <tspan x="222.895044" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="225.256616" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="229.685083" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="234.893823" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">g</tspan> + <tspan x="240.289331" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="243.78396" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="249.013452" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="254.242944" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">m</tspan> + <tspan x="262.522974" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="267.752466" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">n</tspan> + <tspan x="273.139673" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="276.472437" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="279.174341" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="284.403833" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">n</tspan> + <tspan x="289.79104" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="295.020532" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="298.515161" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">g</tspan> + <tspan x="303.910669" y="-0.15625" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">y</tspan> + <tspan x="100.09082" y="-0.15625" style="font-style: oblique; font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">z</tspan> + <tspan x="131.47439" y="-0.15625" style="font-style: oblique; font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">L</tspan> + <tspan x="104.55249" y="1.320312" style="font-style: oblique; font-size: 5.95px; font-family: 'DejaVu Sans'; fill: #8a8a8a">l</tspan> + </text> + </g> + </g> + <g id="text_15"> + <!-- $E(z)\ =\ \sum_l\ \frac{1}{2}\,\|\,z_l - f_l(z_{l-1})\,\|^2$ --> + <g style="fill: #3a3a3a" transform="translate(378.5955 62.505)"> + <text> + <tspan x="0" y="-0.8125" style="font-style: oblique; font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">E</tspan> + <tspan x="10.730713" y="-0.8125" style="font-style: oblique; font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="67.635488" y="-0.8125" style="font-style: oblique; font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="88.37897" y="-0.8125" style="font-style: oblique; font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="98.514785" y="-0.8125" style="font-style: oblique; font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="6.634277" y="-0.8125" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="16.242188" y="-0.8125" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="25.793667" y="-0.8125" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="60.680742" y="-0.8125" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">‖</tspan> + <tspan x="77.535464" y="-0.8125" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">−</tspan> + <tspan x="94.41835" y="-0.8125" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="120.067813" y="-0.8125" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="125.868994" y="-0.8125" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">‖</tspan> + <tspan x="40.046563" y="-0.8125" style="font-size: 10.5px; font-family: 'DejaVu Sans Display'; fill: #3a3a3a">∑</tspan> + <tspan x="44.046563" y="9.992188" style="font-style: oblique; font-size: 7.35px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="73.146963" y="0.992188" style="font-style: oblique; font-size: 7.35px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="92.075503" y="0.992188" style="font-style: oblique; font-size: 7.35px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="104.02626" y="0.992188" style="font-style: oblique; font-size: 7.35px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="52.987202" y="-5.570312" style="font-size: 7.35px; font-family: 'DejaVu Sans'; fill: #3a3a3a">1</tspan> + <tspan x="52.987202" y="3.43125" style="font-size: 7.35px; font-family: 'DejaVu Sans'; fill: #3a3a3a">2</tspan> + <tspan x="107.500283" y="0.992188" style="font-size: 7.35px; font-family: 'DejaVu Sans'; fill: #3a3a3a">−</tspan> + <tspan x="115.090737" y="0.992188" style="font-size: 7.35px; font-family: 'DejaVu Sans'; fill: #3a3a3a">1</tspan> + <tspan x="131.224268" y="-5.023438" style="font-size: 7.35px; font-family: 'DejaVu Sans'; fill: #3a3a3a">2</tspan> + </text> + <rect x="52.987202" y="-3.945312" width="4.676294" height="0.65625"/> + </g> + </g> + <g id="text_16"> + <!-- $z_3 = f_3(z_2)$ --> + <g style="fill: #3a3a3a" transform="translate(367.929 132.769152)"> + <text> + <tspan x="0" y="-0.15625" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="20.026318" y="-0.15625" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="30.960352" y="-0.15625" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="4.724121" y="1.320312" style="font-size: 6.3px; font-family: 'DejaVu Sans'; fill: #3a3a3a">3</tspan> + <tspan x="23.194775" y="1.320312" style="font-size: 6.3px; font-family: 'DejaVu Sans'; fill: #3a3a3a">3</tspan> + <tspan x="35.684473" y="1.320312" style="font-size: 6.3px; font-family: 'DejaVu Sans'; fill: #3a3a3a">2</tspan> + <tspan x="10.731885" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="27.449121" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="39.938818" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + </text> + </g> + </g> + <g id="text_17"> + <text style="font-size: 7.8px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: start; fill: #8a8a8a" x="494.208" y="132.438027" transform="rotate(-0 494.208 132.438027)">term = 0</text> + </g> + <g id="text_18"> + <!-- $z_2 = f_2(z_1)$ --> + <g style="fill: #3a3a3a" transform="translate(367.929 215.054866)"> + <text> + <tspan x="0" y="-0.15625" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="20.026318" y="-0.15625" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="30.960352" y="-0.15625" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="4.724121" y="1.320312" style="font-size: 6.3px; font-family: 'DejaVu Sans'; fill: #3a3a3a">2</tspan> + <tspan x="23.194775" y="1.320312" style="font-size: 6.3px; font-family: 'DejaVu Sans'; fill: #3a3a3a">2</tspan> + <tspan x="35.684473" y="1.320312" style="font-size: 6.3px; font-family: 'DejaVu Sans'; fill: #3a3a3a">1</tspan> + <tspan x="10.731885" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="27.449121" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="39.938818" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + </text> + </g> + </g> + <g id="text_19"> + <text style="font-size: 7.8px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: start; fill: #8a8a8a" x="494.208" y="214.723741" transform="rotate(-0 494.208 214.723741)">term = 0</text> + </g> + <g id="text_20"> + <!-- $z_1 = f_1(\mathrm{emb})$ --> + <g style="fill: #3a3a3a" transform="translate(362.394 297.34058)"> + <text> + <tspan x="0" y="-0.15625" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="20.026318" y="-0.15625" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="4.724121" y="1.320312" style="font-size: 6.3px; font-family: 'DejaVu Sans'; fill: #3a3a3a">1</tspan> + <tspan x="23.194775" y="1.320312" style="font-size: 6.3px; font-family: 'DejaVu Sans'; fill: #3a3a3a">1</tspan> + <tspan x="10.731885" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="27.449121" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="30.960352" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="36.497461" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="45.264551" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">b</tspan> + <tspan x="50.977441" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + </text> + </g> + </g> + <g id="text_21"> + <text style="font-size: 7.8px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: start; fill: #8a8a8a" x="494.208" y="297.009455" transform="rotate(-0 494.208 297.009455)">term = 0</text> + </g> + <g id="patch_18"> + <path d="M 389.664 194.928976 +Q 389.664 171.771648 389.664 149.844158 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.1; stroke-linecap: round"/> + <path d="M 387.464 154.244158 +L 389.664 149.844158 +L 391.864 154.244158 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.1; stroke-linecap: round"/> + </g> + <g id="patch_19"> + <path d="M 389.664 277.21469 +Q 389.664 254.057363 389.664 232.129872 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.1; stroke-linecap: round"/> + <path d="M 387.464 236.529872 +L 389.664 232.129872 +L 391.864 236.529872 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.1; stroke-linecap: round"/> + </g> + <g id="patch_20"> + <path d="M 389.664 351.955413 +Q 389.664 333.943917 389.664 317.162258 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.1; stroke-linecap: round"/> + <path d="M 387.464 321.562258 +L 389.664 317.162258 +L 391.864 321.562258 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.1; stroke-linecap: round"/> + </g> + <g id="text_22"> + <!-- $\mathrm{emb}(x)$ --> + <g style="fill: #3a3a3a" transform="translate(373.464 369.34058)"> + <text> + <tspan x="0" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="5.537109" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="14.304199" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">b</tspan> + <tspan x="20.01709" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="28.854492" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="23.52832" y="-0.15625" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">x</tspan> + </text> + </g> + </g> + <g id="text_23"> + <!-- The minimum is exact: $E=0$, states $\equiv$ forward activations, --> + <g style="fill: #3a3a3a" transform="translate(318.924 416.489304)"> + <text> + <tspan x="0" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">T</tspan> + <tspan x="5.115784" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="10.423767" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="15.576355" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="18.238525" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="26.39679" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="28.723633" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="34.031616" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="36.358459" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="44.516724" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="49.824707" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="57.982971" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="60.645142" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="62.971985" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="67.335327" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="69.997498" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="75.150085" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">x</tspan> + <tspan x="80.106384" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="85.238525" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="89.84314" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="93.126892" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">:</tspan> + <tspan x="95.948547" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="98.610718" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="108.196167" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="116.845154" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">0</tspan> + <tspan x="122.173584" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">,</tspan> + <tspan x="124.835754" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="127.497925" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="130.160095" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="134.523438" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="137.80719" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="142.939331" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="146.223083" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="151.375671" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="155.739014" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="160.032837" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">≡</tspan> + <tspan x="168.681824" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="171.343994" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="174.292419" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="179.416382" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="182.859619" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">w</tspan> + <tspan x="189.70929" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="194.841431" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="198.284668" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="203.60083" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="206.263" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="211.395142" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="215.999756" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="219.283508" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="221.610352" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">v</tspan> + <tspan x="226.56665" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="231.698792" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="234.982544" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="237.309387" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="242.43335" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="247.741333" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="252.104675" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">,</tspan> + <tspan x="101.272888" y="-0.921875" style="font-style: oblique; font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">E</tspan> + </text> + </g> + </g> + <g id="text_24"> + <text style="font-size: 8.4px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="446.688" y="437.060732" transform="rotate(-0 446.688 437.060732)">and one bottom-up pass reaches it.</text> + </g> + <g id="text_25"> + <text style="font-size: 8.2px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #8a8a8a" x="446.688" y="464.434116" transform="rotate(-0 446.688 464.434116)">The free phase adds nothing and changes nothing at inference.</text> + </g> + <g id="text_26"> + <text style="font-weight: 700; font-size: 10.5px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="769.824" y="21.370614" transform="rotate(-0 769.824 21.370614)">(C) Nudged phase (training only)</text> + </g> + <g id="text_27"> + <!-- add the loss at strength $\beta \ll 1$ and settle again: --> + <g style="fill: #8a8a8a" transform="translate(668.844 39.408571)"> + <text> + <tspan x="0" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="5.20874" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">d</tspan> + <tspan x="10.604248" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">d</tspan> + <tspan x="15.999756" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="18.70166" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="22.034424" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">h</tspan> + <tspan x="27.421631" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="32.651123" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="35.353027" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">l</tspan> + <tspan x="37.7146" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">o</tspan> + <tspan x="42.915039" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="47.343506" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="51.771973" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="54.473877" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="59.682617" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="63.015381" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="65.717285" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="70.145752" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="73.478516" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="76.973145" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="82.202637" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">n</tspan> + <tspan x="87.589844" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">g</tspan> + <tspan x="92.985352" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="96.318115" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">h</tspan> + <tspan x="101.705322" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="111.487793" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">≪</tspan> + <tspan x="122.042236" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">1</tspan> + <tspan x="127.450195" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="130.1521" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="135.36084" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">n</tspan> + <tspan x="140.748047" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">d</tspan> + <tspan x="146.143555" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="148.845459" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="153.273926" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="158.503418" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="161.836182" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="165.168945" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">l</tspan> + <tspan x="167.530518" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="172.76001" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="175.461914" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="180.670654" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">g</tspan> + <tspan x="186.066162" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="191.274902" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="193.636475" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">n</tspan> + <tspan x="199.023682" y="-0.109375" style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">:</tspan> + <tspan x="104.407227" y="-0.109375" style="font-style: oblique; font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #8a8a8a">β</tspan> + </text> + </g> + </g> + <g id="text_28"> + <!-- $E(z)\ +\ \beta\cdot\mathrm{CE}(\mathrm{logits}(z_3),\,y)$ --> + <g style="fill: #3a3a3a" transform="translate(701.364 65.34)"> + <text> + <tspan x="0" y="-0.578125" style="font-style: oblique; font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">E</tspan> + <tspan x="10.730713" y="-0.578125" style="font-style: oblique; font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="40.046563" y="-0.578125" style="font-style: oblique; font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">β</tspan> + <tspan x="104.846123" y="-0.578125" style="font-style: oblique; font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="126.519155" y="-0.578125" style="font-style: oblique; font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">y</tspan> + <tspan x="6.634277" y="-0.578125" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="16.242188" y="-0.578125" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="25.793667" y="-0.578125" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">+</tspan> + <tspan x="48.793145" y="-0.578125" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">⋅</tspan> + <tspan x="54.176445" y="-0.578125" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">C</tspan> + <tspan x="61.507988" y="-0.578125" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">E</tspan> + <tspan x="68.142266" y="-0.578125" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="72.238701" y="-0.578125" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="75.155938" y="-0.578125" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="81.58001" y="-0.578125" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">g</tspan> + <tspan x="88.245049" y="-0.578125" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="91.162285" y="-0.578125" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="95.279229" y="-0.578125" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="100.749688" y="-0.578125" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="115.334673" y="-0.578125" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="119.431109" y="-0.578125" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">,</tspan> + <tspan x="132.733023" y="-0.578125" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="110.357598" y="1.226562" style="font-size: 7.35px; font-family: 'DejaVu Sans'; fill: #3a3a3a">3</tspan> + </text> + </g> + </g> + <g id="text_29"> + <!-- pull $-\beta\,\nabla\mathrm{CE}$ --> + <g style="fill: #b03a2e" transform="translate(788.832 89.493714)"> + <text> + <tspan x="0" y="-0.109375" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #b03a2e">p</tspan> + <tspan x="5.474854" y="-0.109375" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #b03a2e">u</tspan> + <tspan x="10.941284" y="-0.109375" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #b03a2e">l</tspan> + <tspan x="13.337585" y="-0.109375" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #b03a2e">l</tspan> + <tspan x="15.733887" y="-0.109375" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #b03a2e"> </tspan> + <tspan x="18.475525" y="-0.109375" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #b03a2e">−</tspan> + <tspan x="32.606992" y="-0.109375" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #b03a2e">∇</tspan> + <tspan x="38.376645" y="-0.109375" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #b03a2e">C</tspan> + <tspan x="44.398984" y="-0.109375" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #b03a2e">E</tspan> + <tspan x="25.702332" y="-0.109375" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #b03a2e">β</tspan> + </text> + </g> + </g> + <g id="text_30"> + <text style="font-size: 7.3px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: start; fill: #b03a2e" x="788.832" y="104.871487" transform="rotate(-0 788.832 104.871487)">(the only place the label enters)</text> + </g> + <g id="patch_21"> + <path d="M 778.866838 93.443471 +Q 755.568635 102.856886 733.825358 111.642049 +" style="fill: none; stroke: #b03a2e; stroke-width: 1.5; stroke-linecap: round"/> + <path d="M 738.729106 112.033524 +L 733.825358 111.642049 +L 737.080788 107.953935 +z +" style="fill: #b03a2e; stroke: #b03a2e; stroke-width: 1.5; stroke-linecap: round"/> + </g> + <g id="text_31"> + <!-- $z_3^{\beta} = z_3 + d_3$ --> + <g style="fill: #3a3a3a" transform="translate(687.555 133.750714)"> + <text> + <tspan x="0" y="-0.51875" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="20.440576" y="-0.51875" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="40.466895" y="-0.51875" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="5.126074" y="-5.176563" style="font-style: oblique; font-size: 6.3px; font-family: 'DejaVu Sans'; fill: #3a3a3a">β</tspan> + <tspan x="4.724121" y="1.942187" style="font-size: 6.3px; font-family: 'DejaVu Sans'; fill: #3a3a3a">3</tspan> + <tspan x="25.164697" y="0.957812" style="font-size: 6.3px; font-family: 'DejaVu Sans'; fill: #3a3a3a">3</tspan> + <tspan x="46.179785" y="0.957812" style="font-size: 6.3px; font-family: 'DejaVu Sans'; fill: #3a3a3a">3</tspan> + <tspan x="11.146143" y="-0.51875" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="31.172461" y="-0.51875" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">+</tspan> + </text> + </g> + </g> + <g id="text_32"> + <!-- $z_2^{\beta} = z_2 + d_2$ --> + <g style="fill: #3a3a3a" transform="translate(687.555 216.081429)"> + <text> + <tspan x="0" y="-0.51875" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="20.440576" y="-0.51875" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="40.466895" y="-0.51875" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="5.126074" y="-5.176563" style="font-style: oblique; font-size: 6.3px; font-family: 'DejaVu Sans'; fill: #3a3a3a">β</tspan> + <tspan x="4.724121" y="1.942187" style="font-size: 6.3px; font-family: 'DejaVu Sans'; fill: #3a3a3a">2</tspan> + <tspan x="25.164697" y="0.957812" style="font-size: 6.3px; font-family: 'DejaVu Sans'; fill: #3a3a3a">2</tspan> + <tspan x="46.179785" y="0.957812" style="font-size: 6.3px; font-family: 'DejaVu Sans'; fill: #3a3a3a">2</tspan> + <tspan x="11.146143" y="-0.51875" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="31.172461" y="-0.51875" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">+</tspan> + </text> + </g> + </g> + <g id="text_33"> + <!-- $z_1^{\beta} = z_1 + d_1$ --> + <g style="fill: #3a3a3a" transform="translate(687.555 298.322143)"> + <text> + <tspan x="0" y="-0.60625" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="20.440576" y="-0.60625" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="40.466895" y="-0.60625" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="5.126074" y="-5.176563" style="font-style: oblique; font-size: 6.3px; font-family: 'DejaVu Sans'; fill: #3a3a3a">β</tspan> + <tspan x="4.724121" y="1.854688" style="font-size: 6.3px; font-family: 'DejaVu Sans'; fill: #3a3a3a">1</tspan> + <tspan x="25.164697" y="0.870313" style="font-size: 6.3px; font-family: 'DejaVu Sans'; fill: #3a3a3a">1</tspan> + <tspan x="46.179785" y="0.870313" style="font-size: 6.3px; font-family: 'DejaVu Sans'; fill: #3a3a3a">1</tspan> + <tspan x="11.146143" y="-0.60625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="31.172461" y="-0.60625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">+</tspan> + </text> + </g> + </g> + <g id="patch_22"> + <path d="M 712.8 194.928976 +Q 712.8 171.771648 712.8 149.844158 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.1; stroke-linecap: round"/> + <path d="M 710.6 154.244158 +L 712.8 149.844158 +L 715 154.244158 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.1; stroke-linecap: round"/> + </g> + <g id="patch_23"> + <path d="M 712.8 277.21469 +Q 712.8 254.057363 712.8 232.129872 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.1; stroke-linecap: round"/> + <path d="M 710.6 236.529872 +L 712.8 232.129872 +L 715 236.529872 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.1; stroke-linecap: round"/> + </g> + <g id="patch_24"> + <path d="M 712.8 351.955413 +Q 712.8 333.943917 712.8 317.162258 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.1; stroke-linecap: round"/> + <path d="M 710.6 321.562258 +L 712.8 317.162258 +L 715 321.562258 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.1; stroke-linecap: round"/> + </g> + <g id="text_34"> + <!-- $\mathrm{emb}(x)$ --> + <g style="fill: #3a3a3a" transform="translate(696.6 369.34058)"> + <text> + <tspan x="0" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="5.537109" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="14.304199" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">b</tspan> + <tspan x="20.01709" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="28.854492" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="23.52832" y="-0.15625" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">x</tspan> + </text> + </g> + </g> + <g id="patch_25"> + <path d="M 821.1456 144.498633 +Q 821.1456 169.370812 821.1456 192.56594 +" style="fill: none; stroke: #d95f02; stroke-width: 1.5; stroke-linecap: round"/> + <path d="M 823.3456 188.16594 +L 821.1456 192.56594 +L 818.9456 188.16594 +z +" style="fill: #d95f02; stroke: #d95f02; stroke-width: 1.5; stroke-linecap: round"/> + </g> + <g id="text_35"> + <!-- $d_2 = J_3^{\top} d_3$ --> + <g style="fill: #d95f02" transform="translate(836.352 171.908429)"> + <text> + <tspan x="0" y="-0.146875" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #d95f02">d</tspan> + <tspan x="20.149713" y="-0.146875" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #d95f02">J</tspan> + <tspan x="31.18288" y="-0.146875" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #d95f02">d</tspan> + <tspan x="5.474854" y="1.329687" style="font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #d95f02">2</tspan> + <tspan x="22.693414" y="2.314062" style="font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #d95f02">3</tspan> + <tspan x="24.501306" y="-3.592188" style="font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #d95f02">⊤</tspan> + <tspan x="36.657733" y="1.329687" style="font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #d95f02">3</tspan> + <tspan x="11.242548" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #d95f02">=</tspan> + </text> + </g> + </g> + <g id="patch_26"> + <path d="M 821.1456 226.784347 +Q 821.1456 251.656526 821.1456 274.851654 +" style="fill: none; stroke: #d95f02; stroke-width: 1.5; stroke-linecap: round"/> + <path d="M 823.3456 270.451654 +L 821.1456 274.851654 +L 818.9456 270.451654 +z +" style="fill: #d95f02; stroke: #d95f02; stroke-width: 1.5; stroke-linecap: round"/> + </g> + <g id="text_36"> + <!-- $d_1 = J_2^{\top} d_2$ --> + <g style="fill: #d95f02" transform="translate(836.352 254.237143)"> + <text> + <tspan x="0" y="-0.146875" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #d95f02">d</tspan> + <tspan x="20.149713" y="-0.146875" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #d95f02">J</tspan> + <tspan x="31.18288" y="-0.146875" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #d95f02">d</tspan> + <tspan x="5.474854" y="1.329687" style="font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #d95f02">1</tspan> + <tspan x="22.693414" y="2.314062" style="font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #d95f02">2</tspan> + <tspan x="24.501306" y="-3.592188" style="font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #d95f02">⊤</tspan> + <tspan x="36.657733" y="1.329687" style="font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #d95f02">2</tspan> + <tspan x="11.242548" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #d95f02">=</tspan> + </text> + </g> + </g> + <g id="text_37"> + <!-- The top state is pulled toward lower loss; each layer's mismatch $d_l$ --> + <g style="fill: #3a3a3a" transform="translate(629.544 416.481429)"> + <text> + <tspan x="0" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">T</tspan> + <tspan x="5.115784" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="10.423767" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="15.576355" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="18.238525" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="21.522278" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="26.64624" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="31.962402" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="34.624573" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="38.987915" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="42.271667" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="47.403809" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="50.687561" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="55.840149" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="58.502319" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="60.829163" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="65.192505" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="67.854675" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="73.170837" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="78.478821" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="80.805664" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="83.132507" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="88.285095" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="93.601257" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="96.263428" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="99.54718" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="104.671143" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">w</tspan> + <tspan x="111.520813" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="116.652954" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="120.096191" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="125.412354" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="128.074524" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="130.401367" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="135.52533" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">w</tspan> + <tspan x="142.375" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="147.527588" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="150.970825" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="153.632996" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="155.959839" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="161.083801" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="165.447144" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="169.810486" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">;</tspan> + <tspan x="172.632141" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="175.294312" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="180.446899" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="185.579041" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="190.183655" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="195.491638" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="198.153809" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="200.480652" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="205.612793" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">y</tspan> + <tspan x="210.569092" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="215.72168" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="219.164917" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">'</tspan> + <tspan x="221.467224" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="225.830566" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="228.492737" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="236.651001" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="238.977844" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="243.341187" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="251.499451" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="256.631592" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="259.915344" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="264.519958" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="269.827942" y="-0.921875" style="font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="272.490112" y="-0.921875" style="font-style: oblique; font-size: 8.4px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="277.806274" y="0.390625" style="font-style: oblique; font-size: 5.88px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + </text> + </g> + </g> + <g id="text_38"> + <text style="font-size: 8.4px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="769.824" y="437.060732" transform="rotate(-0 769.824 437.060732)">transmits DOWN through the same weights, and the stack re-settles.</text> + </g> + <g id="text_39"> + <!-- ($J^{\top}$ = the transpose read a bidirectional physical device provides) --> + <g style="fill: #8a8a8a" transform="translate(633.991 464.918429)"> + <text> + <tspan x="0" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">(</tspan> + <tspan x="13.711999" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="16.334435" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">=</tspan> + <tspan x="23.247033" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="25.869469" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="29.104211" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">h</tspan> + <tspan x="34.33297" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="39.408654" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="42.03109" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="45.265832" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="48.657677" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="53.713219" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">n</tspan> + <tspan x="58.941979" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="63.240197" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">p</tspan> + <tspan x="68.477013" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">o</tspan> + <tspan x="73.524499" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="77.822716" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="82.8984" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="85.520837" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="88.912682" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="93.988366" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="99.043908" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">d</tspan> + <tspan x="104.280724" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="106.903161" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="111.958703" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="114.581139" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">b</tspan> + <tspan x="119.817956" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="122.11007" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">d</tspan> + <tspan x="127.346886" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="129.639001" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="133.030846" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="138.10653" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">c</tspan> + <tspan x="142.642419" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="145.87716" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="148.169274" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">o</tspan> + <tspan x="153.216759" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">n</tspan> + <tspan x="158.445519" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="163.501061" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">l</tspan> + <tspan x="165.793175" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="168.415612" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">p</tspan> + <tspan x="173.652428" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">h</tspan> + <tspan x="178.881188" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">y</tspan> + <tspan x="183.763512" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="188.06173" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="190.353844" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">c</tspan> + <tspan x="194.889733" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="199.945275" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">l</tspan> + <tspan x="202.237389" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="204.859826" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">d</tspan> + <tspan x="210.096642" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="215.172326" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">v</tspan> + <tspan x="220.05465" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="222.346764" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">c</tspan> + <tspan x="226.882653" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="231.958337" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="234.580773" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">p</tspan> + <tspan x="239.81759" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="243.209435" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">o</tspan> + <tspan x="248.256921" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">v</tspan> + <tspan x="253.139245" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="255.431359" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">d</tspan> + <tspan x="260.668175" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="265.743859" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="270.042077" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">)</tspan> + <tspan x="3.218628" y="-0.010938" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #8a8a8a">J</tspan> + <tspan x="7.337572" y="-3.073438" style="font-size: 5.74px; font-family: 'DejaVu Sans'; fill: #8a8a8a">⊤</tspan> + </text> + </g> + </g> + <g id="text_40"> + <text style="font-weight: 700; font-size: 10px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: start; fill: #3a3a3a" x="38.016" y="506.073661" transform="rotate(-0 38.016 506.073661)">(D) The update — a difference measurement between the two settled states</text> + </g> + <g id="text_41"> + <!-- $\hat{g}\ =\ \left[\ \partial_\theta E(z^{\beta})\ -\ \partial_\theta E(z^{0})\ \right]\,/\,\beta$$\qquad\qquad(\partial_\theta E(z^0)\equiv 0\ \mathrm{here,\ since}\ E=0)$ --> + <g style="fill: #3a3a3a" transform="translate(303.3675 532.465714)"> + <text> + <tspan x="6.931885" y="-2.203125" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">̂</tspan> + <tspan x="12.120083" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="33.37299" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">∂</tspan> + <tspan x="50.339616" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="65.430233" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="74.981713" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">−</tspan> + <tspan x="89.234608" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">∂</tspan> + <tspan x="106.201234" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="121.277496" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="134.078689" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">/</tspan> + <tspan x="186.935046" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="191.031482" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">∂</tspan> + <tspan x="207.998108" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="223.074369" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="229.216459" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">≡</tspan> + <tspan x="240.059965" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">0</tspan> + <tspan x="250.149775" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="256.80456" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="263.264521" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="267.581415" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="274.041376" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">,</tspan> + <tspan x="282.834067" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="288.304526" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="291.221762" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="297.876547" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="303.649496" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="322.198779" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="333.042285" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">0</tspan> + <tspan x="339.722704" y="-0.890625" style="font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="0" y="-0.890625" style="font-style: oblique; font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">g</tspan> + <tspan x="43.705339" y="-0.890625" style="font-style: oblique; font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">E</tspan> + <tspan x="54.436052" y="-0.890625" style="font-style: oblique; font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="99.566957" y="-0.890625" style="font-style: oblique; font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">E</tspan> + <tspan x="110.29767" y="-0.890625" style="font-style: oblique; font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="139.321033" y="-0.890625" style="font-style: oblique; font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">β</tspan> + <tspan x="201.363831" y="-0.890625" style="font-style: oblique; font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">E</tspan> + <tspan x="212.094543" y="-0.890625" style="font-style: oblique; font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="313.518847" y="-0.890625" style="font-style: oblique; font-size: 10.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">E</tspan> + <tspan x="26.372979" y="-0.467187" style="font-size: 9.321329px; font-family: 'STIXSizeOneSym'; fill: #3a3a3a">[</tspan> + <tspan x="128.783321" y="-0.467187" style="font-size: 9.321329px; font-family: 'STIXSizeOneSym'; fill: #3a3a3a">]</tspan> + <tspan x="38.907707" y="0.914062" style="font-style: oblique; font-size: 7.35px; font-family: 'DejaVu Sans'; fill: #3a3a3a">θ</tspan> + <tspan x="60.438803" y="-5.101562" style="font-style: oblique; font-size: 7.35px; font-family: 'DejaVu Sans'; fill: #3a3a3a">β</tspan> + <tspan x="94.769325" y="0.914062" style="font-style: oblique; font-size: 7.35px; font-family: 'DejaVu Sans'; fill: #3a3a3a">θ</tspan> + <tspan x="196.566199" y="0.914062" style="font-style: oblique; font-size: 7.35px; font-family: 'DejaVu Sans'; fill: #3a3a3a">θ</tspan> + <tspan x="116.300421" y="-5.101562" style="font-size: 7.35px; font-family: 'DejaVu Sans'; fill: #3a3a3a">0</tspan> + <tspan x="218.097294" y="-5.101562" style="font-size: 7.35px; font-family: 'DejaVu Sans'; fill: #3a3a3a">0</tspan> + </text> + </g> + </g> + <g id="text_42"> + <!-- per layer: $\Delta\theta_l\ \propto\ \langle\ d_l\ ,\ \partial f_l(z_{l-1})/\partial\theta_l\ \rangle\ /\ \beta$ --> + <g style="fill: #3a3a3a" transform="translate(382.955 554.66)"> + <text> + <tspan x="0" y="-0.328125" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="6.030273" y="-0.328125" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="11.875" y="-0.328125" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="15.780762" y="-0.328125" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="18.800537" y="-0.328125" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="21.439941" y="-0.328125" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="27.261475" y="-0.328125" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">y</tspan> + <tspan x="32.883545" y="-0.328125" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="38.728271" y="-0.328125" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="42.634033" y="-0.328125" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">:</tspan> + <tspan x="45.834717" y="-0.328125" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="48.854492" y="-0.328125" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="51.874268" y="-0.328125" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">Δ</tspan> + <tspan x="71.241839" y="-0.328125" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">∝</tspan> + <tspan x="82.963732" y="-0.328125" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">⟨</tspan> + <tspan x="100.990697" y="-0.328125" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">,</tspan> + <tspan x="108.945988" y="-0.328125" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">∂</tspan> + <tspan x="119.323845" y="-0.328125" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="142.531804" y="-0.328125" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="146.238103" y="-0.328125" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">/</tspan> + <tspan x="149.438786" y="-0.328125" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">∂</tspan> + <tspan x="165.369102" y="-0.328125" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">⟩</tspan> + <tspan x="172.160087" y="-0.328125" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">/</tspan> + <tspan x="58.373047" y="-0.328125" style="font-style: oblique; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">θ</tspan> + <tspan x="89.754717" y="-0.328125" style="font-style: oblique; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="113.858342" y="-0.328125" style="font-style: oblique; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="123.030144" y="-0.328125" style="font-style: oblique; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="154.35114" y="-0.328125" style="font-style: oblique; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">θ</tspan> + <tspan x="178.445456" y="-0.328125" style="font-style: oblique; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">β</tspan> + <tspan x="64.185303" y="1.3125" style="font-style: oblique; font-size: 6.65px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="95.78499" y="1.3125" style="font-style: oblique; font-size: 6.65px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="117.202824" y="1.3125" style="font-style: oblique; font-size: 6.65px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="128.016716" y="1.3125" style="font-style: oblique; font-size: 6.65px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="160.163396" y="1.3125" style="font-style: oblique; font-size: 6.65px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="131.15988" y="1.3125" style="font-size: 6.65px; font-family: 'DejaVu Sans'; fill: #3a3a3a">−</tspan> + <tspan x="138.027434" y="1.3125" style="font-size: 6.65px; font-family: 'DejaVu Sans'; fill: #3a3a3a">1</tspan> + </text> + </g> + </g> + <g id="text_43"> + <text style="font-size: 8.2px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; fill: #3a3a3a" transform="translate(257.54125 566.753059)">Each layer updates from its own boundary mismatch — no global backward graph, no global tape, no loss</text> + <!-- derivatives except the top nudge. $\beta\to 0$ gives the exact gradient; bias is $O(\beta)$; a $\pm\beta$ two-sided read cancels it to $O(\beta^2)$. --> + <g style="fill: #3a3a3a" transform="translate(230.881 577.412803)"> + <text> + <tspan x="0" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="5.236816" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="10.3125" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="13.704346" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="15.99646" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">v</tspan> + <tspan x="20.878784" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="25.934326" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="29.169067" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="31.461182" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">v</tspan> + <tspan x="36.343506" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="41.419189" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="45.717407" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="48.339844" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="53.415527" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">x</tspan> + <tspan x="58.297852" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="62.83374" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="67.909424" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="73.14624" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="76.380981" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="79.003418" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="82.238159" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="87.466919" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="92.542603" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="95.165039" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="98.39978" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="103.447266" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="108.684082" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="111.306519" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="116.535278" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="121.764038" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="127.000854" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">g</tspan> + <tspan x="132.237671" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="137.313354" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">.</tspan> + <tspan x="139.935791" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="142.558228" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="152.052979" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">→</tspan> + <tspan x="160.572876" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">0</tspan> + <tspan x="165.821777" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="168.444214" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">g</tspan> + <tspan x="173.68103" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="175.973145" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">v</tspan> + <tspan x="180.855469" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="185.931152" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="190.22937" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="192.851807" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="196.086548" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="201.315308" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="206.390991" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="209.013428" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="214.089111" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">x</tspan> + <tspan x="218.971436" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="224.026978" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="228.562866" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="231.797607" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="234.420044" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">g</tspan> + <tspan x="239.65686" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="243.048706" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="248.104248" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="253.341064" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="255.633179" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="260.708862" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="265.937622" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="269.172363" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">;</tspan> + <tspan x="271.951904" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="274.574341" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">b</tspan> + <tspan x="279.811157" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="282.103271" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="287.158813" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="291.457031" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="294.079468" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="296.371582" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="300.6698" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="309.785889" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="318.269531" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="321.488159" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">;</tspan> + <tspan x="324.2677" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="326.890137" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="331.945679" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="334.568115" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">±</tspan> + <tspan x="346.745728" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="349.368164" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="352.602905" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">w</tspan> + <tspan x="359.350342" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="364.397827" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">-</tspan> + <tspan x="367.374756" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="371.672974" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="373.965088" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="379.201904" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="384.277588" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="389.514404" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="392.136841" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="395.528687" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="400.60437" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="405.659912" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="410.896729" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="413.519165" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="418.055054" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="423.110596" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="428.339355" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="432.875244" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="437.950928" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="440.243042" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="444.54126" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="447.163696" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="449.455811" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="452.690552" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="455.312988" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="458.547729" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="463.595215" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="472.711304" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="485.690219" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="488.908847" y="-0.78125" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">.</tspan> + <tspan x="145.180664" y="-0.78125" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">β</tspan> + <tspan x="303.292236" y="-0.78125" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">O</tspan> + <tspan x="313.004517" y="-0.78125" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">β</tspan> + <tspan x="341.480713" y="-0.78125" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">β</tspan> + <tspan x="466.217651" y="-0.78125" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">O</tspan> + <tspan x="475.929932" y="-0.78125" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">β</tspan> + <tspan x="481.797238" y="-3.84375" style="font-size: 5.74px; font-family: 'DejaVu Sans'; fill: #3a3a3a">2</tspan> + </text> + </g> + </g> + <g id="text_44"> + <text style="font-weight: 700; font-size: 10px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: start; fill: #3a3a3a" x="38.016" y="615.06317" transform="rotate(-0 38.016 615.06317)">(E) This is not predictive coding — same energy family, different measurement</text> + </g> + <g id="text_45"> + <text style="font-size: 8.2px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: start; fill: #3a3a3a" x="42.768" y="637.234116" transform="rotate(-0 42.768 637.234116)">· PC (as typically run): ONE settled phase with the target clamped hard; the update uses the raw errors of that single state → finite-clamp</text> + </g> + <g id="text_46"> + <text style="font-size: 8.2px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: start; fill: #3a3a3a" x="42.768" y="653.005545" transform="rotate(-0 42.768 653.005545)"> bias. Its "exact-BP" results require freezing predictions during error transport (fixed-prediction) — backprop re-expressed in local variables.</text> + </g> + <g id="text_47"> + <text style="font-size: 8.2px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: start; fill: #3a3a3a" x="42.768" y="670.859741" transform="rotate(-0 42.768 670.859741)">· EP (here): TWO phases and an infinitesimal nudge; the update is a difference quotient in β → bias is measured and controllable, and the</text> + </g> + <g id="text_48"> + <text style="font-size: 8.2px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: start; fill: #3a3a3a" x="42.768" y="686.605545" transform="rotate(-0 42.768 686.605545)"> settle stays fully self-consistent — which is what physical hardware actually does.</text> + </g> + <g id="text_49"> + <text style="font-size: 8.2px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: start; fill: #3a3a3a" x="42.768" y="701.691259" transform="rotate(-0 42.768 701.691259)">· The free phase is the zero-reference of the measurement: on analog hardware the subtraction cancels state-independent device offsets.</text> + </g> + </g> + </g> + <defs> + <clipPath id="p7fae8feff8"> + <rect x="0" y="0" width="950.4" height="720"/> + </clipPath> + </defs> +</svg> diff --git a/assets/figs/fig_primer_block.emf b/assets/figs/fig_primer_block.emf Binary files differnew file mode 100644 index 0000000..7993cc9 --- /dev/null +++ b/assets/figs/fig_primer_block.emf diff --git a/assets/figs/fig_primer_block.png b/assets/figs/fig_primer_block.png Binary files differnew file mode 100644 index 0000000..b5eb58d --- /dev/null +++ b/assets/figs/fig_primer_block.png diff --git a/assets/figs/fig_primer_block.svg b/assets/figs/fig_primer_block.svg new file mode 100644 index 0000000..7a901a9 --- /dev/null +++ b/assets/figs/fig_primer_block.svg @@ -0,0 +1,1956 @@ +<?xml version="1.0" encoding="utf-8" standalone="no"?> +<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" + "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"> +<svg xmlns:xlink="http://www.w3.org/1999/xlink" width="720pt" height="1022.4pt" viewBox="0 0 720 1022.4" xmlns="http://www.w3.org/2000/svg" version="1.1"> + <metadata> + <rdf:RDF xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:cc="http://creativecommons.org/ns#" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"> + <cc:Work> + <dc:type rdf:resource="http://purl.org/dc/dcmitype/StillImage"/> + <dc:date>2026-07-15T09:29:45.734111</dc:date> + <dc:format>image/svg+xml</dc:format> + <dc:creator> + <cc:Agent> + <dc:title>Matplotlib v3.10.8, https://matplotlib.org/</dc:title> + </cc:Agent> + </dc:creator> + </cc:Work> + </rdf:RDF> + </metadata> + <defs> + <style type="text/css">*{stroke-linejoin: round; stroke-linecap: butt}</style> + </defs> + <g id="figure_1"> + <g id="patch_1"> + <path d="M 0 1022.4 +L 720 1022.4 +L 720 0 +L 0 0 +z +" style="fill: #ffffff"/> + </g> + <g id="axes_1"> + <g id="line2d_1"> + <path d="M 288 83.52 +L 633.6 83.52 +L 633.6 505.44 +" clip-path="url(#pdba9adb895)" style="fill: none; stroke: #555555; stroke-width: 1.4; stroke-linecap: round"/> + </g> + <g id="patch_2"> + <path d="M 182.16 122.04 +L 393.84 122.04 +Q 397.8 122.04 397.8 118.08 +L 397.8 99.36 +Q 397.8 95.4 393.84 95.4 +L 182.16 95.4 +Q 178.2 95.4 178.2 99.36 +L 178.2 118.08 +Q 178.2 122.04 182.16 122.04 +z +" clip-path="url(#pdba9adb895)" style="fill: #eaf1fa; stroke: #2c6fbb; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_3"> + <path d="M 54.36 165.24 +L 176.04 165.24 +Q 180 165.24 180 161.28 +L 180 142.56 +Q 180 138.6 176.04 138.6 +L 54.36 138.6 +Q 50.4 138.6 50.4 142.56 +L 50.4 161.28 +Q 50.4 165.24 54.36 165.24 +z +" clip-path="url(#pdba9adb895)" style="fill: #faf6ee; stroke: #c9b895; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_4"> + <path d="M 227.16 165.24 +L 348.84 165.24 +Q 352.8 165.24 352.8 161.28 +L 352.8 142.56 +Q 352.8 138.6 348.84 138.6 +L 227.16 138.6 +Q 223.2 138.6 223.2 142.56 +L 223.2 161.28 +Q 223.2 165.24 227.16 165.24 +z +" clip-path="url(#pdba9adb895)" style="fill: #faf6ee; stroke: #c9b895; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_5"> + <path d="M 54.36 199.08 +L 176.04 199.08 +Q 180 199.08 180 195.12 +L 180 176.4 +Q 180 172.44 176.04 172.44 +L 54.36 172.44 +Q 50.4 172.44 50.4 176.4 +L 50.4 195.12 +Q 50.4 199.08 54.36 199.08 +z +" clip-path="url(#pdba9adb895)" style="fill: #f4f4f4; stroke: #9a9a9a; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_6"> + <path d="M 227.16 199.08 +L 348.84 199.08 +Q 352.8 199.08 352.8 195.12 +L 352.8 176.4 +Q 352.8 172.44 348.84 172.44 +L 227.16 172.44 +Q 223.2 172.44 223.2 176.4 +L 223.2 195.12 +Q 223.2 199.08 227.16 199.08 +z +" clip-path="url(#pdba9adb895)" style="fill: #f4f4f4; stroke: #9a9a9a; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_7"> + <path d="M 399.96 199.08 +L 521.64 199.08 +Q 525.6 199.08 525.6 195.12 +L 525.6 176.4 +Q 525.6 172.44 521.64 172.44 +L 399.96 172.44 +Q 396 172.44 396 176.4 +L 396 195.12 +Q 396 199.08 399.96 199.08 +z +" clip-path="url(#pdba9adb895)" style="fill: #f4f4f4; stroke: #9a9a9a; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_8"> + <path d="M 54.36 232.92 +L 176.04 232.92 +Q 180 232.92 180 228.96 +L 180 210.24 +Q 180 206.28 176.04 206.28 +L 54.36 206.28 +Q 50.4 206.28 50.4 210.24 +L 50.4 228.96 +Q 50.4 232.92 54.36 232.92 +z +" clip-path="url(#pdba9adb895)" style="fill: #f4f4f4; stroke: #9a9a9a; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_9"> + <path d="M 227.16 232.92 +L 348.84 232.92 +Q 352.8 232.92 352.8 228.96 +L 352.8 210.24 +Q 352.8 206.28 348.84 206.28 +L 227.16 206.28 +Q 223.2 206.28 223.2 210.24 +L 223.2 228.96 +Q 223.2 232.92 227.16 232.92 +z +" clip-path="url(#pdba9adb895)" style="fill: #f4f4f4; stroke: #9a9a9a; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_10"> + <path d="M 174.96 276.12 +L 401.04 276.12 +Q 405 276.12 405 272.16 +L 405 253.44 +Q 405 249.48 401.04 249.48 +L 174.96 249.48 +Q 171 249.48 171 253.44 +L 171 272.16 +Q 171 276.12 174.96 276.12 +z +" clip-path="url(#pdba9adb895)" style="fill: #f1ebf8; stroke: #7b5aa6; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_11"> + <path d="M 174.96 309.96 +L 401.04 309.96 +Q 405 309.96 405 306 +L 405 287.28 +Q 405 283.32 401.04 283.32 +L 174.96 283.32 +Q 171 283.32 171 287.28 +L 171 306 +Q 171 309.96 174.96 309.96 +z +" clip-path="url(#pdba9adb895)" style="fill: #f4f4f4; stroke: #9a9a9a; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_12"> + <path d="M 174.96 343.8 +L 401.04 343.8 +Q 405 343.8 405 339.84 +L 405 321.12 +Q 405 317.16 401.04 317.16 +L 174.96 317.16 +Q 171 317.16 171 321.12 +L 171 339.84 +Q 171 343.8 174.96 343.8 +z +" clip-path="url(#pdba9adb895)" style="fill: #f4f4f4; stroke: #9a9a9a; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_13"> + <path d="M 174.96 377.64 +L 401.04 377.64 +Q 405 377.64 405 373.68 +L 405 354.96 +Q 405 351 401.04 351 +L 174.96 351 +Q 171 351 171 354.96 +L 171 373.68 +Q 171 377.64 174.96 377.64 +z +" clip-path="url(#pdba9adb895)" style="fill: #f1ebf8; stroke: #7b5aa6; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_14"> + <path d="M 174.96 411.48 +L 401.04 411.48 +Q 405 411.48 405 407.52 +L 405 388.8 +Q 405 384.84 401.04 384.84 +L 174.96 384.84 +Q 171 384.84 171 388.8 +L 171 407.52 +Q 171 411.48 174.96 411.48 +z +" clip-path="url(#pdba9adb895)" style="fill: #f4f4f4; stroke: #9a9a9a; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_15"> + <path d="M 174.96 445.32 +L 401.04 445.32 +Q 405 445.32 405 441.36 +L 405 422.64 +Q 405 418.68 401.04 418.68 +L 174.96 418.68 +Q 171 418.68 171 422.64 +L 171 441.36 +Q 171 445.32 174.96 445.32 +z +" clip-path="url(#pdba9adb895)" style="fill: #eaf1fa; stroke: #2c6fbb; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_16"> + <path d="M 174.96 479.16 +L 401.04 479.16 +Q 405 479.16 405 475.2 +L 405 456.48 +Q 405 452.52 401.04 452.52 +L 174.96 452.52 +Q 171 452.52 171 456.48 +L 171 475.2 +Q 171 479.16 174.96 479.16 +z +" clip-path="url(#pdba9adb895)" style="fill: #faf6ee; stroke: #c9b895; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="line2d_2"> + <path d="M 288 540 +L 633.6 540 +L 633.6 754.56 +" clip-path="url(#pdba9adb895)" style="fill: none; stroke: #555555; stroke-width: 1.4; stroke-linecap: round"/> + </g> + <g id="patch_17"> + <path d="M 113.76 586.44 +L 260.64 586.44 +Q 264.6 586.44 264.6 582.48 +L 264.6 563.76 +Q 264.6 559.8 260.64 559.8 +L 113.76 559.8 +Q 109.8 559.8 109.8 563.76 +L 109.8 582.48 +Q 109.8 586.44 113.76 586.44 +z +" clip-path="url(#pdba9adb895)" style="fill: #eaf1fa; stroke: #2c6fbb; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_18"> + <path d="M 315.36 586.44 +L 462.24 586.44 +Q 466.2 586.44 466.2 582.48 +L 466.2 563.76 +Q 466.2 559.8 462.24 559.8 +L 315.36 559.8 +Q 311.4 559.8 311.4 563.76 +L 311.4 582.48 +Q 311.4 586.44 315.36 586.44 +z +" clip-path="url(#pdba9adb895)" style="fill: #eaf1fa; stroke: #2c6fbb; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_19"> + <path d="M 146.16 620.28 +L 228.24 620.28 +Q 232.2 620.28 232.2 616.32 +L 232.2 597.6 +Q 232.2 593.64 228.24 593.64 +L 146.16 593.64 +Q 142.2 593.64 142.2 597.6 +L 142.2 616.32 +Q 142.2 620.28 146.16 620.28 +z +" clip-path="url(#pdba9adb895)" style="fill: #f4f4f4; stroke: #9a9a9a; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_20"> + <path d="M 207.36 658.44 +L 368.64 658.44 +Q 372.6 658.44 372.6 654.48 +L 372.6 635.76 +Q 372.6 631.8 368.64 631.8 +L 207.36 631.8 +Q 203.4 631.8 203.4 635.76 +L 203.4 654.48 +Q 203.4 658.44 207.36 658.44 +z +" clip-path="url(#pdba9adb895)" style="fill: #f4f4f4; stroke: #9a9a9a; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_21"> + <path d="M 200.16 696.6 +L 375.84 696.6 +Q 379.8 696.6 379.8 692.64 +L 379.8 673.92 +Q 379.8 669.96 375.84 669.96 +L 200.16 669.96 +Q 196.2 669.96 196.2 673.92 +L 196.2 692.64 +Q 196.2 696.6 200.16 696.6 +z +" clip-path="url(#pdba9adb895)" style="fill: #eaf1fa; stroke: #2c6fbb; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_22"> + <path d="M 200.16 730.44 +L 375.84 730.44 +Q 379.8 730.44 379.8 726.48 +L 379.8 707.76 +Q 379.8 703.8 375.84 703.8 +L 200.16 703.8 +Q 196.2 703.8 196.2 707.76 +L 196.2 726.48 +Q 196.2 730.44 200.16 730.44 +z +" clip-path="url(#pdba9adb895)" style="fill: #faf6ee; stroke: #c9b895; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_23"> + <path d="M 37.44 846.36 +L 59.04 846.36 +Q 61.56 846.36 61.56 843.84 +L 61.56 833.76 +Q 61.56 831.24 59.04 831.24 +L 37.44 831.24 +Q 34.92 831.24 34.92 833.76 +L 34.92 843.84 +Q 34.92 846.36 37.44 846.36 +z +" clip-path="url(#pdba9adb895)" style="fill: #eaf1fa; stroke: #2c6fbb; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_24"> + <path d="M 37.44 870.84 +L 59.04 870.84 +Q 61.56 870.84 61.56 868.32 +L 61.56 858.24 +Q 61.56 855.72 59.04 855.72 +L 37.44 855.72 +Q 34.92 855.72 34.92 858.24 +L 34.92 868.32 +Q 34.92 870.84 37.44 870.84 +z +" clip-path="url(#pdba9adb895)" style="fill: #f1ebf8; stroke: #7b5aa6; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_25"> + <path d="M 37.44 895.32 +L 59.04 895.32 +Q 61.56 895.32 61.56 892.8 +L 61.56 882.72 +Q 61.56 880.2 59.04 880.2 +L 37.44 880.2 +Q 34.92 880.2 34.92 882.72 +L 34.92 892.8 +Q 34.92 895.32 37.44 895.32 +z +" clip-path="url(#pdba9adb895)" style="fill: #faf6ee; stroke: #c9b895; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_26"> + <path d="M 275.04 895.32 +L 296.64 895.32 +Q 299.16 895.32 299.16 892.8 +L 299.16 882.72 +Q 299.16 880.2 296.64 880.2 +L 275.04 880.2 +Q 272.52 880.2 272.52 882.72 +L 272.52 892.8 +Q 272.52 895.32 275.04 895.32 +z +" clip-path="url(#pdba9adb895)" style="fill: #f4f4f4; stroke: #9a9a9a; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="text_1"> + <text style="font-weight: 700; font-size: 12.5px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="360" y="26.440391" transform="rotate(-0 360 26.440391)">One transformer block, op by op</text> + </g> + <g id="text_2"> + <!-- OLMo2 ordering — RMSNorm after each sublayer, inside the residual. $C{=}512$ $H{=}8$ $h_d{=}64$ $T{=}256$ $h_{ff}{=}1408$ --> + <g style="fill: #8a8a8a" transform="translate(100.664 46.34)"> + <text> + <tspan x="0" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">O</tspan> + <tspan x="6.887207" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">L</tspan> + <tspan x="11.762085" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">M</tspan> + <tspan x="19.311523" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">o</tspan> + <tspan x="24.664917" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">2</tspan> + <tspan x="30.231934" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="33.013306" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">o</tspan> + <tspan x="38.366699" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="41.964111" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">d</tspan> + <tspan x="47.518311" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="52.901611" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="56.499023" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="58.930054" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">n</tspan> + <tspan x="64.475708" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">g</tspan> + <tspan x="70.029907" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="81.561279" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="84.342651" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">R</tspan> + <tspan x="90.422363" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">M</tspan> + <tspan x="97.971802" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">S</tspan> + <tspan x="103.526001" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">N</tspan> + <tspan x="110.071411" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">o</tspan> + <tspan x="115.424805" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="119.022217" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">m</tspan> + <tspan x="127.545776" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="130.327148" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="135.689087" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">f</tspan> + <tspan x="138.769531" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="142.200317" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="147.583618" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="151.18103" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="153.962402" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="159.345703" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="164.707642" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">c</tspan> + <tspan x="169.518433" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">h</tspan> + <tspan x="175.064087" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="177.845459" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="182.404175" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">u</tspan> + <tspan x="187.949829" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">b</tspan> + <tspan x="193.504028" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">l</tspan> + <tspan x="195.935059" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="201.296997" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">y</tspan> + <tspan x="206.47522" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="211.858521" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="215.455933" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">,</tspan> + <tspan x="218.237305" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="221.018677" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="223.449707" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">n</tspan> + <tspan x="228.995361" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="233.554077" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="235.985107" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">d</tspan> + <tspan x="241.539307" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="246.922607" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="249.703979" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="253.134766" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">h</tspan> + <tspan x="258.68042" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="264.063721" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="266.845093" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="270.442505" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="275.825806" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="280.384521" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="282.815552" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">d</tspan> + <tspan x="288.369751" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">u</tspan> + <tspan x="293.915405" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="299.277344" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">l</tspan> + <tspan x="301.708374" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">.</tspan> + <tspan x="304.489746" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="307.271118" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="310.05249" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="312.833862" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="315.615234" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="326.210938" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">=</tspan> + <tspan x="335.247192" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">5</tspan> + <tspan x="340.814209" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">1</tspan> + <tspan x="346.381226" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">2</tspan> + <tspan x="351.948242" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="354.729614" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="357.510986" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="368.57666" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">=</tspan> + <tspan x="377.612915" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">8</tspan> + <tspan x="383.179932" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="385.961304" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="388.742676" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="402.908447" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">=</tspan> + <tspan x="411.944702" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">6</tspan> + <tspan x="417.511719" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">4</tspan> + <tspan x="423.078735" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="425.860107" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="428.641479" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="438.472412" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">=</tspan> + <tspan x="447.508667" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">2</tspan> + <tspan x="453.075684" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">5</tspan> + <tspan x="458.6427" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">6</tspan> + <tspan x="464.209717" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="466.991089" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="469.772461" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="484.362915" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">=</tspan> + <tspan x="493.39917" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">1</tspan> + <tspan x="498.966187" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">4</tspan> + <tspan x="504.533203" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">0</tspan> + <tspan x="510.10022" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">8</tspan> + <tspan x="72.811279" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans Display'; fill: #8a8a8a">—</tspan> + <tspan x="318.396606" y="-0.15625" style="font-style: oblique; font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">C</tspan> + <tspan x="360.292358" y="-0.15625" style="font-style: oblique; font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">H</tspan> + <tspan x="391.524048" y="-0.15625" style="font-style: oblique; font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">h</tspan> + <tspan x="431.422852" y="-0.15625" style="font-style: oblique; font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">T</tspan> + <tspan x="472.553833" y="-0.15625" style="font-style: oblique; font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">h</tspan> + <tspan x="397.069702" y="1.320312" style="font-style: oblique; font-size: 6.16px; font-family: 'DejaVu Sans'; fill: #8a8a8a">d</tspan> + <tspan x="478.099487" y="1.320312" style="font-style: oblique; font-size: 6.16px; font-family: 'DejaVu Sans'; fill: #8a8a8a">f</tspan> + <tspan x="480.255798" y="1.320312" style="font-style: oblique; font-size: 6.16px; font-family: 'DejaVu Sans'; fill: #8a8a8a">f</tspan> + </text> + </g> + </g> + <g id="text_3"> + <text style="font-style: italic; font-size: 9.5px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; fill: #8a8a8a" transform="translate(25.661406 345.993867) rotate(-90)">attention sublayer</text> + </g> + <g id="text_4"> + <!-- input $z$ $(B,T,C)$ --> + <g style="fill: #3a3a3a" transform="translate(246.77 77.4925)"> + <text> + <tspan x="0" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="3.256348" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="10.019531" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="16.819824" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="23.583008" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="28.124268" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="36.418213" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="39.725586" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="43.032959" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="31.431641" y="-0.40625" style="font-style: oblique; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="50.046631" y="-0.40625" style="font-style: oblique; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">B</tspan> + <tspan x="61.43457" y="-0.40625" style="font-style: oblique; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">T</tspan> + <tspan x="72.108154" y="-0.40625" style="font-style: oblique; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">C</tspan> + <tspan x="46.340332" y="-0.40625" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="56.563965" y="-0.40625" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">,</tspan> + <tspan x="67.237549" y="-0.40625" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">,</tspan> + <tspan x="78.741455" y="-0.40625" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + </text> + </g> + </g> + <g id="patch_27"> + <path d="M 633.19823 501.12524 +Q 467.999756 503.280003 304.366397 505.414351 +" style="fill: none; stroke: #555555; stroke-width: 1.4; stroke-linecap: round"/> + <path d="M 308.392142 507.362012 +L 304.366397 505.414351 +L 308.339972 503.362352 +z +" style="fill: #555555; stroke: #555555; stroke-width: 1.4; stroke-linecap: round"/> + </g> + <g id="text_5"> + <text style="font-size: 8px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; fill: #8a8a8a" transform="translate(651.6475 318.636875) rotate(-90)">residual rail</text> + </g> + <g id="patch_28"> + <path d="M 288 85.362188 +Q 288 90.721592 288 94.795257 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 290 90.795257 +L 288 94.795257 +L 286 90.795257 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="text_6"> + <!-- matmul $W_{qkv}: C \to 3C$ --> + <g style="fill: #3a3a3a" transform="translate(241.689 110.655)"> + <text> + <tspan x="0" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="8.401794" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="13.687134" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="17.068909" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="25.470703" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="30.937134" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="33.333435" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="36.075073" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="60.172998" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">:</tspan> + <tspan x="72.461938" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">→</tspan> + <tspan x="81.369104" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">3</tspan> + <tspan x="38.816711" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">W</tspan> + <tspan x="64.759241" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">C</tspan> + <tspan x="86.856592" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">C</tspan> + <tspan x="47.344849" y="1.320312" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">q</tspan> + <tspan x="51.177246" y="1.320312" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">k</tspan> + <tspan x="54.673572" y="1.320312" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">v</tspan> + </text> + </g> + </g> + <g id="text_7"> + <!-- $(B,T,3C)$ — split into $q,k,v$ --> + <g style="fill: #8a8a8a" transform="translate(406.8 110.81)"> + <text> + <tspan x="0" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">(</tspan> + <tspan x="8.205811" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">,</tspan> + <tspan x="16.772766" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">,</tspan> + <tspan x="20.682068" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">3</tspan> + <tspan x="30.857422" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">)</tspan> + <tspan x="33.832214" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="43.880981" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="46.304749" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="50.277344" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">p</tspan> + <tspan x="55.117432" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">l</tspan> + <tspan x="57.235901" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="59.35437" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="62.344055" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="64.767822" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="66.886292" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">n</tspan> + <tspan x="71.718933" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="74.708618" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">o</tspan> + <tspan x="79.373718" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="86.637573" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">,</tspan> + <tspan x="94.962524" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">,</tspan> + <tspan x="2.974792" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">B</tspan> + <tspan x="12.115112" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">T</tspan> + <tspan x="25.533325" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">C</tspan> + <tspan x="81.797485" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">q</tspan> + <tspan x="90.546875" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">k</tspan> + <tspan x="98.871826" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">v</tspan> + <tspan x="36.255981" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans Display'; fill: #8a8a8a">—</tspan> + </text> + </g> + </g> + <g id="text_8"> + <!-- $q$ --> + <g style="fill: #3a3a3a" transform="translate(112.16 137.242109)"> + <text> + <tspan x="0" y="-0.40625" style="font-style: oblique; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">q</tspan> + </text> + </g> + </g> + <g id="text_9"> + <!-- $k$ --> + <g style="fill: #3a3a3a" transform="translate(285.245 137.261406)"> + <text> + <tspan x="0" y="-0.40625" style="font-style: oblique; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">k</tspan> + </text> + </g> + </g> + <g id="text_10"> + <!-- $v$ --> + <g style="fill: #3a3a3a" transform="translate(457.95 137.261406)"> + <text> + <tspan x="0" y="-0.53125" style="font-style: oblique; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">v</tspan> + </text> + </g> + </g> + <g id="patch_29"> + <path d="M 266.000098 120.295986 +Q 194.399445 130.320078 124.072113 140.165904 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 128.310776 141.591996 +L 124.072113 140.165904 +L 127.756184 137.630629 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="patch_30"> + <path d="M 288 120.638672 +Q 288 130.319117 288 138.713824 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 290 134.713824 +L 288 138.713824 +L 286 134.713824 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="patch_31"> + <path d="M 309.999902 120.295986 +Q 381.600555 130.320078 451.927887 140.165904 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 448.243816 137.630629 +L 451.927887 140.165904 +L 447.689224 141.591996 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="text_11"> + <!-- RMSNorm$_q$ --> + <g style="fill: #3a3a3a" transform="translate(91.894 153.812)"> + <text> + <tspan x="0" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">R</tspan> + <tspan x="5.992859" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">M</tspan> + <tspan x="13.434448" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">S</tspan> + <tspan x="18.909302" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">N</tspan> + <tspan x="25.361206" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="30.638123" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="34.184143" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="42.67207" y="1.164062" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">q</tspan> + </text> + </g> + </g> + <g id="text_12"> + <!-- RMSNorm$_k$ --> + <g style="fill: #3a3a3a" transform="translate(264.866 154.293062)"> + <text> + <tspan x="0" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">R</tspan> + <tspan x="5.992859" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">M</tspan> + <tspan x="13.434448" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">S</tspan> + <tspan x="18.909302" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">N</tspan> + <tspan x="25.361206" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="30.638123" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="34.184143" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="42.67207" y="1.164062" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">k</tspan> + </text> + </g> + </g> + <g id="text_13"> + <!-- split heads ($H{=}8$) --> + <g style="fill: #3a3a3a" transform="translate(75.812 188.125)"> + <text> + <tspan x="0" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="4.493591" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="9.968445" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="12.364746" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="14.761047" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="18.142822" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="20.88446" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="26.350891" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="31.657288" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="36.942627" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="42.41748" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="46.911072" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="49.65271" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="61.183594" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="70.090759" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">8</tspan> + <tspan x="75.578247" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="53.017639" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">H</tspan> + </text> + </g> + </g> + <g id="text_14"> + <!-- split heads ($H{=}8$) --> + <g style="fill: #3a3a3a" transform="translate(248.612 188.125)"> + <text> + <tspan x="0" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="4.493591" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="9.968445" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="12.364746" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="14.761047" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="18.142822" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="20.88446" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="26.350891" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="31.657288" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="36.942627" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="42.41748" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="46.911072" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="49.65271" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="61.183594" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="70.090759" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">8</tspan> + <tspan x="75.578247" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="53.017639" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">H</tspan> + </text> + </g> + </g> + <g id="text_15"> + <!-- split heads ($H{=}8$) --> + <g style="fill: #3a3a3a" transform="translate(421.412 188.125)"> + <text> + <tspan x="0" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="4.493591" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="9.968445" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="12.364746" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="14.761047" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="18.142822" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="20.88446" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="26.350891" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="31.657288" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="36.942627" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="42.41748" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="46.911072" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="49.65271" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="61.183594" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="70.090759" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">8</tspan> + <tspan x="75.578247" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="53.017639" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">H</tspan> + </text> + </g> + </g> + <g id="text_16"> + <text style="font-size: 8.6px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="115.2" y="221.973062" transform="rotate(-0 115.2 221.973062)">RoPE</text> + </g> + <g id="text_17"> + <text style="font-size: 8.6px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="288" y="221.973062" transform="rotate(-0 288 221.973062)">RoPE</text> + </g> + <g id="text_18"> + <!-- $\theta = 5{\cdot}10^5$ --> + <g style="fill: #8a8a8a" transform="translate(361.44 221.697125)"> + <text> + <tspan x="0" y="-0.857813" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">θ</tspan> + <tspan x="6.150635" y="-0.857813" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">=</tspan> + <tspan x="14.025085" y="-0.857813" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">5</tspan> + <tspan x="18.876343" y="-0.857813" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">⋅</tspan> + <tspan x="21.30011" y="-0.857813" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">1</tspan> + <tspan x="26.151367" y="-0.857813" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">0</tspan> + <tspan x="31.079187" y="-3.920313" style="font-size: 5.32px; font-family: 'DejaVu Sans'; fill: #8a8a8a">5</tspan> + </text> + </g> + </g> + <g id="patch_32"> + <path d="M 115.2 163.838145 +Q 115.2 168.83972 115.2 172.555557 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 117.2 168.555557 +L 115.2 172.555557 +L 113.2 168.555557 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="patch_33"> + <path d="M 115.2 197.678145 +Q 115.2 202.67972 115.2 206.395557 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 117.2 202.395557 +L 115.2 206.395557 +L 113.2 202.395557 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="patch_34"> + <path d="M 288 163.838145 +Q 288 168.83972 288 172.555557 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 290 168.555557 +L 288 172.555557 +L 286 168.555557 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="patch_35"> + <path d="M 288 197.678145 +Q 288 202.67972 288 206.395557 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 290 202.395557 +L 288 206.395557 +L 286 202.395557 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="patch_36"> + <path d="M 460.8 145.11999 +Q 460.8 159.479155 460.8 172.552581 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 462.8 168.552581 +L 460.8 172.552581 +L 458.8 168.552581 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="patch_37"> + <path d="M 115.590674 231.195344 +Q 165.597693 240.839555 214.342237 250.240288 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 210.793347 247.519005 +L 214.342237 250.240288 +L 210.035876 251.44663 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="patch_38"> + <path d="M 288 231.523418 +Q 288 240.839538 288 248.869919 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 290 244.869919 +L 288 248.869919 +L 286 244.869919 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="patch_39"> + <path d="M 460.44624 197.466987 +Q 410.399379 223.920328 361.489234 249.772834 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 365.960228 249.671794 +L 361.489234 249.772834 +L 364.090999 246.135414 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="text_19"> + <!-- matmul $S = q\,k^{\top} \cdot 1/\sqrt{h_d}$ --> + <g style="fill: #3a3a3a" transform="translate(234.336 266.756)"> + <text> + <tspan x="0" y="-0.969938" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="8.401794" y="-0.969938" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="13.687134" y="-0.969938" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="17.068909" y="-0.969938" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="25.470703" y="-0.969938" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="30.937134" y="-0.969938" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="33.333435" y="-0.969938" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="36.075073" y="-0.969938" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="45.971924" y="-0.969938" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="76.95822" y="-0.969938" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">⋅</tspan> + <tspan x="81.380217" y="-0.969938" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">1</tspan> + <tspan x="86.867705" y="-0.969938" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">/</tspan> + <tspan x="38.816711" y="-0.969938" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">S</tspan> + <tspan x="54.879089" y="-0.969938" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">q</tspan> + <tspan x="61.75427" y="-0.969938" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">k</tspan> + <tspan x="96.926332" y="-0.61875" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="68.596288" y="-4.41525" style="font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">⊤</tspan> + <tspan x="89.773589" y="-1.1715" style="font-size: 5.73953px; font-family: 'STIXSizeOneSym'; fill: #3a3a3a">√</tspan> + <tspan x="102.392763" y="0.857812" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + </text> + <rect x="95.851332" y="-10.925" width="11.694922" height="0.5375"/> + </g> + </g> + <g id="text_20"> + <!-- $(B,H,T,T)$ --> + <g style="fill: #8a8a8a" transform="translate(414 264.897125)"> + <text> + <tspan x="0" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">(</tspan> + <tspan x="8.205811" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">,</tspan> + <tspan x="17.848755" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">,</tspan> + <tspan x="26.41571" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">,</tspan> + <tspan x="34.982666" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">)</tspan> + <tspan x="2.974792" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">B</tspan> + <tspan x="12.115112" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">H</tspan> + <tspan x="21.758057" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">T</tspan> + <tspan x="30.325012" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">T</tspan> + </text> + </g> + </g> + <g id="patch_40"> + <path d="M 288 274.718145 +Q 288 279.71972 288 283.435557 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 290 279.435557 +L 288 283.435557 +L 286 279.435557 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="text_21"> + <text style="font-size: 8.6px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="288" y="299.013062" transform="rotate(-0 288 299.013062)">causal mask</text> + </g> + <g id="patch_41"> + <path d="M 288 308.558145 +Q 288 313.55972 288 317.275557 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 290 313.275557 +L 288 317.275557 +L 286 313.275557 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="text_22"> + <text style="font-size: 8.6px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="288" y="332.853062" transform="rotate(-0 288 332.853062)">softmax over keys</text> + </g> + <g id="patch_42"> + <path d="M 288 342.398145 +Q 288 347.39972 288 351.115557 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 290 347.115557 +L 288 351.115557 +L 286 347.115557 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="text_23"> + <!-- matmul $y = A\,v$ --> + <g style="fill: #3a3a3a" transform="translate(254.632 366.685)"> + <text> + <tspan x="0" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="8.401794" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="13.687134" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="17.068909" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="25.470703" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="30.937134" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="33.333435" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="36.075073" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="45.601318" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="38.816711" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">y</tspan> + <tspan x="54.508484" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">A</tspan> + <tspan x="61.809018" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">v</tspan> + </text> + </g> + </g> + <g id="text_24"> + <!-- $(B,H,T,h_d)$ --> + <g style="fill: #8a8a8a" transform="translate(414 366.417125)"> + <text> + <tspan x="0" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">(</tspan> + <tspan x="8.205811" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">,</tspan> + <tspan x="17.848755" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">,</tspan> + <tspan x="26.41571" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">,</tspan> + <tspan x="38.764465" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">)</tspan> + <tspan x="2.974792" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">B</tspan> + <tspan x="12.115112" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">H</tspan> + <tspan x="21.758057" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">T</tspan> + <tspan x="30.325012" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">h</tspan> + <tspan x="35.157654" y="0.390625" style="font-style: oblique; font-size: 5.32px; font-family: 'DejaVu Sans'; fill: #8a8a8a">d</tspan> + </text> + </g> + </g> + <g id="patch_43"> + <path d="M 288 376.238145 +Q 288 381.23972 288 384.955557 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 290 380.955557 +L 288 384.955557 +L 286 380.955557 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="text_25"> + <text style="font-size: 8.6px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="288" y="400.533062" transform="rotate(-0 288 400.533062)">merge heads</text> + </g> + <g id="text_26"> + <!-- $(B,T,C)$ --> + <g style="fill: #8a8a8a" transform="translate(414 400.257125)"> + <text> + <tspan x="0" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">(</tspan> + <tspan x="8.205811" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">,</tspan> + <tspan x="16.772766" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">,</tspan> + <tspan x="26.006165" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">)</tspan> + <tspan x="2.974792" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">B</tspan> + <tspan x="12.115112" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">T</tspan> + <tspan x="20.682068" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">C</tspan> + </text> + </g> + </g> + <g id="patch_44"> + <path d="M 288 410.078145 +Q 288 415.07972 288 418.795557 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 290 414.795557 +L 288 418.795557 +L 286 414.795557 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="text_27"> + <!-- matmul $W_{proj}: C \to C$ --> + <g style="fill: #3a3a3a" transform="translate(244.011 433.935)"> + <text> + <tspan x="0" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="8.401794" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="13.687134" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="17.068909" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="25.470703" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="30.937134" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="33.333435" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="36.075073" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="60.957166" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">:</tspan> + <tspan x="73.246106" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">→</tspan> + <tspan x="38.816711" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">W</tspan> + <tspan x="65.543408" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">C</tspan> + <tspan x="82.153271" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">C</tspan> + <tspan x="47.344849" y="1.320312" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="51.177246" y="1.320312" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="53.65946" y="1.320312" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="57.353302" y="1.320312" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">j</tspan> + </text> + </g> + </g> + <g id="patch_45"> + <path d="M 288 443.918145 +Q 288 448.91972 288 452.635557 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 290 448.635557 +L 288 452.635557 +L 286 448.635557 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="text_28"> + <!-- RMSNorm$_{attn}$ --> + <g style="fill: #3a3a3a" transform="translate(260.48 468.213062)"> + <text> + <tspan x="0" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">R</tspan> + <tspan x="5.992859" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">M</tspan> + <tspan x="13.434448" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">S</tspan> + <tspan x="18.909302" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">N</tspan> + <tspan x="25.361206" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="30.638123" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="34.184143" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="42.67207" y="1.164062" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="46.371808" y="1.164062" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="48.73905" y="1.164062" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="51.106293" y="1.164062" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + </text> + </g> + </g> + <g id="text_29"> + <text style="font-size: 7.6px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; fill: #3a3a3a" transform="translate(414 463.681956)">◄ OLMo2: norm on the sublayer OUTPUT,</text> + <text style="font-size: 7.6px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; fill: #3a3a3a" transform="translate(414 472.192294)"> not on its input</text> + </g> + <g id="patch_46"> + <path d="M 288 477.757441 +Q 288 484.199288 288 489.355395 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 290 485.355395 +L 288 489.355395 +L 286 485.355395 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="patch_47"> + <path d="M 288 519.12 +C 291.627978 519.12 295.107853 517.678589 297.673221 515.113221 +C 300.238589 512.547853 301.68 509.067978 301.68 505.44 +C 301.68 501.812022 300.238589 498.332147 297.673221 495.766779 +C 295.107853 493.201411 291.627978 491.76 288 491.76 +C 284.372022 491.76 280.892147 493.201411 278.326779 495.766779 +C 275.761411 498.332147 274.32 501.812022 274.32 505.44 +C 274.32 509.067978 275.761411 512.547853 278.326779 515.113221 +C 280.892147 517.678589 284.372022 519.12 288 519.12 +z +" clip-path="url(#pdba9adb895)" style="fill: #ffffff; stroke: #3a3a3a; stroke-width: 1.3; stroke-linejoin: miter"/> + </g> + <g id="text_30"> + <text style="font-weight: 700; font-size: 11px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="288" y="508.475312" transform="rotate(-0 288 508.475312)">+</text> + </g> + <g id="patch_48"> + <path d="M 288 519.517617 +Q 288 528.47975 288 536.156143 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 290 532.156143 +L 288 536.156143 +L 286 532.156143 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="text_31"> + <!-- $z'$ --> + <g style="fill: #3a3a3a" transform="translate(303.84 530.519375)"> + <text> + <tspan x="0" y="-0.25625" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="5.695638" y="-4.084375" style="font-size: 7px; font-family: 'cmsy10'; fill: #3a3a3a">0</tspan> + </text> + </g> + </g> + <g id="text_32"> + <text style="font-style: italic; font-size: 9.5px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; fill: #8a8a8a" transform="translate(25.661406 702.350469) rotate(-90)">SwiGLU sublayer</text> + </g> + <g id="patch_49"> + <path d="M 633.19823 750.24524 +Q 467.999756 752.400003 304.366397 754.534351 +" style="fill: none; stroke: #555555; stroke-width: 1.4; stroke-linecap: round"/> + <path d="M 308.392142 756.482012 +L 304.366397 754.534351 +L 308.339972 752.482352 +z +" style="fill: #555555; stroke: #555555; stroke-width: 1.4; stroke-linecap: round"/> + </g> + <g id="patch_50"> + <path d="M 273.208447 544.40543 +Q 233.999032 552.960211 196.045804 561.240915 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 200.380202 562.342279 +L 196.045804 561.240915 +L 199.527533 558.434216 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="patch_51"> + <path d="M 302.791553 544.40543 +Q 342.000968 552.960211 379.954196 561.240915 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 376.472467 558.434216 +L 379.954196 561.240915 +L 375.619798 562.342279 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="text_33"> + <!-- matmul $W_1: C \to h_{ff}$ --> + <g style="fill: #3a3a3a" transform="translate(145.189 575.493062)"> + <text> + <tspan x="0" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="8.401794" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="13.687134" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="17.068909" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="25.470703" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="30.937134" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="33.333435" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="36.075073" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="53.112543" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">:</tspan> + <tspan x="65.401483" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">→</tspan> + <tspan x="38.816711" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">W</tspan> + <tspan x="57.698785" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">C</tspan> + <tspan x="74.308649" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="47.344849" y="1.320312" style="font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">1</tspan> + <tspan x="79.775079" y="1.320312" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="81.900586" y="1.320312" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + </text> + </g> + </g> + <g id="text_34"> + <!-- matmul $W_3: C \to h_{ff}$ --> + <g style="fill: #3a3a3a" transform="translate(346.789 575.493062)"> + <text> + <tspan x="0" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="8.401794" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="13.687134" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="17.068909" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="25.470703" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="30.937134" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="33.333435" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="36.075073" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="53.112543" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">:</tspan> + <tspan x="65.401483" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">→</tspan> + <tspan x="38.816711" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">W</tspan> + <tspan x="57.698785" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">C</tspan> + <tspan x="74.308649" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="47.344849" y="1.320312" style="font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">3</tspan> + <tspan x="79.775079" y="1.320312" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="81.900586" y="1.320312" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + </text> + </g> + </g> + <g id="text_35"> + <!-- $h_{ff} = 1408$ --> + <g style="fill: #8a8a8a" transform="translate(473.04 575.217125)"> + <text> + <tspan x="0" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">h</tspan> + <tspan x="4.832642" y="0.390625" style="font-style: oblique; font-size: 5.32px; font-family: 'DejaVu Sans'; fill: #8a8a8a">f</tspan> + <tspan x="6.711713" y="0.390625" style="font-style: oblique; font-size: 5.32px; font-family: 'DejaVu Sans'; fill: #8a8a8a">f</tspan> + <tspan x="10.295068" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">=</tspan> + <tspan x="18.169519" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">1</tspan> + <tspan x="23.020776" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">4</tspan> + <tspan x="27.872034" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">0</tspan> + <tspan x="32.723291" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">8</tspan> + </text> + </g> + </g> + <g id="patch_52"> + <path d="M 187.2 585.038145 +Q 187.2 590.03972 187.2 593.755557 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 189.2 589.755557 +L 187.2 593.755557 +L 185.2 589.755557 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="text_36"> + <text style="font-size: 8.6px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="187.2" y="609.333062" transform="rotate(-0 187.2 609.333062)">SiLU</text> + </g> + <g id="patch_53"> + <path d="M 187.58562 618.582832 +Q 214.198105 625.679495 239.568263 632.44487 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 236.218648 629.48175 +L 239.568263 632.44487 +L 235.187998 633.346689 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="patch_54"> + <path d="M 388.500073 584.907935 +Q 361.799167 608.760744 336.057115 631.756977 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 340.372587 630.583646 +L 336.057115 631.756977 +L 337.707733 627.600601 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="text_37"> + <text style="font-size: 8.6px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="288" y="647.493062" transform="rotate(-0 288 647.493062)">⊙ elementwise gate</text> + </g> + <g id="patch_55"> + <path d="M 288 657.038145 +Q 288 662.03972 288 665.755557 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 290 661.755557 +L 288 665.755557 +L 286 661.755557 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="text_38"> + <!-- matmul $W_2: h_{ff} \to C$ --> + <g style="fill: #3a3a3a" transform="translate(245.989 685.653063)"> + <text> + <tspan x="0" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="8.401794" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="13.687134" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="17.068909" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="25.470703" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="30.937134" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="33.333435" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="36.075073" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="53.112543" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">:</tspan> + <tspan x="69.342682" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">→</tspan> + <tspan x="38.816711" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">W</tspan> + <tspan x="57.698785" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="78.249847" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">C</tspan> + <tspan x="47.344849" y="1.320312" style="font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">2</tspan> + <tspan x="63.165216" y="1.320312" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="65.290723" y="1.320312" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + </text> + </g> + </g> + <g id="patch_56"> + <path d="M 288 695.198145 +Q 288 700.19972 288 703.915557 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 290 699.915557 +L 288 703.915557 +L 286 699.915557 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="text_39"> + <!-- RMSNorm$_{ff}$ --> + <g style="fill: #3a3a3a" transform="translate(264.479 719.493062)"> + <text> + <tspan x="0" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">R</tspan> + <tspan x="5.992859" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">M</tspan> + <tspan x="13.434448" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">S</tspan> + <tspan x="18.909302" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">N</tspan> + <tspan x="25.361206" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="30.638123" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="34.184143" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="42.67207" y="1.164062" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="44.797577" y="1.164062" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + </text> + </g> + </g> + <g id="text_40"> + <text style="font-size: 7.6px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: start; fill: #3a3a3a" x="388.8" y="719.217125" transform="rotate(-0 388.8 719.217125)">◄ norm after the sublayer, again</text> + </g> + <g id="patch_57"> + <path d="M 288 729.042187 +Q 288 734.401592 288 738.475257 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 290 734.475257 +L 288 738.475257 +L 286 734.475257 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="patch_58"> + <path d="M 288 768.24 +C 291.627978 768.24 295.107853 766.798589 297.673221 764.233221 +C 300.238589 761.667853 301.68 758.187978 301.68 754.56 +C 301.68 750.932022 300.238589 747.452147 297.673221 744.886779 +C 295.107853 742.321411 291.627978 740.88 288 740.88 +C 284.372022 740.88 280.892147 742.321411 278.326779 744.886779 +C 275.761411 747.452147 274.32 750.932022 274.32 754.56 +C 274.32 758.187978 275.761411 761.667853 278.326779 764.233221 +C 280.892147 766.798589 284.372022 768.24 288 768.24 +z +" clip-path="url(#pdba9adb895)" style="fill: #ffffff; stroke: #3a3a3a; stroke-width: 1.3; stroke-linejoin: miter"/> + </g> + <g id="text_41"> + <text style="font-weight: 700; font-size: 11px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="288" y="757.595312" transform="rotate(-0 288 757.595312)">+</text> + </g> + <g id="patch_59"> + <path d="M 288 768.639902 +Q 288 777.240145 288 784.554649 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 290 780.554649 +L 288 784.554649 +L 286 780.554649 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="text_42"> + <!-- output $z^{\prime\prime}$ — one full $f_l$ done --> + <g style="fill: #3a3a3a" transform="translate(212.8075 799.7475)"> + <text> + <tspan x="0" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="6.526611" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="13.289795" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="17.831055" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="24.631348" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="31.394531" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="35.935791" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="48.60599" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="51.913363" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="64.720736" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="68.028109" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="71.335482" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="77.862093" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="84.625277" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="91.068392" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="94.375765" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="98.508822" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="105.272005" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="108.528353" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="111.784701" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="120.557576" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="123.86495" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="130.665243" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="137.191854" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="143.955037" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="39.243164" y="-0.40625" style="font-style: oblique; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="115.092074" y="-0.40625" style="font-style: oblique; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="44.676351" y="-4.234375" style="font-size: 6.65px; font-family: 'cmsy10'; fill: #3a3a3a">0</tspan> + <tspan x="46.504451" y="-4.234375" style="font-size: 6.65px; font-family: 'cmsy10'; fill: #3a3a3a">0</tspan> + <tspan x="55.220736" y="-0.40625" style="font-size: 9.5px; font-family: 'DejaVu Sans Display'; fill: #3a3a3a">—</tspan> + <tspan x="118.436556" y="1.234375" style="font-style: oblique; font-size: 6.65px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + </text> + </g> + </g> + <g id="text_43"> + <!-- weight × activation matmul — 7 logical ($W_{qkv}$ fuses $W_q,W_k,W_v$); crossbar-mappable on analog hardware --> + <g style="fill: #3a3a3a" transform="translate(68.4 840.645)"> + <text> + <tspan x="0" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">w</tspan> + <tspan x="6.747437" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="11.82312" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="14.115234" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">g</tspan> + <tspan x="19.352051" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="24.580811" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="27.815552" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="30.437988" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">×</tspan> + <tspan x="37.350586" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="39.973022" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="45.028564" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="49.564453" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="52.799194" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="55.091309" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">v</tspan> + <tspan x="59.973633" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="65.029175" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="68.263916" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="70.55603" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="75.603516" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="80.832275" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="83.454712" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="91.491211" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="96.546753" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="99.781494" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="107.817993" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="113.046753" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="115.338867" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="126.211304" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="128.83374" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">7</tspan> + <tspan x="134.082642" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="136.705078" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="138.997192" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="144.044678" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">g</tspan> + <tspan x="149.281494" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="151.573608" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="156.109497" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="161.165039" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="163.457153" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="166.07959" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="188.102026" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="190.724463" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="193.628882" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="198.857642" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="203.155859" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="208.231543" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="212.529761" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="227.194067" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">,</tspan> + <tspan x="243.144214" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">,</tspan> + <tspan x="259.167676" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="262.386304" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">;</tspan> + <tspan x="265.165845" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="267.788281" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="272.32417" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="275.716016" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="280.763501" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="285.061719" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="289.359937" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">b</tspan> + <tspan x="294.596753" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="299.652295" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="303.044141" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">-</tspan> + <tspan x="305.896069" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="313.932568" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="318.98811" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="324.224927" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="329.461743" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="334.517285" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">b</tspan> + <tspan x="339.754102" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="342.046216" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="347.121899" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="349.744336" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="354.791821" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="360.020581" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="362.643018" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="367.69856" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="372.927319" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="377.982861" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="380.274976" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="385.322461" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">g</tspan> + <tspan x="390.559277" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="393.181714" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="398.410474" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="403.466016" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="406.857861" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="412.094678" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">w</tspan> + <tspan x="418.842114" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="423.897656" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="427.289502" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="117.961304" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans Display'; fill: #3a3a3a">—</tspan> + <tspan x="169.298218" y="-0.921875" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">W</tspan> + <tspan x="215.152197" y="-0.921875" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">W</tspan> + <tspan x="231.423804" y="-0.921875" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">W</tspan> + <tspan x="247.37395" y="-0.921875" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">W</tspan> + <tspan x="177.455566" y="0.390625" style="font-style: oblique; font-size: 5.74px; font-family: 'DejaVu Sans'; fill: #3a3a3a">q</tspan> + <tspan x="181.121338" y="0.390625" style="font-style: oblique; font-size: 5.74px; font-family: 'DejaVu Sans'; fill: #3a3a3a">k</tspan> + <tspan x="184.465649" y="0.390625" style="font-style: oblique; font-size: 5.74px; font-family: 'DejaVu Sans'; fill: #3a3a3a">v</tspan> + <tspan x="223.309546" y="0.390625" style="font-style: oblique; font-size: 5.74px; font-family: 'DejaVu Sans'; fill: #3a3a3a">q</tspan> + <tspan x="239.581152" y="0.390625" style="font-style: oblique; font-size: 5.74px; font-family: 'DejaVu Sans'; fill: #3a3a3a">k</tspan> + <tspan x="255.531299" y="0.390625" style="font-style: oblique; font-size: 5.74px; font-family: 'DejaVu Sans'; fill: #3a3a3a">v</tspan> + </text> + </g> + </g> + <g id="text_44"> + <!-- activation × activation matmul — 2 ($q\,k^{\top}$ and $A\,v$); computed on the fly: the non-crossbar part of attention --> + <g style="fill: #3a3a3a" transform="translate(68.4 866.027)"> + <text> + <tspan x="0" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="5.055542" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="9.591431" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="12.826172" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="15.118286" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">v</tspan> + <tspan x="20.00061" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="25.056152" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="28.290894" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="30.583008" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="35.630493" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="40.859253" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="43.481689" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">×</tspan> + <tspan x="50.394287" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="53.016724" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="58.072266" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="62.608154" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="65.842896" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="68.13501" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">v</tspan> + <tspan x="73.017334" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="78.072876" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="81.307617" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="83.599731" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="88.647217" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="93.875977" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="96.498413" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="104.534912" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="109.590454" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="112.825195" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="120.861694" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="126.090454" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="128.382568" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="139.255005" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="141.877441" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">2</tspan> + <tspan x="147.126343" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="149.748779" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="172.41652" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="175.038957" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="180.094499" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="185.323258" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="190.560075" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="205.047956" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="208.266584" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">;</tspan> + <tspan x="211.046125" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="213.668561" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="218.20445" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="223.251935" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="231.288434" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="236.525251" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="241.75401" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="244.988751" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="250.064435" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="255.301251" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="257.923688" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="262.971173" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="268.199933" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="270.82237" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="274.057111" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="279.285871" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="284.361554" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="286.983991" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="289.88841" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="292.180524" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">y</tspan> + <tspan x="297.062848" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">:</tspan> + <tspan x="299.842389" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="302.464826" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="305.699567" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="310.928327" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="316.00401" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="318.626447" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="323.855207" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="328.902692" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="334.131452" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">-</tspan> + <tspan x="337.10838" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="341.644269" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="345.036115" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="350.0836" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="354.381818" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="358.680036" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">b</tspan> + <tspan x="363.916852" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="368.972394" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="372.36424" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="374.986676" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="380.223493" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="385.279035" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="388.67088" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="391.905622" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="394.528058" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="399.575543" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="402.479962" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="405.102399" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="410.157941" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="413.392682" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="416.627423" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="421.703107" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="426.931867" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="430.166608" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="432.458722" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="437.506208" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="131.005005" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans Display'; fill: #3a3a3a">—</tspan> + <tspan x="152.967407" y="-0.010938" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">q</tspan> + <tspan x="159.543667" y="-0.010938" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">k</tspan> + <tspan x="193.182511" y="-0.010938" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">A</tspan> + <tspan x="200.165631" y="-0.010938" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">v</tspan> + <tspan x="166.042094" y="-3.073438" style="font-size: 5.74px; font-family: 'DejaVu Sans'; fill: #3a3a3a">⊤</tspan> + </text> + </g> + </g> + <g id="text_45"> + <text style="font-size: 8.2px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: start; fill: #3a3a3a" x="68.4" y="890.022687" transform="rotate(-0 68.4 890.022687)">RMSNorm (learned gain)</text> + </g> + <g id="text_46"> + <text style="font-size: 8.2px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: start; fill: #3a3a3a" x="306" y="890.022687" transform="rotate(-0 306 890.022687)">elementwise / reshape</text> + </g> + <g id="text_47"> + <!-- No biases anywhere in the network. The $1/\sqrt{h_d}$ scale is folded into $S$; mask and softmax run along the key axis. --> + <g style="fill: #3a3a3a" transform="translate(36 921.052)"> + <text> + <tspan x="0" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">N</tspan> + <tspan x="6.171387" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="11.218872" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="13.841309" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">b</tspan> + <tspan x="19.078125" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="21.370239" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="26.425781" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="30.723999" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="35.799683" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="40.0979" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="42.720337" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="47.775879" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="53.004639" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">y</tspan> + <tspan x="57.886963" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">w</tspan> + <tspan x="64.634399" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="69.863159" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="74.938843" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="78.330688" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="83.406372" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="86.028809" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="88.320923" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="93.549683" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="96.172119" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="99.40686" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="104.63562" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="109.711304" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="112.33374" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="117.5625" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="122.638184" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="125.872925" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">w</tspan> + <tspan x="132.620361" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="137.667847" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="141.059692" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">k</tspan> + <tspan x="145.83728" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">.</tspan> + <tspan x="148.459717" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="151.082153" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="153.70459" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">T</tspan> + <tspan x="158.744019" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="163.972778" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="169.048462" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="171.670898" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">1</tspan> + <tspan x="176.9198" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">/</tspan> + <tspan x="196.808255" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="199.430692" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="203.728909" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="208.264798" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="213.32034" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="215.612454" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="220.688138" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="223.310574" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="225.602689" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="229.900906" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="232.523343" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="235.427762" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="240.475247" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="242.767361" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="248.004178" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="253.079861" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="258.316678" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="260.939114" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="263.231229" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="268.459988" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="271.69473" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="276.742215" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="284.601468" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">;</tspan> + <tspan x="287.381009" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="290.003445" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="298.039944" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="303.095486" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="307.393704" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">k</tspan> + <tspan x="312.171292" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="314.793729" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="319.849271" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="325.07803" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="330.314847" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="332.937283" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="337.235501" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="342.282986" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="345.187405" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="348.422147" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="356.458646" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="361.514188" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">x</tspan> + <tspan x="366.396512" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="369.018948" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="372.410794" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="377.639554" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="382.868314" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="385.49075" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="390.546292" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="392.838406" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="397.885892" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="403.114651" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">g</tspan> + <tspan x="408.351468" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="410.973904" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="414.208646" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="419.437405" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="424.513089" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="427.135526" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">k</tspan> + <tspan x="431.913113" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="436.988797" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">y</tspan> + <tspan x="441.871121" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="444.493558" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="449.5491" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">x</tspan> + <tspan x="454.431424" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="456.723538" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="461.021756" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">.</tspan> + <tspan x="179.699341" y="-1.1955" style="font-size: 5.564038px; font-family: 'STIXSizeOneSym'; fill: #3a3a3a">√</tspan> + <tspan x="186.669974" y="-0.409375" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="279.364651" y="-0.818938" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">S</tspan> + <tspan x="191.898734" y="0.903125" style="font-style: oblique; font-size: 5.74px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + </text> + <rect x="185.644974" y="-10.975" width="11.163281" height="0.5125"/> + </g> + </g> + <g id="text_48"> + <!-- $h_{ff} = 1408 \approx 8C/3$, rounded up to a multiple of 64. QK-norm acts on the full width $C$, before the head split. --> + <g style="fill: #3a3a3a" transform="translate(36 938.975)"> + <text> + <tspan x="0" y="-0.921875" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="56.012598" y="-0.921875" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">C</tspan> + <tspan x="337.75332" y="-0.921875" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">C</tspan> + <tspan x="5.22876" y="0.390625" style="font-style: oblique; font-size: 5.74px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="7.261853" y="0.390625" style="font-style: oblique; font-size: 5.74px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="11.120996" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="19.640894" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">1</tspan> + <tspan x="24.889795" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">4</tspan> + <tspan x="30.138696" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">0</tspan> + <tspan x="35.387598" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">8</tspan> + <tspan x="42.243799" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">≈</tspan> + <tspan x="50.763696" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">8</tspan> + <tspan x="61.773096" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">/</tspan> + <tspan x="64.552637" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">3</tspan> + <tspan x="69.801538" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">,</tspan> + <tspan x="72.423975" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="75.046411" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="78.438257" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="83.485742" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="88.714502" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="93.943262" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="99.180078" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="104.255762" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="109.492578" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="112.115015" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="117.343774" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="122.580591" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="125.203027" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="128.437769" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="133.485254" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="136.10769" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="141.163232" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="143.785669" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="151.822168" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="157.050928" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="159.343042" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="162.577783" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="164.869897" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="170.106714" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="172.398828" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="177.474512" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="180.096948" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="185.144434" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="188.048853" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="190.671289" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">6</tspan> + <tspan x="195.92019" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">4</tspan> + <tspan x="201.169092" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">.</tspan> + <tspan x="203.791528" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="206.413965" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="209.036401" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">Q</tspan> + <tspan x="215.530054" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">K</tspan> + <tspan x="220.940088" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">-</tspan> + <tspan x="223.917017" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="229.145776" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="234.193262" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="237.585107" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="245.621606" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="248.244043" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="253.299585" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="257.835474" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="261.070215" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="265.368433" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="267.990869" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="273.038354" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="278.267114" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="280.889551" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="284.124292" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="289.353052" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="294.428735" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="297.051172" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="299.955591" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="305.184351" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="307.476465" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="309.768579" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="312.391016" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">w</tspan> + <tspan x="319.138452" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="321.430566" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="326.667383" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="329.902124" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="335.130884" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="343.513818" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">,</tspan> + <tspan x="346.136255" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="348.758691" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">b</tspan> + <tspan x="353.995508" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="359.071191" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="361.97561" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="367.023096" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="370.414941" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="375.490625" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="378.113062" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="381.347803" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="386.576563" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="391.652246" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="394.274683" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="399.503442" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="404.579126" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="409.634668" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="414.871484" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="417.493921" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="421.792139" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="427.028955" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="429.321069" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="431.613184" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="434.847925" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">.</tspan> + </text> + </g> + </g> + <g id="text_49"> + <!-- The boxed pipeline, input $z$ to output $z^{\prime\prime}$, is exactly $f_l$ on the companion page; the state $z_l$ lives on the residual rail. --> + <g style="fill: #3a3a3a" transform="translate(36 958.497)"> + <text> + <tspan x="0" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">T</tspan> + <tspan x="5.039429" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="10.268188" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="15.343872" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="17.966309" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">b</tspan> + <tspan x="23.203125" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="28.25061" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">x</tspan> + <tspan x="33.132935" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="38.208618" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="43.445435" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="46.067871" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="51.304688" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="53.596802" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="58.833618" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="63.909302" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="66.201416" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="68.49353" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="73.72229" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="78.797974" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">,</tspan> + <tspan x="81.42041" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="84.042847" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="86.334961" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="91.563721" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="96.800537" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="102.029297" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="105.264038" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="112.216919" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="114.839355" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="118.074097" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="123.121582" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="125.744019" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="130.791504" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="136.020264" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="139.255005" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="144.491821" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="149.720581" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="152.955322" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="163.635313" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">,</tspan> + <tspan x="166.257749" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="168.880186" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="171.1723" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="175.470518" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="178.092955" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="183.168638" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">x</tspan> + <tspan x="188.050962" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="193.106504" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="197.642393" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="200.877134" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="203.169248" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">y</tspan> + <tspan x="208.051573" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="215.401658" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="218.024095" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="223.07158" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="228.30034" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="230.922776" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="234.157517" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="239.386277" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="244.461961" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="247.084397" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="251.620286" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="256.667771" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="264.70427" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="269.941087" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="274.996629" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="280.225389" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="282.517503" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="287.564988" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="292.793748" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="295.416184" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="300.653001" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="305.708543" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">g</tspan> + <tspan x="310.945359" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="316.021043" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">;</tspan> + <tspan x="318.800584" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="321.42302" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="324.657762" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="329.886521" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="334.962205" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="337.584642" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="341.882859" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="345.117601" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="350.173142" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="353.407884" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="358.483567" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="367.259678" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="369.882115" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="372.174229" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="374.466343" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">v</tspan> + <tspan x="379.348667" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="384.424351" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="388.722569" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="391.345005" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="396.392491" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="401.62125" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="404.243687" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="407.478428" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="412.707188" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="417.782872" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="420.405308" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="423.797154" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="428.872837" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="433.171055" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="435.463169" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="440.699986" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="445.928746" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="450.984288" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="453.276402" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="455.898838" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="459.290684" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="464.346226" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="466.63834" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="468.930455" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">.</tspan> + <tspan x="107.886475" y="-0.732812" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="155.577759" y="-0.732812" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="210.674009" y="-0.732812" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="361.106004" y="-0.732812" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="160.265495" y="-3.795312" style="font-size: 5.74px; font-family: 'cmsy10'; fill: #3a3a3a">0</tspan> + <tspan x="161.841029" y="-3.795312" style="font-size: 5.74px; font-family: 'cmsy10'; fill: #3a3a3a">0</tspan> + <tspan x="213.578428" y="0.579688" style="font-style: oblique; font-size: 5.74px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="365.436448" y="0.579688" style="font-style: oblique; font-size: 5.74px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + </text> + </g> + </g> + </g> + </g> + <defs> + <clipPath id="pdba9adb895"> + <rect x="0" y="0" width="720" height="1022.4"/> + </clipPath> + </defs> +</svg> diff --git a/assets/figs/fig_primer_phases.emf b/assets/figs/fig_primer_phases.emf Binary files differnew file mode 100644 index 0000000..c28c459 --- /dev/null +++ b/assets/figs/fig_primer_phases.emf diff --git a/assets/figs/fig_primer_phases.png b/assets/figs/fig_primer_phases.png Binary files differnew file mode 100644 index 0000000..a6db577 --- /dev/null +++ b/assets/figs/fig_primer_phases.png diff --git a/assets/figs/fig_primer_phases.svg b/assets/figs/fig_primer_phases.svg new file mode 100644 index 0000000..da9ed6a --- /dev/null +++ b/assets/figs/fig_primer_phases.svg @@ -0,0 +1,925 @@ +<?xml version="1.0" encoding="utf-8" standalone="no"?> +<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" + "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"> +<svg xmlns:xlink="http://www.w3.org/1999/xlink" width="878.4pt" height="792pt" viewBox="0 0 878.4 792" xmlns="http://www.w3.org/2000/svg" version="1.1"> + <metadata> + <rdf:RDF xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:cc="http://creativecommons.org/ns#" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"> + <cc:Work> + <dc:type rdf:resource="http://purl.org/dc/dcmitype/StillImage"/> + <dc:date>2026-07-15T09:44:53.517354</dc:date> + <dc:format>image/svg+xml</dc:format> + <dc:creator> + <cc:Agent> + <dc:title>Matplotlib v3.10.8, https://matplotlib.org/</dc:title> + </cc:Agent> + </dc:creator> + </cc:Work> + </rdf:RDF> + </metadata> + <defs> + <style type="text/css">*{stroke-linejoin: round; stroke-linecap: butt}</style> + </defs> + <g id="figure_1"> + <g id="patch_1"> + <path d="M 0 792 +L 878.4 792 +L 878.4 0 +L 0 0 +z +" style="fill: #ffffff"/> + </g> + <g id="axes_1"> + <g id="patch_2"> + <path d="M 77.7384 199.779775 +L 256.0536 199.779775 +Q 261.324 199.779775 261.324 194.440449 +L 261.324 161.514607 +Q 261.324 156.175281 256.0536 156.175281 +L 77.7384 156.175281 +Q 72.468 156.175281 72.468 161.514607 +L 72.468 194.440449 +Q 72.468 199.779775 77.7384 199.779775 +z +" clip-path="url(#p90a79555a2)" style="fill: #eaf1fa; stroke: #2c6fbb; stroke-width: 1.2; stroke-linejoin: miter"/> + </g> + <g id="patch_3"> + <path d="M 77.7384 306.566292 +L 256.0536 306.566292 +Q 261.324 306.566292 261.324 301.226966 +L 261.324 268.301124 +Q 261.324 262.961798 256.0536 262.961798 +L 77.7384 262.961798 +Q 72.468 262.961798 72.468 268.301124 +L 72.468 301.226966 +Q 72.468 306.566292 77.7384 306.566292 +z +" clip-path="url(#p90a79555a2)" style="fill: #eaf1fa; stroke: #2c6fbb; stroke-width: 1.2; stroke-linejoin: miter"/> + </g> + <g id="patch_4"> + <path d="M 77.7384 413.352809 +L 256.0536 413.352809 +Q 261.324 413.352809 261.324 408.013483 +L 261.324 375.08764 +Q 261.324 369.748315 256.0536 369.748315 +L 77.7384 369.748315 +Q 72.468 369.748315 72.468 375.08764 +L 72.468 408.013483 +Q 72.468 413.352809 77.7384 413.352809 +z +" clip-path="url(#p90a79555a2)" style="fill: #eaf1fa; stroke: #2c6fbb; stroke-width: 1.2; stroke-linejoin: miter"/> + </g> + <g id="patch_5"> + <path d="M 473.0184 199.779775 +L 651.3336 199.779775 +Q 656.604 199.779775 656.604 194.440449 +L 656.604 161.514607 +Q 656.604 156.175281 651.3336 156.175281 +L 473.0184 156.175281 +Q 467.748 156.175281 467.748 161.514607 +L 467.748 194.440449 +Q 467.748 199.779775 473.0184 199.779775 +z +" clip-path="url(#p90a79555a2)" style="fill: #fdeee2; stroke: #d95f02; stroke-width: 1.2; stroke-linejoin: miter"/> + </g> + <g id="patch_6"> + <path d="M 473.0184 306.566292 +L 651.3336 306.566292 +Q 656.604 306.566292 656.604 301.226966 +L 656.604 268.301124 +Q 656.604 262.961798 651.3336 262.961798 +L 473.0184 262.961798 +Q 467.748 262.961798 467.748 268.301124 +L 467.748 301.226966 +Q 467.748 306.566292 473.0184 306.566292 +z +" clip-path="url(#p90a79555a2)" style="fill: #fdeee2; stroke: #d95f02; stroke-width: 1.2; stroke-linejoin: miter"/> + </g> + <g id="patch_7"> + <path d="M 473.0184 413.352809 +L 651.3336 413.352809 +Q 656.604 413.352809 656.604 408.013483 +L 656.604 375.08764 +Q 656.604 369.748315 651.3336 369.748315 +L 473.0184 369.748315 +Q 467.748 369.748315 467.748 375.08764 +L 467.748 408.013483 +Q 467.748 413.352809 473.0184 413.352809 +z +" clip-path="url(#p90a79555a2)" style="fill: #fdeee2; stroke: #d95f02; stroke-width: 1.2; stroke-linejoin: miter"/> + </g> + <g id="patch_8"> + <path d="M 20.6424 771.977528 +L 857.7576 771.977528 +Q 863.028 771.977528 863.028 766.638202 +L 863.028 639.38427 +Q 863.028 634.044944 857.7576 634.044944 +L 20.6424 634.044944 +Q 15.372 634.044944 15.372 639.38427 +L 15.372 766.638202 +Q 15.372 771.977528 20.6424 771.977528 +z +" clip-path="url(#p90a79555a2)" style="fill: #faf6ee; stroke: #c9b895; stroke-width: 1.2; stroke-linejoin: miter"/> + </g> + <g id="text_1"> + <text style="font-weight: 700; font-size: 11.5px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="219.6" y="27.200248" transform="rotate(-0 219.6 27.200248)">Free phase (= inference)</text> + </g> + <g id="text_2"> + <!-- give each layer a state $z_l$ (3 of $L$ drawn); settle the disagreement energy --> + <g style="fill: #8a8a8a" transform="translate(59.66 52.253708)"> + <text> + <tspan x="0" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">g</tspan> + <tspan x="5.554199" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="7.985229" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">v</tspan> + <tspan x="13.163452" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="18.546753" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="21.328125" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="26.711426" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="32.073364" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">c</tspan> + <tspan x="36.884155" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">h</tspan> + <tspan x="42.42981" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="45.211182" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">l</tspan> + <tspan x="47.642212" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="53.00415" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">y</tspan> + <tspan x="58.182373" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="63.565674" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="67.163086" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="69.944458" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="75.306396" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="78.087769" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="82.646484" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="86.077271" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="91.439209" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="94.869995" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="100.253296" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="109.575378" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="112.35675" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">(</tspan> + <tspan x="115.770447" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">3</tspan> + <tspan x="121.337463" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="124.118835" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">o</tspan> + <tspan x="129.472229" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">f</tspan> + <tspan x="132.552673" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="140.208923" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="142.990295" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">d</tspan> + <tspan x="148.544495" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="152.141907" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="157.503845" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">w</tspan> + <tspan x="164.660217" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">n</tspan> + <tspan x="170.205872" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">)</tspan> + <tspan x="173.619568" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">;</tspan> + <tspan x="176.567566" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="179.348938" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="183.907654" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="189.290955" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="192.721741" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="196.152527" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">l</tspan> + <tspan x="198.583557" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="203.966858" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="206.74823" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="210.179016" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">h</tspan> + <tspan x="215.72467" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="221.107971" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="223.889343" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">d</tspan> + <tspan x="229.443542" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="231.874573" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="236.433289" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="241.795227" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">g</tspan> + <tspan x="247.349426" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="250.946838" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="256.330139" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="261.71344" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">m</tspan> + <tspan x="270.237" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="275.6203" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">n</tspan> + <tspan x="281.165955" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="284.596741" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="287.378113" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="292.761414" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">n</tspan> + <tspan x="298.307068" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="303.690369" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="307.287781" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">g</tspan> + <tspan x="312.84198" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">y</tspan> + <tspan x="103.034668" y="-0.15625" style="font-style: oblique; font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">z</tspan> + <tspan x="135.334045" y="-0.15625" style="font-style: oblique; font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">L</tspan> + <tspan x="107.627563" y="1.320312" style="font-style: oblique; font-size: 6.16px; font-family: 'DejaVu Sans'; fill: #8a8a8a">l</tspan> + </text> + </g> + </g> + <g id="text_3"> + <!-- $E(z)\ =\ \sum_l\ \frac{1}{2}\,\|\,z_l - f_l(z_{l-1})\,\|^2$ --> + <g style="fill: #3a3a3a" transform="translate(145.0225 85.544213)"> + <text> + <tspan x="0" y="-0.796875" style="font-style: oblique; font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">E</tspan> + <tspan x="11.752686" y="-0.796875" style="font-style: oblique; font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="74.075475" y="-0.796875" style="font-style: oblique; font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="96.793224" y="-0.796875" style="font-style: oblique; font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="107.893053" y="-0.796875" style="font-style: oblique; font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="7.266113" y="-0.796875" style="font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="17.789062" y="-0.796875" style="font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="28.250207" y="-0.796875" style="font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="66.458373" y="-0.796875" style="font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">‖</tspan> + <tspan x="84.917004" y="-0.796875" style="font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">−</tspan> + <tspan x="103.406481" y="-0.796875" style="font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="131.497448" y="-0.796875" style="font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="137.851123" y="-0.796875" style="font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">‖</tspan> + <tspan x="43.860521" y="-0.796875" style="font-size: 11.5px; font-family: 'DejaVu Sans Display'; fill: #3a3a3a">∑</tspan> + <tspan x="47.860521" y="11.0125" style="font-style: oblique; font-size: 8.05px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="80.111852" y="1.171875" style="font-style: oblique; font-size: 8.05px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="100.841808" y="1.171875" style="font-style: oblique; font-size: 8.05px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="113.92943" y="1.171875" style="font-style: oblique; font-size: 8.05px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="58.032115" y="-5.992188" style="font-size: 8.05px; font-family: 'DejaVu Sans'; fill: #3a3a3a">1</tspan> + <tspan x="58.032115" y="3.835938" style="font-size: 8.05px; font-family: 'DejaVu Sans'; fill: #3a3a3a">2</tspan> + <tspan x="117.734313" y="1.171875" style="font-size: 8.05px; font-family: 'DejaVu Sans'; fill: #3a3a3a">−</tspan> + <tspan x="126.047668" y="1.171875" style="font-size: 8.05px; font-family: 'DejaVu Sans'; fill: #3a3a3a">1</tspan> + <tspan x="143.715967" y="-5.390625" style="font-size: 8.05px; font-family: 'DejaVu Sans'; fill: #3a3a3a">2</tspan> + </text> + <rect x="58.032115" y="-4.117188" width="5.121655" height="0.71875"/> + </g> + </g> + <g id="text_4"> + <!-- $z_3 = f_3(z_2)$ --> + <g style="fill: #3a3a3a" transform="translate(142.746 180.736903)"> + <text> + <tspan x="0" y="-0.40625" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="22.251465" y="-0.40625" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="34.400391" y="-0.40625" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="5.249023" y="1.234375" style="font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">3</tspan> + <tspan x="25.771973" y="1.234375" style="font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">3</tspan> + <tspan x="39.649414" y="1.234375" style="font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">2</tspan> + <tspan x="11.924316" y="-0.40625" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="30.499023" y="-0.40625" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="44.376465" y="-0.40625" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + </text> + </g> + </g> + <g id="text_5"> + <text style="font-size: 8.4px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: start; fill: #8a8a8a" x="274.0608" y="180.295403" transform="rotate(-0 274.0608 180.295403)">term = 0</text> + </g> + <g id="text_6"> + <!-- $z_2 = f_2(z_1)$ --> + <g style="fill: #3a3a3a" transform="translate(142.746 287.52342)"> + <text> + <tspan x="0" y="-0.40625" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="22.251465" y="-0.40625" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="34.400391" y="-0.40625" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="5.249023" y="1.234375" style="font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">2</tspan> + <tspan x="25.771973" y="1.234375" style="font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">2</tspan> + <tspan x="39.649414" y="1.234375" style="font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">1</tspan> + <tspan x="11.924316" y="-0.40625" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="30.499023" y="-0.40625" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="44.376465" y="-0.40625" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + </text> + </g> + </g> + <g id="text_7"> + <text style="font-size: 8.4px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: start; fill: #8a8a8a" x="274.0608" y="287.08192" transform="rotate(-0 274.0608 287.08192)">term = 0</text> + </g> + <g id="text_8"> + <!-- $z_1 = f_1(\mathrm{emb})$ --> + <g style="fill: #3a3a3a" transform="translate(136.596 394.309937)"> + <text> + <tspan x="0" y="-0.40625" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="22.251465" y="-0.40625" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="5.249023" y="1.234375" style="font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">1</tspan> + <tspan x="25.771973" y="1.234375" style="font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">1</tspan> + <tspan x="11.924316" y="-0.40625" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="30.499023" y="-0.40625" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="34.400391" y="-0.40625" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="40.552734" y="-0.40625" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="50.293945" y="-0.40625" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">b</tspan> + <tspan x="56.641602" y="-0.40625" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + </text> + </g> + </g> + <g id="text_9"> + <text style="font-size: 8.4px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: start; fill: #8a8a8a" x="274.0608" y="393.868437" transform="rotate(-0 274.0608 393.868437)">term = 0</text> + </g> + <g id="patch_9"> + <path d="M 166.896 260.239287 +Q 166.896 231.817088 166.896 204.680629 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 164.696 209.080629 +L 166.896 204.680629 +L 169.096 209.080629 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="patch_10"> + <path d="M 166.896 367.025804 +Q 166.896 338.603605 166.896 311.467146 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 164.696 315.867146 +L 166.896 311.467146 +L 169.096 315.867146 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="patch_11"> + <path d="M 166.896 467.579034 +Q 166.896 443.164529 166.896 420.035763 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 164.696 424.435763 +L 166.896 420.035763 +L 169.096 424.435763 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="text_10"> + <!-- $\mathrm{emb}(x)$ --> + <g style="fill: #3a3a3a" transform="translate(148.896 490.417802)"> + <text> + <tspan x="0" y="-0.40625" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="6.152344" y="-0.40625" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="15.893555" y="-0.40625" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">b</tspan> + <tspan x="22.241211" y="-0.40625" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="32.060547" y="-0.40625" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="26.142578" y="-0.40625" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">x</tspan> + </text> + </g> + </g> + <g id="text_11"> + <!-- The minimum is exact: $E=0$, states $\equiv$ forward activations, --> + <g style="fill: #3a3a3a" transform="translate(82.71 547.094673)"> + <text> + <tspan x="0" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">T</tspan> + <tspan x="5.497559" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="11.20166" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="16.73877" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="19.599609" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="28.366699" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="30.867188" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="36.571289" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="39.071777" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="47.838867" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="53.542969" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="62.310059" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="65.170898" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="67.671387" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="72.360352" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="75.221191" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="80.758301" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">x</tspan> + <tspan x="86.084473" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="91.599609" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="96.547852" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="100.07666" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">:</tspan> + <tspan x="103.108887" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="105.969727" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="116.270508" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="125.564941" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">0</tspan> + <tspan x="131.291016" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">,</tspan> + <tspan x="134.151855" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="137.012695" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="139.873535" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="144.5625" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="148.091309" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="153.606445" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="157.135254" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="162.672363" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="167.361328" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="171.975586" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">≡</tspan> + <tspan x="181.27002" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="184.130859" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="187.299316" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="192.805664" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="196.505859" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">w</tspan> + <tspan x="203.866699" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="209.381836" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="213.082031" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="218.794922" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="221.655762" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="227.170898" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="232.119141" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="235.647949" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="238.148438" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">v</tspan> + <tspan x="243.474609" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="248.989746" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="252.518555" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="255.019043" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="260.525391" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="266.229492" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="270.918457" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">,</tspan> + <tspan x="108.830566" y="-0.15625" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">E</tspan> + </text> + </g> + </g> + <g id="text_12"> + <text style="font-size: 9px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="219.6" y="573.791303" transform="rotate(-0 219.6 573.791303)">and one bottom-up pass reaches it.</text> + </g> + <g id="text_13"> + <text style="font-size: 8.6px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #8a8a8a" x="219.6" y="605.716883" transform="rotate(-0 219.6 605.716883)">The free phase adds nothing and changes nothing at inference.</text> + </g> + <g id="text_14"> + <text style="font-weight: 700; font-size: 11.5px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="614.88" y="27.155326" transform="rotate(-0 614.88 27.155326)">Nudged phase (training only)</text> + </g> + <g id="text_15"> + <!-- add the loss at strength $\beta \ll 1$ and settle again: --> + <g style="fill: #8a8a8a" transform="translate(510.336 52.297708)"> + <text> + <tspan x="0" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="5.361938" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">d</tspan> + <tspan x="10.916138" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">d</tspan> + <tspan x="16.470337" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="19.251709" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="22.682495" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">h</tspan> + <tspan x="28.228149" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="33.61145" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="36.392822" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">l</tspan> + <tspan x="38.823853" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">o</tspan> + <tspan x="44.177246" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="48.735962" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="53.294678" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="56.07605" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="61.437988" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="64.868774" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="67.650146" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="72.208862" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="75.639648" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="79.237061" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="84.620361" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">n</tspan> + <tspan x="90.166016" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">g</tspan> + <tspan x="95.720215" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="99.151001" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">h</tspan> + <tspan x="104.696655" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="114.766846" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">≪</tspan> + <tspan x="125.631714" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">1</tspan> + <tspan x="131.19873" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="133.980103" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="139.342041" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">n</tspan> + <tspan x="144.887695" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">d</tspan> + <tspan x="150.441895" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="153.223267" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="157.781982" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="163.165283" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="166.596069" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="170.026855" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">l</tspan> + <tspan x="172.457886" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="177.841187" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="180.622559" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="185.984497" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">g</tspan> + <tspan x="191.538696" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="196.900635" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="199.331665" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">n</tspan> + <tspan x="204.877319" y="-0.109375" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">:</tspan> + <tspan x="107.478027" y="-0.109375" style="font-style: oblique; font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">β</tspan> + </text> + </g> + </g> + <g id="text_16"> + <!-- $E(z)\ +\ \beta\cdot\mathrm{CE}(\mathrm{logits}(z_3),\,y)$ --> + <g style="fill: #3a3a3a" transform="translate(539.9 88.649213)"> + <text> + <tspan x="0" y="-0.8125" style="font-style: oblique; font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">E</tspan> + <tspan x="11.752686" y="-0.8125" style="font-style: oblique; font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="43.860521" y="-0.8125" style="font-style: oblique; font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">β</tspan> + <tspan x="114.831468" y="-0.8125" style="font-style: oblique; font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="138.567297" y="-0.8125" style="font-style: oblique; font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">y</tspan> + <tspan x="7.266113" y="-0.8125" style="font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="17.789062" y="-0.8125" style="font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="28.250207" y="-0.8125" style="font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">+</tspan> + <tspan x="53.440111" y="-0.8125" style="font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">⋅</tspan> + <tspan x="59.336107" y="-0.8125" style="font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">C</tspan> + <tspan x="67.365892" y="-0.8125" style="font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">E</tspan> + <tspan x="74.632005" y="-0.8125" style="font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="79.118578" y="-0.8125" style="font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="82.313646" y="-0.8125" style="font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="89.349535" y="-0.8125" style="font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">g</tspan> + <tspan x="96.649339" y="-0.8125" style="font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="99.844408" y="-0.8125" style="font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="104.353441" y="-0.8125" style="font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="110.344896" y="-0.8125" style="font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="126.317626" y="-0.8125" style="font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="130.804198" y="-0.8125" style="font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">,</tspan> + <tspan x="145.372961" y="-0.8125" style="font-size: 11.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="120.867845" y="1.15625" style="font-size: 8.05px; font-family: 'DejaVu Sans'; fill: #3a3a3a">3</tspan> + </text> + </g> + </g> + <g id="text_17"> + <!-- pull $-\beta\,\nabla\mathrm{CE}$ --> + <g style="fill: #b03a2e" transform="translate(641.232 120.097169)"> + <text> + <tspan x="0" y="-0.109375" style="font-size: 9.4px; font-family: 'DejaVu Sans'; fill: #b03a2e">p</tspan> + <tspan x="5.950928" y="-0.109375" style="font-size: 9.4px; font-family: 'DejaVu Sans'; fill: #b03a2e">u</tspan> + <tspan x="11.8927" y="-0.109375" style="font-size: 9.4px; font-family: 'DejaVu Sans'; fill: #b03a2e">l</tspan> + <tspan x="14.497375" y="-0.109375" style="font-size: 9.4px; font-family: 'DejaVu Sans'; fill: #b03a2e">l</tspan> + <tspan x="17.102051" y="-0.109375" style="font-size: 9.4px; font-family: 'DejaVu Sans'; fill: #b03a2e"> </tspan> + <tspan x="20.082092" y="-0.109375" style="font-size: 9.4px; font-family: 'DejaVu Sans'; fill: #b03a2e">−</tspan> + <tspan x="35.442383" y="-0.109375" style="font-size: 9.4px; font-family: 'DejaVu Sans'; fill: #b03a2e">∇</tspan> + <tspan x="41.713745" y="-0.109375" style="font-size: 9.4px; font-family: 'DejaVu Sans'; fill: #b03a2e">C</tspan> + <tspan x="48.259766" y="-0.109375" style="font-size: 9.4px; font-family: 'DejaVu Sans'; fill: #b03a2e">E</tspan> + <tspan x="27.937317" y="-0.109375" style="font-style: oblique; font-size: 9.4px; font-family: 'DejaVu Sans'; fill: #b03a2e">β</tspan> + </text> + </g> + </g> + <g id="text_18"> + <text style="font-size: 7.8px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: start; fill: #b03a2e" x="641.232" y="140.974784" transform="rotate(-0 641.232 140.974784)">(the only place the label enters)</text> + </g> + <g id="patch_12"> + <path d="M 632.014537 125.721668 +Q 608.292978 139.266857 586.124861 151.925019 +" style="fill: none; stroke: #b03a2e; stroke-width: 1.6; stroke-linecap: round"/> + <path d="M 591.036723 151.653702 +L 586.124861 151.925019 +L 588.854924 147.83274 +z +" style="fill: #b03a2e; stroke: #b03a2e; stroke-width: 1.6; stroke-linecap: round"/> + </g> + <g id="text_19"> + <!-- $z_3^{\beta} = z_3 + d_3$ --> + <g style="fill: #3a3a3a" transform="translate(534.126 181.827528)"> + <text> + <tspan x="0" y="-0.475" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="22.711751" y="-0.475" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="44.963216" y="-0.475" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="5.695638" y="-5.640625" style="font-style: oblique; font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">β</tspan> + <tspan x="5.249023" y="2.259375" style="font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">3</tspan> + <tspan x="27.960775" y="1.165625" style="font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">3</tspan> + <tspan x="51.310872" y="1.165625" style="font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">3</tspan> + <tspan x="12.384603" y="-0.475" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="34.636068" y="-0.475" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">+</tspan> + </text> + </g> + </g> + <g id="text_20"> + <!-- $z_2^{\beta} = z_2 + d_2$ --> + <g style="fill: #3a3a3a" transform="translate(534.126 288.664045)"> + <text> + <tspan x="0" y="-0.475" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="22.711751" y="-0.475" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="44.963216" y="-0.475" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="5.695638" y="-5.640625" style="font-style: oblique; font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">β</tspan> + <tspan x="5.249023" y="2.259375" style="font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">2</tspan> + <tspan x="27.960775" y="1.165625" style="font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">2</tspan> + <tspan x="51.310872" y="1.165625" style="font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">2</tspan> + <tspan x="12.384603" y="-0.475" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="34.636068" y="-0.475" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">+</tspan> + </text> + </g> + </g> + <g id="text_21"> + <!-- $z_1^{\beta} = z_1 + d_1$ --> + <g style="fill: #3a3a3a" transform="translate(534.126 395.400562)"> + <text> + <tspan x="0" y="-0.5625" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="22.711751" y="-0.5625" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="44.963216" y="-0.5625" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="5.695638" y="-5.640625" style="font-style: oblique; font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">β</tspan> + <tspan x="5.249023" y="2.171875" style="font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">1</tspan> + <tspan x="27.960775" y="1.078125" style="font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">1</tspan> + <tspan x="51.310872" y="1.078125" style="font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">1</tspan> + <tspan x="12.384603" y="-0.5625" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="34.636068" y="-0.5625" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">+</tspan> + </text> + </g> + </g> + <g id="patch_13"> + <path d="M 562.176 260.239287 +Q 562.176 231.817088 562.176 204.680629 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 559.976 209.080629 +L 562.176 204.680629 +L 564.376 209.080629 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="patch_14"> + <path d="M 562.176 367.025804 +Q 562.176 338.603605 562.176 311.467146 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 559.976 315.867146 +L 562.176 311.467146 +L 564.376 315.867146 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="patch_15"> + <path d="M 562.176 467.579034 +Q 562.176 443.164529 562.176 420.035763 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 559.976 424.435763 +L 562.176 420.035763 +L 564.376 424.435763 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="text_22"> + <!-- $\mathrm{emb}(x)$ --> + <g style="fill: #3a3a3a" transform="translate(544.176 490.417802)"> + <text> + <tspan x="0" y="-0.40625" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="6.152344" y="-0.40625" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="15.893555" y="-0.40625" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">b</tspan> + <tspan x="22.241211" y="-0.40625" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="32.060547" y="-0.40625" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="26.142578" y="-0.40625" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">x</tspan> + </text> + </g> + </g> + <g id="patch_16"> + <path d="M 671.0976 196.271932 +Q 671.0976 227.809281 671.0976 257.557775 +" style="fill: none; stroke: #d95f02; stroke-width: 1.6; stroke-linecap: round"/> + <path d="M 673.2976 253.157775 +L 671.0976 257.557775 +L 668.8976 253.157775 +z +" style="fill: #d95f02; stroke: #d95f02; stroke-width: 1.6; stroke-linecap: round"/> + </g> + <g id="text_23"> + <!-- $d_2 = J_3^{\top} d_3$ --> + <g style="fill: #d95f02" transform="translate(686.9088 230.584236)"> + <text> + <tspan x="0" y="-0.146875" style="font-style: oblique; font-size: 9.4px; font-family: 'DejaVu Sans'; fill: #d95f02">d</tspan> + <tspan x="21.880463" y="-0.146875" style="font-style: oblique; font-size: 9.4px; font-family: 'DejaVu Sans'; fill: #d95f02">J</tspan> + <tspan x="33.79671" y="-0.146875" style="font-style: oblique; font-size: 9.4px; font-family: 'DejaVu Sans'; fill: #d95f02">d</tspan> + <tspan x="5.950928" y="1.329687" style="font-size: 6.58px; font-family: 'DejaVu Sans'; fill: #d95f02">2</tspan> + <tspan x="24.645355" y="2.314062" style="font-size: 6.58px; font-family: 'DejaVu Sans'; fill: #d95f02">3</tspan> + <tspan x="26.55553" y="-3.592188" style="font-size: 6.58px; font-family: 'DejaVu Sans'; fill: #d95f02">⊤</tspan> + <tspan x="39.747638" y="1.329687" style="font-size: 6.58px; font-family: 'DejaVu Sans'; fill: #d95f02">3</tspan> + <tspan x="12.198761" y="-0.146875" style="font-size: 9.4px; font-family: 'DejaVu Sans'; fill: #d95f02">=</tspan> + </text> + </g> + </g> + <g id="patch_17"> + <path d="M 671.0976 303.058449 +Q 671.0976 334.595797 671.0976 364.344291 +" style="fill: none; stroke: #d95f02; stroke-width: 1.6; stroke-linecap: round"/> + <path d="M 673.2976 359.944291 +L 671.0976 364.344291 +L 668.8976 359.944291 +z +" style="fill: #d95f02; stroke: #d95f02; stroke-width: 1.6; stroke-linecap: round"/> + </g> + <g id="text_24"> + <!-- $d_1 = J_2^{\top} d_2$ --> + <g style="fill: #d95f02" transform="translate(686.9088 337.417753)"> + <text> + <tspan x="0" y="-0.146875" style="font-style: oblique; font-size: 9.4px; font-family: 'DejaVu Sans'; fill: #d95f02">d</tspan> + <tspan x="21.880463" y="-0.146875" style="font-style: oblique; font-size: 9.4px; font-family: 'DejaVu Sans'; fill: #d95f02">J</tspan> + <tspan x="33.79671" y="-0.146875" style="font-style: oblique; font-size: 9.4px; font-family: 'DejaVu Sans'; fill: #d95f02">d</tspan> + <tspan x="5.950928" y="1.329687" style="font-size: 6.58px; font-family: 'DejaVu Sans'; fill: #d95f02">1</tspan> + <tspan x="24.645355" y="2.314062" style="font-size: 6.58px; font-family: 'DejaVu Sans'; fill: #d95f02">2</tspan> + <tspan x="26.55553" y="-3.592188" style="font-size: 6.58px; font-family: 'DejaVu Sans'; fill: #d95f02">⊤</tspan> + <tspan x="39.747638" y="1.329687" style="font-size: 6.58px; font-family: 'DejaVu Sans'; fill: #d95f02">2</tspan> + <tspan x="12.198761" y="-0.146875" style="font-size: 9.4px; font-family: 'DejaVu Sans'; fill: #d95f02">=</tspan> + </text> + </g> + </g> + <g id="text_25"> + <!-- The top state is pulled toward lower loss; each layer's mismatch $d_l$ --> + <g style="fill: #3a3a3a" transform="translate(464.58 547.086236)"> + <text> + <tspan x="0" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">T</tspan> + <tspan x="5.497559" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="11.20166" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="16.73877" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="19.599609" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="23.128418" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="28.634766" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="34.347656" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="37.208496" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="41.897461" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="45.42627" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="50.941406" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="54.470215" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="60.007324" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="62.868164" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="65.368652" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="70.057617" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="72.918457" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="78.631348" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="84.335449" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="86.835938" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="89.336426" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="94.873535" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="100.586426" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="103.447266" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="106.976074" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="112.482422" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">w</tspan> + <tspan x="119.843262" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="125.358398" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="129.058594" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="134.771484" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="137.632324" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="140.132812" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="145.63916" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">w</tspan> + <tspan x="153" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="158.537109" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="162.237305" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="165.098145" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="167.598633" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="173.10498" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="177.793945" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="182.48291" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">;</tspan> + <tspan x="185.515137" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="188.375977" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="193.913086" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="199.428223" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="204.376465" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="210.080566" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="212.941406" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="215.441895" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="220.957031" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">y</tspan> + <tspan x="226.283203" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="231.820312" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="235.520508" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">'</tspan> + <tspan x="237.994629" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="242.683594" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="245.544434" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="254.311523" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="256.812012" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="261.500977" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="270.268066" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="275.783203" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="279.312012" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="284.260254" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="289.964355" y="-0.15625" style="font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="292.825195" y="-0.15625" style="font-style: oblique; font-size: 9px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="298.538086" y="1.320312" style="font-style: oblique; font-size: 6.3px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + </text> + </g> + </g> + <g id="text_26"> + <text style="font-size: 9px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="614.88" y="573.791303" transform="rotate(-0 614.88 573.791303)">transmits DOWN through the same weights, and the stack re-settles.</text> + </g> + <g id="text_27"> + <!-- ($J^{\top}$ = the transpose read a bidirectional physical device provides) --> + <g style="fill: #8a8a8a" transform="translate(472.421 606.22482)"> + <text> + <tspan x="0" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">(</tspan> + <tspan x="14.398096" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="17.139734" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">=</tspan> + <tspan x="24.366541" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="27.108179" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="30.489954" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">h</tspan> + <tspan x="35.956384" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="41.262781" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="44.004419" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="47.386194" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="50.932214" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="56.217554" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">n</tspan> + <tspan x="61.683984" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="66.177576" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">p</tspan> + <tspan x="71.652429" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">o</tspan> + <tspan x="76.929346" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="81.422937" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="86.729333" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="89.470972" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="93.016992" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="98.323389" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="103.608728" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">d</tspan> + <tspan x="109.083582" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="111.82522" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="117.110559" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="119.852197" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">b</tspan> + <tspan x="125.327051" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="127.723352" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">d</tspan> + <tspan x="133.198206" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="135.594507" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="139.140527" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="144.446924" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">c</tspan> + <tspan x="149.188989" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="152.570764" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="154.967065" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">o</tspan> + <tspan x="160.243982" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">n</tspan> + <tspan x="165.710413" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="170.995752" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">l</tspan> + <tspan x="173.392053" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="176.133691" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">p</tspan> + <tspan x="181.608545" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">h</tspan> + <tspan x="187.074976" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">y</tspan> + <tspan x="192.179224" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="196.672815" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="199.069116" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">c</tspan> + <tspan x="203.811182" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="209.096521" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">l</tspan> + <tspan x="211.492822" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="214.23446" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">d</tspan> + <tspan x="219.709314" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="225.01571" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">v</tspan> + <tspan x="230.119958" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="232.51626" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">c</tspan> + <tspan x="237.258325" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="242.564722" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="245.30636" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">p</tspan> + <tspan x="250.781213" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="254.327234" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">o</tspan> + <tspan x="259.60415" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">v</tspan> + <tspan x="264.708398" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="267.1047" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">d</tspan> + <tspan x="272.579553" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="277.88595" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="282.379541" y="-0.146875" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">)</tspan> + <tspan x="3.364929" y="-0.146875" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">J</tspan> + <tspan x="7.716522" y="-3.592188" style="font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #8a8a8a">⊤</tspan> + </text> + </g> + </g> + <g id="text_28"> + <text style="font-weight: 700; font-size: 11px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: start; fill: #3a3a3a" x="35.136" y="662.442054" transform="rotate(-0 35.136 662.442054)">The update — a difference measurement between the two settled states</text> + </g> + <g id="text_29"> + <!-- $\hat{g}\ =\ \left[\ \partial_\theta E(z^{\beta})\ -\ \partial_\theta E(z^{0})\ \right]\,/\,\beta$$\qquad\qquad(\partial_\theta E(z^0)\equiv 0\ \mathrm{here,\ since}\ E=0)$ --> + <g style="fill: #3a3a3a" transform="translate(259.185 697.182472)"> + <text> + <tspan x="7.214355" y="-1.970313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">̂</tspan> + <tspan x="12.69723" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="34.934829" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">∂</tspan> + <tspan x="52.690054" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="68.461555" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="78.467867" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">−</tspan> + <tspan x="93.399471" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">∂</tspan> + <tspan x="111.154696" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="126.911158" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="140.294581" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">/</tspan> + <tspan x="195.667907" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="199.959411" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">∂</tspan> + <tspan x="217.714636" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="233.471097" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="239.905668" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">≡</tspan> + <tspan x="251.265531" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">0</tspan> + <tspan x="261.835808" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="268.807487" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="275.575066" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="280.097526" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="286.865105" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">,</tspan> + <tspan x="296.076495" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="301.807452" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="304.863604" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="311.835284" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="317.883135" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="337.315717" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="348.67558" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">0</tspan> + <tspan x="355.674115" y="-0.595313" style="font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="0" y="-0.595313" style="font-style: oblique; font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">g</tspan> + <tspan x="45.739859" y="-0.595313" style="font-style: oblique; font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">E</tspan> + <tspan x="56.981558" y="-0.595313" style="font-style: oblique; font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="104.204501" y="-0.595313" style="font-style: oblique; font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">E</tspan> + <tspan x="115.4462" y="-0.595313" style="font-style: oblique; font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="145.78656" y="-0.595313" style="font-style: oblique; font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">β</tspan> + <tspan x="210.76444" y="-0.595313" style="font-style: oblique; font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">E</tspan> + <tspan x="222.00614" y="-0.595313" style="font-style: oblique; font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="328.222455" y="-0.595313" style="font-style: oblique; font-size: 11px; font-family: 'DejaVu Sans'; fill: #3a3a3a">E</tspan> + <tspan x="27.628835" y="-0.34375" style="font-size: 9.765202px; font-family: 'STIXSizeOneSym'; fill: #3a3a3a">[</tspan> + <tspan x="134.774403" y="-0.34375" style="font-size: 9.765202px; font-family: 'STIXSizeOneSym'; fill: #3a3a3a">]</tspan> + <tspan x="40.728091" y="1.209375" style="font-style: oblique; font-size: 7.7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">θ</tspan> + <tspan x="63.24676" y="-4.80625" style="font-style: oblique; font-size: 7.7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">β</tspan> + <tspan x="99.192733" y="1.209375" style="font-style: oblique; font-size: 7.7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">θ</tspan> + <tspan x="205.752673" y="1.209375" style="font-style: oblique; font-size: 7.7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">θ</tspan> + <tspan x="121.711402" y="-4.80625" style="font-size: 7.7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">0</tspan> + <tspan x="228.271342" y="-4.80625" style="font-size: 7.7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">0</tspan> + </text> + </g> + </g> + <g id="text_30"> + <!-- per layer: $\Delta\theta_l\ \propto\ \langle\ d_l\ ,\ \partial f_l(z_{l-1})/\partial\theta_l\ \rangle\ /\ \beta$ --> + <g style="fill: #3a3a3a" transform="translate(342.1 728.948315)"> + <text> + <tspan x="0" y="-0.328125" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="6.347656" y="-0.328125" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="12.5" y="-0.328125" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="16.611328" y="-0.328125" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="19.790039" y="-0.328125" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="22.568359" y="-0.328125" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="28.696289" y="-0.328125" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">y</tspan> + <tspan x="34.614258" y="-0.328125" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="40.766602" y="-0.328125" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="44.87793" y="-0.328125" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">:</tspan> + <tspan x="48.24707" y="-0.328125" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="51.425781" y="-0.328125" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="54.604492" y="-0.328125" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">Δ</tspan> + <tspan x="74.977018" y="-0.328125" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">∝</tspan> + <tspan x="87.315853" y="-0.328125" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">⟨</tspan> + <tspan x="106.277214" y="-0.328125" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">,</tspan> + <tspan x="114.651205" y="-0.328125" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">∂</tspan> + <tspan x="125.560873" y="-0.328125" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="149.975912" y="-0.328125" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="153.877279" y="-0.328125" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">/</tspan> + <tspan x="157.24642" y="-0.328125" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">∂</tspan> + <tspan x="174.000782" y="-0.328125" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">⟩</tspan> + <tspan x="181.149187" y="-0.328125" style="font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">/</tspan> + <tspan x="61.445312" y="-0.328125" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">θ</tspan> + <tspan x="94.464258" y="-0.328125" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="119.822103" y="-0.328125" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="129.46224" y="-0.328125" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="162.417318" y="-0.328125" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">θ</tspan> + <tspan x="187.765365" y="-0.328125" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">β</tspan> + <tspan x="67.563477" y="1.3125" style="font-style: oblique; font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="100.811914" y="1.3125" style="font-style: oblique; font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="123.342611" y="1.3125" style="font-style: oblique; font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="134.711263" y="1.3125" style="font-style: oblique; font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="168.535482" y="1.3125" style="font-style: oblique; font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="138.019857" y="1.3125" style="font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">−</tspan> + <tspan x="145.248861" y="1.3125" style="font-size: 7px; font-family: 'DejaVu Sans'; fill: #3a3a3a">1</tspan> + </text> + </g> + </g> + <g id="text_31"> + <text style="font-size: 8.6px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="439.2" y="755.218006" transform="rotate(-0 439.2 755.218006)">Each layer updates from its own boundary mismatch — no global backward graph, no global tape, no loss derivatives except the top nudge.</text> + </g> + </g> + </g> + <defs> + <clipPath id="p90a79555a2"> + <rect x="0" y="0" width="878.4" height="792"/> + </clipPath> + </defs> +</svg> diff --git a/assets/figs/frozen_vs_adaptive.png b/assets/figs/frozen_vs_adaptive.png Binary files differnew file mode 100644 index 0000000..e45e77b --- /dev/null +++ b/assets/figs/frozen_vs_adaptive.png |
