diff options
Diffstat (limited to 'assets/figs/fig_primer_block.svg')
| -rw-r--r-- | assets/figs/fig_primer_block.svg | 1956 |
1 files changed, 1956 insertions, 0 deletions
diff --git a/assets/figs/fig_primer_block.svg b/assets/figs/fig_primer_block.svg new file mode 100644 index 0000000..7a901a9 --- /dev/null +++ b/assets/figs/fig_primer_block.svg @@ -0,0 +1,1956 @@ +<?xml version="1.0" encoding="utf-8" standalone="no"?> +<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" + "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"> +<svg xmlns:xlink="http://www.w3.org/1999/xlink" width="720pt" height="1022.4pt" viewBox="0 0 720 1022.4" xmlns="http://www.w3.org/2000/svg" version="1.1"> + <metadata> + <rdf:RDF xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:cc="http://creativecommons.org/ns#" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"> + <cc:Work> + <dc:type rdf:resource="http://purl.org/dc/dcmitype/StillImage"/> + <dc:date>2026-07-15T09:29:45.734111</dc:date> + <dc:format>image/svg+xml</dc:format> + <dc:creator> + <cc:Agent> + <dc:title>Matplotlib v3.10.8, https://matplotlib.org/</dc:title> + </cc:Agent> + </dc:creator> + </cc:Work> + </rdf:RDF> + </metadata> + <defs> + <style type="text/css">*{stroke-linejoin: round; stroke-linecap: butt}</style> + </defs> + <g id="figure_1"> + <g id="patch_1"> + <path d="M 0 1022.4 +L 720 1022.4 +L 720 0 +L 0 0 +z +" style="fill: #ffffff"/> + </g> + <g id="axes_1"> + <g id="line2d_1"> + <path d="M 288 83.52 +L 633.6 83.52 +L 633.6 505.44 +" clip-path="url(#pdba9adb895)" style="fill: none; stroke: #555555; stroke-width: 1.4; stroke-linecap: round"/> + </g> + <g id="patch_2"> + <path d="M 182.16 122.04 +L 393.84 122.04 +Q 397.8 122.04 397.8 118.08 +L 397.8 99.36 +Q 397.8 95.4 393.84 95.4 +L 182.16 95.4 +Q 178.2 95.4 178.2 99.36 +L 178.2 118.08 +Q 178.2 122.04 182.16 122.04 +z +" clip-path="url(#pdba9adb895)" style="fill: #eaf1fa; stroke: #2c6fbb; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_3"> + <path d="M 54.36 165.24 +L 176.04 165.24 +Q 180 165.24 180 161.28 +L 180 142.56 +Q 180 138.6 176.04 138.6 +L 54.36 138.6 +Q 50.4 138.6 50.4 142.56 +L 50.4 161.28 +Q 50.4 165.24 54.36 165.24 +z +" clip-path="url(#pdba9adb895)" style="fill: #faf6ee; stroke: #c9b895; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_4"> + <path d="M 227.16 165.24 +L 348.84 165.24 +Q 352.8 165.24 352.8 161.28 +L 352.8 142.56 +Q 352.8 138.6 348.84 138.6 +L 227.16 138.6 +Q 223.2 138.6 223.2 142.56 +L 223.2 161.28 +Q 223.2 165.24 227.16 165.24 +z +" clip-path="url(#pdba9adb895)" style="fill: #faf6ee; stroke: #c9b895; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_5"> + <path d="M 54.36 199.08 +L 176.04 199.08 +Q 180 199.08 180 195.12 +L 180 176.4 +Q 180 172.44 176.04 172.44 +L 54.36 172.44 +Q 50.4 172.44 50.4 176.4 +L 50.4 195.12 +Q 50.4 199.08 54.36 199.08 +z +" clip-path="url(#pdba9adb895)" style="fill: #f4f4f4; stroke: #9a9a9a; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_6"> + <path d="M 227.16 199.08 +L 348.84 199.08 +Q 352.8 199.08 352.8 195.12 +L 352.8 176.4 +Q 352.8 172.44 348.84 172.44 +L 227.16 172.44 +Q 223.2 172.44 223.2 176.4 +L 223.2 195.12 +Q 223.2 199.08 227.16 199.08 +z +" clip-path="url(#pdba9adb895)" style="fill: #f4f4f4; stroke: #9a9a9a; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_7"> + <path d="M 399.96 199.08 +L 521.64 199.08 +Q 525.6 199.08 525.6 195.12 +L 525.6 176.4 +Q 525.6 172.44 521.64 172.44 +L 399.96 172.44 +Q 396 172.44 396 176.4 +L 396 195.12 +Q 396 199.08 399.96 199.08 +z +" clip-path="url(#pdba9adb895)" style="fill: #f4f4f4; stroke: #9a9a9a; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_8"> + <path d="M 54.36 232.92 +L 176.04 232.92 +Q 180 232.92 180 228.96 +L 180 210.24 +Q 180 206.28 176.04 206.28 +L 54.36 206.28 +Q 50.4 206.28 50.4 210.24 +L 50.4 228.96 +Q 50.4 232.92 54.36 232.92 +z +" clip-path="url(#pdba9adb895)" style="fill: #f4f4f4; stroke: #9a9a9a; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_9"> + <path d="M 227.16 232.92 +L 348.84 232.92 +Q 352.8 232.92 352.8 228.96 +L 352.8 210.24 +Q 352.8 206.28 348.84 206.28 +L 227.16 206.28 +Q 223.2 206.28 223.2 210.24 +L 223.2 228.96 +Q 223.2 232.92 227.16 232.92 +z +" clip-path="url(#pdba9adb895)" style="fill: #f4f4f4; stroke: #9a9a9a; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_10"> + <path d="M 174.96 276.12 +L 401.04 276.12 +Q 405 276.12 405 272.16 +L 405 253.44 +Q 405 249.48 401.04 249.48 +L 174.96 249.48 +Q 171 249.48 171 253.44 +L 171 272.16 +Q 171 276.12 174.96 276.12 +z +" clip-path="url(#pdba9adb895)" style="fill: #f1ebf8; stroke: #7b5aa6; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_11"> + <path d="M 174.96 309.96 +L 401.04 309.96 +Q 405 309.96 405 306 +L 405 287.28 +Q 405 283.32 401.04 283.32 +L 174.96 283.32 +Q 171 283.32 171 287.28 +L 171 306 +Q 171 309.96 174.96 309.96 +z +" clip-path="url(#pdba9adb895)" style="fill: #f4f4f4; stroke: #9a9a9a; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_12"> + <path d="M 174.96 343.8 +L 401.04 343.8 +Q 405 343.8 405 339.84 +L 405 321.12 +Q 405 317.16 401.04 317.16 +L 174.96 317.16 +Q 171 317.16 171 321.12 +L 171 339.84 +Q 171 343.8 174.96 343.8 +z +" clip-path="url(#pdba9adb895)" style="fill: #f4f4f4; stroke: #9a9a9a; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_13"> + <path d="M 174.96 377.64 +L 401.04 377.64 +Q 405 377.64 405 373.68 +L 405 354.96 +Q 405 351 401.04 351 +L 174.96 351 +Q 171 351 171 354.96 +L 171 373.68 +Q 171 377.64 174.96 377.64 +z +" clip-path="url(#pdba9adb895)" style="fill: #f1ebf8; stroke: #7b5aa6; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_14"> + <path d="M 174.96 411.48 +L 401.04 411.48 +Q 405 411.48 405 407.52 +L 405 388.8 +Q 405 384.84 401.04 384.84 +L 174.96 384.84 +Q 171 384.84 171 388.8 +L 171 407.52 +Q 171 411.48 174.96 411.48 +z +" clip-path="url(#pdba9adb895)" style="fill: #f4f4f4; stroke: #9a9a9a; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_15"> + <path d="M 174.96 445.32 +L 401.04 445.32 +Q 405 445.32 405 441.36 +L 405 422.64 +Q 405 418.68 401.04 418.68 +L 174.96 418.68 +Q 171 418.68 171 422.64 +L 171 441.36 +Q 171 445.32 174.96 445.32 +z +" clip-path="url(#pdba9adb895)" style="fill: #eaf1fa; stroke: #2c6fbb; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_16"> + <path d="M 174.96 479.16 +L 401.04 479.16 +Q 405 479.16 405 475.2 +L 405 456.48 +Q 405 452.52 401.04 452.52 +L 174.96 452.52 +Q 171 452.52 171 456.48 +L 171 475.2 +Q 171 479.16 174.96 479.16 +z +" clip-path="url(#pdba9adb895)" style="fill: #faf6ee; stroke: #c9b895; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="line2d_2"> + <path d="M 288 540 +L 633.6 540 +L 633.6 754.56 +" clip-path="url(#pdba9adb895)" style="fill: none; stroke: #555555; stroke-width: 1.4; stroke-linecap: round"/> + </g> + <g id="patch_17"> + <path d="M 113.76 586.44 +L 260.64 586.44 +Q 264.6 586.44 264.6 582.48 +L 264.6 563.76 +Q 264.6 559.8 260.64 559.8 +L 113.76 559.8 +Q 109.8 559.8 109.8 563.76 +L 109.8 582.48 +Q 109.8 586.44 113.76 586.44 +z +" clip-path="url(#pdba9adb895)" style="fill: #eaf1fa; stroke: #2c6fbb; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_18"> + <path d="M 315.36 586.44 +L 462.24 586.44 +Q 466.2 586.44 466.2 582.48 +L 466.2 563.76 +Q 466.2 559.8 462.24 559.8 +L 315.36 559.8 +Q 311.4 559.8 311.4 563.76 +L 311.4 582.48 +Q 311.4 586.44 315.36 586.44 +z +" clip-path="url(#pdba9adb895)" style="fill: #eaf1fa; stroke: #2c6fbb; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_19"> + <path d="M 146.16 620.28 +L 228.24 620.28 +Q 232.2 620.28 232.2 616.32 +L 232.2 597.6 +Q 232.2 593.64 228.24 593.64 +L 146.16 593.64 +Q 142.2 593.64 142.2 597.6 +L 142.2 616.32 +Q 142.2 620.28 146.16 620.28 +z +" clip-path="url(#pdba9adb895)" style="fill: #f4f4f4; stroke: #9a9a9a; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_20"> + <path d="M 207.36 658.44 +L 368.64 658.44 +Q 372.6 658.44 372.6 654.48 +L 372.6 635.76 +Q 372.6 631.8 368.64 631.8 +L 207.36 631.8 +Q 203.4 631.8 203.4 635.76 +L 203.4 654.48 +Q 203.4 658.44 207.36 658.44 +z +" clip-path="url(#pdba9adb895)" style="fill: #f4f4f4; stroke: #9a9a9a; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_21"> + <path d="M 200.16 696.6 +L 375.84 696.6 +Q 379.8 696.6 379.8 692.64 +L 379.8 673.92 +Q 379.8 669.96 375.84 669.96 +L 200.16 669.96 +Q 196.2 669.96 196.2 673.92 +L 196.2 692.64 +Q 196.2 696.6 200.16 696.6 +z +" clip-path="url(#pdba9adb895)" style="fill: #eaf1fa; stroke: #2c6fbb; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_22"> + <path d="M 200.16 730.44 +L 375.84 730.44 +Q 379.8 730.44 379.8 726.48 +L 379.8 707.76 +Q 379.8 703.8 375.84 703.8 +L 200.16 703.8 +Q 196.2 703.8 196.2 707.76 +L 196.2 726.48 +Q 196.2 730.44 200.16 730.44 +z +" clip-path="url(#pdba9adb895)" style="fill: #faf6ee; stroke: #c9b895; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_23"> + <path d="M 37.44 846.36 +L 59.04 846.36 +Q 61.56 846.36 61.56 843.84 +L 61.56 833.76 +Q 61.56 831.24 59.04 831.24 +L 37.44 831.24 +Q 34.92 831.24 34.92 833.76 +L 34.92 843.84 +Q 34.92 846.36 37.44 846.36 +z +" clip-path="url(#pdba9adb895)" style="fill: #eaf1fa; stroke: #2c6fbb; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_24"> + <path d="M 37.44 870.84 +L 59.04 870.84 +Q 61.56 870.84 61.56 868.32 +L 61.56 858.24 +Q 61.56 855.72 59.04 855.72 +L 37.44 855.72 +Q 34.92 855.72 34.92 858.24 +L 34.92 868.32 +Q 34.92 870.84 37.44 870.84 +z +" clip-path="url(#pdba9adb895)" style="fill: #f1ebf8; stroke: #7b5aa6; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_25"> + <path d="M 37.44 895.32 +L 59.04 895.32 +Q 61.56 895.32 61.56 892.8 +L 61.56 882.72 +Q 61.56 880.2 59.04 880.2 +L 37.44 880.2 +Q 34.92 880.2 34.92 882.72 +L 34.92 892.8 +Q 34.92 895.32 37.44 895.32 +z +" clip-path="url(#pdba9adb895)" style="fill: #faf6ee; stroke: #c9b895; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="patch_26"> + <path d="M 275.04 895.32 +L 296.64 895.32 +Q 299.16 895.32 299.16 892.8 +L 299.16 882.72 +Q 299.16 880.2 296.64 880.2 +L 275.04 880.2 +Q 272.52 880.2 272.52 882.72 +L 272.52 892.8 +Q 272.52 895.32 275.04 895.32 +z +" clip-path="url(#pdba9adb895)" style="fill: #f4f4f4; stroke: #9a9a9a; stroke-width: 1.1; stroke-linejoin: miter"/> + </g> + <g id="text_1"> + <text style="font-weight: 700; font-size: 12.5px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="360" y="26.440391" transform="rotate(-0 360 26.440391)">One transformer block, op by op</text> + </g> + <g id="text_2"> + <!-- OLMo2 ordering — RMSNorm after each sublayer, inside the residual. $C{=}512$ $H{=}8$ $h_d{=}64$ $T{=}256$ $h_{ff}{=}1408$ --> + <g style="fill: #8a8a8a" transform="translate(100.664 46.34)"> + <text> + <tspan x="0" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">O</tspan> + <tspan x="6.887207" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">L</tspan> + <tspan x="11.762085" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">M</tspan> + <tspan x="19.311523" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">o</tspan> + <tspan x="24.664917" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">2</tspan> + <tspan x="30.231934" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="33.013306" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">o</tspan> + <tspan x="38.366699" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="41.964111" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">d</tspan> + <tspan x="47.518311" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="52.901611" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="56.499023" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="58.930054" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">n</tspan> + <tspan x="64.475708" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">g</tspan> + <tspan x="70.029907" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="81.561279" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="84.342651" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">R</tspan> + <tspan x="90.422363" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">M</tspan> + <tspan x="97.971802" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">S</tspan> + <tspan x="103.526001" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">N</tspan> + <tspan x="110.071411" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">o</tspan> + <tspan x="115.424805" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="119.022217" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">m</tspan> + <tspan x="127.545776" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="130.327148" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="135.689087" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">f</tspan> + <tspan x="138.769531" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="142.200317" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="147.583618" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="151.18103" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="153.962402" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="159.345703" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="164.707642" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">c</tspan> + <tspan x="169.518433" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">h</tspan> + <tspan x="175.064087" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="177.845459" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="182.404175" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">u</tspan> + <tspan x="187.949829" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">b</tspan> + <tspan x="193.504028" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">l</tspan> + <tspan x="195.935059" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="201.296997" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">y</tspan> + <tspan x="206.47522" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="211.858521" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="215.455933" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">,</tspan> + <tspan x="218.237305" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="221.018677" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="223.449707" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">n</tspan> + <tspan x="228.995361" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="233.554077" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="235.985107" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">d</tspan> + <tspan x="241.539307" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="246.922607" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="249.703979" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="253.134766" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">h</tspan> + <tspan x="258.68042" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="264.063721" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="266.845093" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">r</tspan> + <tspan x="270.442505" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">e</tspan> + <tspan x="275.825806" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="280.384521" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="282.815552" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">d</tspan> + <tspan x="288.369751" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">u</tspan> + <tspan x="293.915405" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">a</tspan> + <tspan x="299.277344" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">l</tspan> + <tspan x="301.708374" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">.</tspan> + <tspan x="304.489746" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="307.271118" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="310.05249" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="312.833862" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="315.615234" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="326.210938" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">=</tspan> + <tspan x="335.247192" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">5</tspan> + <tspan x="340.814209" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">1</tspan> + <tspan x="346.381226" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">2</tspan> + <tspan x="351.948242" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="354.729614" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="357.510986" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="368.57666" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">=</tspan> + <tspan x="377.612915" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">8</tspan> + <tspan x="383.179932" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="385.961304" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="388.742676" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="402.908447" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">=</tspan> + <tspan x="411.944702" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">6</tspan> + <tspan x="417.511719" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">4</tspan> + <tspan x="423.078735" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="425.860107" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="428.641479" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="438.472412" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">=</tspan> + <tspan x="447.508667" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">2</tspan> + <tspan x="453.075684" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">5</tspan> + <tspan x="458.6427" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">6</tspan> + <tspan x="464.209717" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="466.991089" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="469.772461" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="484.362915" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">=</tspan> + <tspan x="493.39917" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">1</tspan> + <tspan x="498.966187" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">4</tspan> + <tspan x="504.533203" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">0</tspan> + <tspan x="510.10022" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">8</tspan> + <tspan x="72.811279" y="-0.15625" style="font-size: 8.8px; font-family: 'DejaVu Sans Display'; fill: #8a8a8a">—</tspan> + <tspan x="318.396606" y="-0.15625" style="font-style: oblique; font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">C</tspan> + <tspan x="360.292358" y="-0.15625" style="font-style: oblique; font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">H</tspan> + <tspan x="391.524048" y="-0.15625" style="font-style: oblique; font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">h</tspan> + <tspan x="431.422852" y="-0.15625" style="font-style: oblique; font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">T</tspan> + <tspan x="472.553833" y="-0.15625" style="font-style: oblique; font-size: 8.8px; font-family: 'DejaVu Sans'; fill: #8a8a8a">h</tspan> + <tspan x="397.069702" y="1.320312" style="font-style: oblique; font-size: 6.16px; font-family: 'DejaVu Sans'; fill: #8a8a8a">d</tspan> + <tspan x="478.099487" y="1.320312" style="font-style: oblique; font-size: 6.16px; font-family: 'DejaVu Sans'; fill: #8a8a8a">f</tspan> + <tspan x="480.255798" y="1.320312" style="font-style: oblique; font-size: 6.16px; font-family: 'DejaVu Sans'; fill: #8a8a8a">f</tspan> + </text> + </g> + </g> + <g id="text_3"> + <text style="font-style: italic; font-size: 9.5px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; fill: #8a8a8a" transform="translate(25.661406 345.993867) rotate(-90)">attention sublayer</text> + </g> + <g id="text_4"> + <!-- input $z$ $(B,T,C)$ --> + <g style="fill: #3a3a3a" transform="translate(246.77 77.4925)"> + <text> + <tspan x="0" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="3.256348" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="10.019531" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="16.819824" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="23.583008" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="28.124268" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="36.418213" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="39.725586" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="43.032959" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="31.431641" y="-0.40625" style="font-style: oblique; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="50.046631" y="-0.40625" style="font-style: oblique; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">B</tspan> + <tspan x="61.43457" y="-0.40625" style="font-style: oblique; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">T</tspan> + <tspan x="72.108154" y="-0.40625" style="font-style: oblique; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">C</tspan> + <tspan x="46.340332" y="-0.40625" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="56.563965" y="-0.40625" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">,</tspan> + <tspan x="67.237549" y="-0.40625" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">,</tspan> + <tspan x="78.741455" y="-0.40625" style="font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + </text> + </g> + </g> + <g id="patch_27"> + <path d="M 633.19823 501.12524 +Q 467.999756 503.280003 304.366397 505.414351 +" style="fill: none; stroke: #555555; stroke-width: 1.4; stroke-linecap: round"/> + <path d="M 308.392142 507.362012 +L 304.366397 505.414351 +L 308.339972 503.362352 +z +" style="fill: #555555; stroke: #555555; stroke-width: 1.4; stroke-linecap: round"/> + </g> + <g id="text_5"> + <text style="font-size: 8px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; fill: #8a8a8a" transform="translate(651.6475 318.636875) rotate(-90)">residual rail</text> + </g> + <g id="patch_28"> + <path d="M 288 85.362188 +Q 288 90.721592 288 94.795257 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 290 90.795257 +L 288 94.795257 +L 286 90.795257 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="text_6"> + <!-- matmul $W_{qkv}: C \to 3C$ --> + <g style="fill: #3a3a3a" transform="translate(241.689 110.655)"> + <text> + <tspan x="0" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="8.401794" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="13.687134" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="17.068909" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="25.470703" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="30.937134" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="33.333435" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="36.075073" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="60.172998" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">:</tspan> + <tspan x="72.461938" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">→</tspan> + <tspan x="81.369104" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">3</tspan> + <tspan x="38.816711" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">W</tspan> + <tspan x="64.759241" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">C</tspan> + <tspan x="86.856592" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">C</tspan> + <tspan x="47.344849" y="1.320312" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">q</tspan> + <tspan x="51.177246" y="1.320312" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">k</tspan> + <tspan x="54.673572" y="1.320312" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">v</tspan> + </text> + </g> + </g> + <g id="text_7"> + <!-- $(B,T,3C)$ — split into $q,k,v$ --> + <g style="fill: #8a8a8a" transform="translate(406.8 110.81)"> + <text> + <tspan x="0" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">(</tspan> + <tspan x="8.205811" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">,</tspan> + <tspan x="16.772766" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">,</tspan> + <tspan x="20.682068" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">3</tspan> + <tspan x="30.857422" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">)</tspan> + <tspan x="33.832214" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="43.880981" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="46.304749" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">s</tspan> + <tspan x="50.277344" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">p</tspan> + <tspan x="55.117432" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">l</tspan> + <tspan x="57.235901" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="59.35437" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="62.344055" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="64.767822" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">i</tspan> + <tspan x="66.886292" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">n</tspan> + <tspan x="71.718933" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">t</tspan> + <tspan x="74.708618" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">o</tspan> + <tspan x="79.373718" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a"> </tspan> + <tspan x="86.637573" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">,</tspan> + <tspan x="94.962524" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">,</tspan> + <tspan x="2.974792" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">B</tspan> + <tspan x="12.115112" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">T</tspan> + <tspan x="25.533325" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">C</tspan> + <tspan x="81.797485" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">q</tspan> + <tspan x="90.546875" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">k</tspan> + <tspan x="98.871826" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">v</tspan> + <tspan x="36.255981" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans Display'; fill: #8a8a8a">—</tspan> + </text> + </g> + </g> + <g id="text_8"> + <!-- $q$ --> + <g style="fill: #3a3a3a" transform="translate(112.16 137.242109)"> + <text> + <tspan x="0" y="-0.40625" style="font-style: oblique; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">q</tspan> + </text> + </g> + </g> + <g id="text_9"> + <!-- $k$ --> + <g style="fill: #3a3a3a" transform="translate(285.245 137.261406)"> + <text> + <tspan x="0" y="-0.40625" style="font-style: oblique; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">k</tspan> + </text> + </g> + </g> + <g id="text_10"> + <!-- $v$ --> + <g style="fill: #3a3a3a" transform="translate(457.95 137.261406)"> + <text> + <tspan x="0" y="-0.53125" style="font-style: oblique; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">v</tspan> + </text> + </g> + </g> + <g id="patch_29"> + <path d="M 266.000098 120.295986 +Q 194.399445 130.320078 124.072113 140.165904 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 128.310776 141.591996 +L 124.072113 140.165904 +L 127.756184 137.630629 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="patch_30"> + <path d="M 288 120.638672 +Q 288 130.319117 288 138.713824 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 290 134.713824 +L 288 138.713824 +L 286 134.713824 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="patch_31"> + <path d="M 309.999902 120.295986 +Q 381.600555 130.320078 451.927887 140.165904 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 448.243816 137.630629 +L 451.927887 140.165904 +L 447.689224 141.591996 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="text_11"> + <!-- RMSNorm$_q$ --> + <g style="fill: #3a3a3a" transform="translate(91.894 153.812)"> + <text> + <tspan x="0" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">R</tspan> + <tspan x="5.992859" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">M</tspan> + <tspan x="13.434448" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">S</tspan> + <tspan x="18.909302" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">N</tspan> + <tspan x="25.361206" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="30.638123" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="34.184143" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="42.67207" y="1.164062" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">q</tspan> + </text> + </g> + </g> + <g id="text_12"> + <!-- RMSNorm$_k$ --> + <g style="fill: #3a3a3a" transform="translate(264.866 154.293062)"> + <text> + <tspan x="0" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">R</tspan> + <tspan x="5.992859" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">M</tspan> + <tspan x="13.434448" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">S</tspan> + <tspan x="18.909302" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">N</tspan> + <tspan x="25.361206" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="30.638123" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="34.184143" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="42.67207" y="1.164062" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">k</tspan> + </text> + </g> + </g> + <g id="text_13"> + <!-- split heads ($H{=}8$) --> + <g style="fill: #3a3a3a" transform="translate(75.812 188.125)"> + <text> + <tspan x="0" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="4.493591" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="9.968445" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="12.364746" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="14.761047" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="18.142822" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="20.88446" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="26.350891" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="31.657288" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="36.942627" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="42.41748" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="46.911072" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="49.65271" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="61.183594" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="70.090759" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">8</tspan> + <tspan x="75.578247" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="53.017639" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">H</tspan> + </text> + </g> + </g> + <g id="text_14"> + <!-- split heads ($H{=}8$) --> + <g style="fill: #3a3a3a" transform="translate(248.612 188.125)"> + <text> + <tspan x="0" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="4.493591" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="9.968445" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="12.364746" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="14.761047" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="18.142822" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="20.88446" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="26.350891" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="31.657288" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="36.942627" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="42.41748" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="46.911072" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="49.65271" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="61.183594" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="70.090759" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">8</tspan> + <tspan x="75.578247" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="53.017639" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">H</tspan> + </text> + </g> + </g> + <g id="text_15"> + <!-- split heads ($H{=}8$) --> + <g style="fill: #3a3a3a" transform="translate(421.412 188.125)"> + <text> + <tspan x="0" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="4.493591" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="9.968445" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="12.364746" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="14.761047" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="18.142822" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="20.88446" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="26.350891" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="31.657288" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="36.942627" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="42.41748" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="46.911072" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="49.65271" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="61.183594" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="70.090759" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">8</tspan> + <tspan x="75.578247" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="53.017639" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">H</tspan> + </text> + </g> + </g> + <g id="text_16"> + <text style="font-size: 8.6px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="115.2" y="221.973062" transform="rotate(-0 115.2 221.973062)">RoPE</text> + </g> + <g id="text_17"> + <text style="font-size: 8.6px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="288" y="221.973062" transform="rotate(-0 288 221.973062)">RoPE</text> + </g> + <g id="text_18"> + <!-- $\theta = 5{\cdot}10^5$ --> + <g style="fill: #8a8a8a" transform="translate(361.44 221.697125)"> + <text> + <tspan x="0" y="-0.857813" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">θ</tspan> + <tspan x="6.150635" y="-0.857813" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">=</tspan> + <tspan x="14.025085" y="-0.857813" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">5</tspan> + <tspan x="18.876343" y="-0.857813" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">⋅</tspan> + <tspan x="21.30011" y="-0.857813" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">1</tspan> + <tspan x="26.151367" y="-0.857813" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">0</tspan> + <tspan x="31.079187" y="-3.920313" style="font-size: 5.32px; font-family: 'DejaVu Sans'; fill: #8a8a8a">5</tspan> + </text> + </g> + </g> + <g id="patch_32"> + <path d="M 115.2 163.838145 +Q 115.2 168.83972 115.2 172.555557 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 117.2 168.555557 +L 115.2 172.555557 +L 113.2 168.555557 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="patch_33"> + <path d="M 115.2 197.678145 +Q 115.2 202.67972 115.2 206.395557 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 117.2 202.395557 +L 115.2 206.395557 +L 113.2 202.395557 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="patch_34"> + <path d="M 288 163.838145 +Q 288 168.83972 288 172.555557 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 290 168.555557 +L 288 172.555557 +L 286 168.555557 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="patch_35"> + <path d="M 288 197.678145 +Q 288 202.67972 288 206.395557 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 290 202.395557 +L 288 206.395557 +L 286 202.395557 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="patch_36"> + <path d="M 460.8 145.11999 +Q 460.8 159.479155 460.8 172.552581 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 462.8 168.552581 +L 460.8 172.552581 +L 458.8 168.552581 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="patch_37"> + <path d="M 115.590674 231.195344 +Q 165.597693 240.839555 214.342237 250.240288 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 210.793347 247.519005 +L 214.342237 250.240288 +L 210.035876 251.44663 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="patch_38"> + <path d="M 288 231.523418 +Q 288 240.839538 288 248.869919 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 290 244.869919 +L 288 248.869919 +L 286 244.869919 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="patch_39"> + <path d="M 460.44624 197.466987 +Q 410.399379 223.920328 361.489234 249.772834 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 365.960228 249.671794 +L 361.489234 249.772834 +L 364.090999 246.135414 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="text_19"> + <!-- matmul $S = q\,k^{\top} \cdot 1/\sqrt{h_d}$ --> + <g style="fill: #3a3a3a" transform="translate(234.336 266.756)"> + <text> + <tspan x="0" y="-0.969938" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="8.401794" y="-0.969938" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="13.687134" y="-0.969938" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="17.068909" y="-0.969938" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="25.470703" y="-0.969938" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="30.937134" y="-0.969938" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="33.333435" y="-0.969938" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="36.075073" y="-0.969938" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="45.971924" y="-0.969938" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="76.95822" y="-0.969938" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">⋅</tspan> + <tspan x="81.380217" y="-0.969938" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">1</tspan> + <tspan x="86.867705" y="-0.969938" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">/</tspan> + <tspan x="38.816711" y="-0.969938" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">S</tspan> + <tspan x="54.879089" y="-0.969938" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">q</tspan> + <tspan x="61.75427" y="-0.969938" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">k</tspan> + <tspan x="96.926332" y="-0.61875" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="68.596288" y="-4.41525" style="font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">⊤</tspan> + <tspan x="89.773589" y="-1.1715" style="font-size: 5.73953px; font-family: 'STIXSizeOneSym'; fill: #3a3a3a">√</tspan> + <tspan x="102.392763" y="0.857812" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + </text> + <rect x="95.851332" y="-10.925" width="11.694922" height="0.5375"/> + </g> + </g> + <g id="text_20"> + <!-- $(B,H,T,T)$ --> + <g style="fill: #8a8a8a" transform="translate(414 264.897125)"> + <text> + <tspan x="0" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">(</tspan> + <tspan x="8.205811" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">,</tspan> + <tspan x="17.848755" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">,</tspan> + <tspan x="26.41571" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">,</tspan> + <tspan x="34.982666" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">)</tspan> + <tspan x="2.974792" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">B</tspan> + <tspan x="12.115112" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">H</tspan> + <tspan x="21.758057" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">T</tspan> + <tspan x="30.325012" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">T</tspan> + </text> + </g> + </g> + <g id="patch_40"> + <path d="M 288 274.718145 +Q 288 279.71972 288 283.435557 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 290 279.435557 +L 288 283.435557 +L 286 279.435557 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="text_21"> + <text style="font-size: 8.6px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="288" y="299.013062" transform="rotate(-0 288 299.013062)">causal mask</text> + </g> + <g id="patch_41"> + <path d="M 288 308.558145 +Q 288 313.55972 288 317.275557 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 290 313.275557 +L 288 317.275557 +L 286 313.275557 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="text_22"> + <text style="font-size: 8.6px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="288" y="332.853062" transform="rotate(-0 288 332.853062)">softmax over keys</text> + </g> + <g id="patch_42"> + <path d="M 288 342.398145 +Q 288 347.39972 288 351.115557 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 290 347.115557 +L 288 351.115557 +L 286 347.115557 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="text_23"> + <!-- matmul $y = A\,v$ --> + <g style="fill: #3a3a3a" transform="translate(254.632 366.685)"> + <text> + <tspan x="0" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="8.401794" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="13.687134" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="17.068909" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="25.470703" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="30.937134" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="33.333435" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="36.075073" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="45.601318" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="38.816711" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">y</tspan> + <tspan x="54.508484" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">A</tspan> + <tspan x="61.809018" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">v</tspan> + </text> + </g> + </g> + <g id="text_24"> + <!-- $(B,H,T,h_d)$ --> + <g style="fill: #8a8a8a" transform="translate(414 366.417125)"> + <text> + <tspan x="0" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">(</tspan> + <tspan x="8.205811" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">,</tspan> + <tspan x="17.848755" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">,</tspan> + <tspan x="26.41571" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">,</tspan> + <tspan x="38.764465" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">)</tspan> + <tspan x="2.974792" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">B</tspan> + <tspan x="12.115112" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">H</tspan> + <tspan x="21.758057" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">T</tspan> + <tspan x="30.325012" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">h</tspan> + <tspan x="35.157654" y="0.390625" style="font-style: oblique; font-size: 5.32px; font-family: 'DejaVu Sans'; fill: #8a8a8a">d</tspan> + </text> + </g> + </g> + <g id="patch_43"> + <path d="M 288 376.238145 +Q 288 381.23972 288 384.955557 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 290 380.955557 +L 288 384.955557 +L 286 380.955557 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="text_25"> + <text style="font-size: 8.6px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="288" y="400.533062" transform="rotate(-0 288 400.533062)">merge heads</text> + </g> + <g id="text_26"> + <!-- $(B,T,C)$ --> + <g style="fill: #8a8a8a" transform="translate(414 400.257125)"> + <text> + <tspan x="0" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">(</tspan> + <tspan x="8.205811" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">,</tspan> + <tspan x="16.772766" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">,</tspan> + <tspan x="26.006165" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">)</tspan> + <tspan x="2.974792" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">B</tspan> + <tspan x="12.115112" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">T</tspan> + <tspan x="20.682068" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">C</tspan> + </text> + </g> + </g> + <g id="patch_44"> + <path d="M 288 410.078145 +Q 288 415.07972 288 418.795557 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 290 414.795557 +L 288 418.795557 +L 286 414.795557 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="text_27"> + <!-- matmul $W_{proj}: C \to C$ --> + <g style="fill: #3a3a3a" transform="translate(244.011 433.935)"> + <text> + <tspan x="0" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="8.401794" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="13.687134" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="17.068909" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="25.470703" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="30.937134" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="33.333435" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="36.075073" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="60.957166" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">:</tspan> + <tspan x="73.246106" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">→</tspan> + <tspan x="38.816711" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">W</tspan> + <tspan x="65.543408" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">C</tspan> + <tspan x="82.153271" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">C</tspan> + <tspan x="47.344849" y="1.320312" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="51.177246" y="1.320312" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="53.65946" y="1.320312" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="57.353302" y="1.320312" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">j</tspan> + </text> + </g> + </g> + <g id="patch_45"> + <path d="M 288 443.918145 +Q 288 448.91972 288 452.635557 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 290 448.635557 +L 288 452.635557 +L 286 448.635557 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="text_28"> + <!-- RMSNorm$_{attn}$ --> + <g style="fill: #3a3a3a" transform="translate(260.48 468.213062)"> + <text> + <tspan x="0" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">R</tspan> + <tspan x="5.992859" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">M</tspan> + <tspan x="13.434448" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">S</tspan> + <tspan x="18.909302" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">N</tspan> + <tspan x="25.361206" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="30.638123" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="34.184143" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="42.67207" y="1.164062" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="46.371808" y="1.164062" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="48.73905" y="1.164062" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="51.106293" y="1.164062" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + </text> + </g> + </g> + <g id="text_29"> + <text style="font-size: 7.6px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; fill: #3a3a3a" transform="translate(414 463.681956)">◄ OLMo2: norm on the sublayer OUTPUT,</text> + <text style="font-size: 7.6px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; fill: #3a3a3a" transform="translate(414 472.192294)"> not on its input</text> + </g> + <g id="patch_46"> + <path d="M 288 477.757441 +Q 288 484.199288 288 489.355395 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 290 485.355395 +L 288 489.355395 +L 286 485.355395 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="patch_47"> + <path d="M 288 519.12 +C 291.627978 519.12 295.107853 517.678589 297.673221 515.113221 +C 300.238589 512.547853 301.68 509.067978 301.68 505.44 +C 301.68 501.812022 300.238589 498.332147 297.673221 495.766779 +C 295.107853 493.201411 291.627978 491.76 288 491.76 +C 284.372022 491.76 280.892147 493.201411 278.326779 495.766779 +C 275.761411 498.332147 274.32 501.812022 274.32 505.44 +C 274.32 509.067978 275.761411 512.547853 278.326779 515.113221 +C 280.892147 517.678589 284.372022 519.12 288 519.12 +z +" clip-path="url(#pdba9adb895)" style="fill: #ffffff; stroke: #3a3a3a; stroke-width: 1.3; stroke-linejoin: miter"/> + </g> + <g id="text_30"> + <text style="font-weight: 700; font-size: 11px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="288" y="508.475312" transform="rotate(-0 288 508.475312)">+</text> + </g> + <g id="patch_48"> + <path d="M 288 519.517617 +Q 288 528.47975 288 536.156143 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 290 532.156143 +L 288 536.156143 +L 286 532.156143 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="text_31"> + <!-- $z'$ --> + <g style="fill: #3a3a3a" transform="translate(303.84 530.519375)"> + <text> + <tspan x="0" y="-0.25625" style="font-style: oblique; font-size: 10px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="5.695638" y="-4.084375" style="font-size: 7px; font-family: 'cmsy10'; fill: #3a3a3a">0</tspan> + </text> + </g> + </g> + <g id="text_32"> + <text style="font-style: italic; font-size: 9.5px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; fill: #8a8a8a" transform="translate(25.661406 702.350469) rotate(-90)">SwiGLU sublayer</text> + </g> + <g id="patch_49"> + <path d="M 633.19823 750.24524 +Q 467.999756 752.400003 304.366397 754.534351 +" style="fill: none; stroke: #555555; stroke-width: 1.4; stroke-linecap: round"/> + <path d="M 308.392142 756.482012 +L 304.366397 754.534351 +L 308.339972 752.482352 +z +" style="fill: #555555; stroke: #555555; stroke-width: 1.4; stroke-linecap: round"/> + </g> + <g id="patch_50"> + <path d="M 273.208447 544.40543 +Q 233.999032 552.960211 196.045804 561.240915 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 200.380202 562.342279 +L 196.045804 561.240915 +L 199.527533 558.434216 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="patch_51"> + <path d="M 302.791553 544.40543 +Q 342.000968 552.960211 379.954196 561.240915 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 376.472467 558.434216 +L 379.954196 561.240915 +L 375.619798 562.342279 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="text_33"> + <!-- matmul $W_1: C \to h_{ff}$ --> + <g style="fill: #3a3a3a" transform="translate(145.189 575.493062)"> + <text> + <tspan x="0" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="8.401794" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="13.687134" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="17.068909" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="25.470703" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="30.937134" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="33.333435" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="36.075073" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="53.112543" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">:</tspan> + <tspan x="65.401483" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">→</tspan> + <tspan x="38.816711" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">W</tspan> + <tspan x="57.698785" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">C</tspan> + <tspan x="74.308649" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="47.344849" y="1.320312" style="font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">1</tspan> + <tspan x="79.775079" y="1.320312" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="81.900586" y="1.320312" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + </text> + </g> + </g> + <g id="text_34"> + <!-- matmul $W_3: C \to h_{ff}$ --> + <g style="fill: #3a3a3a" transform="translate(346.789 575.493062)"> + <text> + <tspan x="0" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="8.401794" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="13.687134" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="17.068909" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="25.470703" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="30.937134" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="33.333435" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="36.075073" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="53.112543" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">:</tspan> + <tspan x="65.401483" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">→</tspan> + <tspan x="38.816711" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">W</tspan> + <tspan x="57.698785" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">C</tspan> + <tspan x="74.308649" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="47.344849" y="1.320312" style="font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">3</tspan> + <tspan x="79.775079" y="1.320312" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="81.900586" y="1.320312" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + </text> + </g> + </g> + <g id="text_35"> + <!-- $h_{ff} = 1408$ --> + <g style="fill: #8a8a8a" transform="translate(473.04 575.217125)"> + <text> + <tspan x="0" y="-0.921875" style="font-style: oblique; font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">h</tspan> + <tspan x="4.832642" y="0.390625" style="font-style: oblique; font-size: 5.32px; font-family: 'DejaVu Sans'; fill: #8a8a8a">f</tspan> + <tspan x="6.711713" y="0.390625" style="font-style: oblique; font-size: 5.32px; font-family: 'DejaVu Sans'; fill: #8a8a8a">f</tspan> + <tspan x="10.295068" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">=</tspan> + <tspan x="18.169519" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">1</tspan> + <tspan x="23.020776" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">4</tspan> + <tspan x="27.872034" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">0</tspan> + <tspan x="32.723291" y="-0.921875" style="font-size: 7.6px; font-family: 'DejaVu Sans'; fill: #8a8a8a">8</tspan> + </text> + </g> + </g> + <g id="patch_52"> + <path d="M 187.2 585.038145 +Q 187.2 590.03972 187.2 593.755557 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 189.2 589.755557 +L 187.2 593.755557 +L 185.2 589.755557 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="text_36"> + <text style="font-size: 8.6px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="187.2" y="609.333062" transform="rotate(-0 187.2 609.333062)">SiLU</text> + </g> + <g id="patch_53"> + <path d="M 187.58562 618.582832 +Q 214.198105 625.679495 239.568263 632.44487 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 236.218648 629.48175 +L 239.568263 632.44487 +L 235.187998 633.346689 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="patch_54"> + <path d="M 388.500073 584.907935 +Q 361.799167 608.760744 336.057115 631.756977 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 340.372587 630.583646 +L 336.057115 631.756977 +L 337.707733 627.600601 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="text_37"> + <text style="font-size: 8.6px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="288" y="647.493062" transform="rotate(-0 288 647.493062)">⊙ elementwise gate</text> + </g> + <g id="patch_55"> + <path d="M 288 657.038145 +Q 288 662.03972 288 665.755557 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 290 661.755557 +L 288 665.755557 +L 286 661.755557 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="text_38"> + <!-- matmul $W_2: h_{ff} \to C$ --> + <g style="fill: #3a3a3a" transform="translate(245.989 685.653063)"> + <text> + <tspan x="0" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="8.401794" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="13.687134" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="17.068909" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="25.470703" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="30.937134" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="33.333435" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="36.075073" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="53.112543" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">:</tspan> + <tspan x="69.342682" y="-0.15625" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">→</tspan> + <tspan x="38.816711" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">W</tspan> + <tspan x="57.698785" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="78.249847" y="-0.15625" style="font-style: oblique; font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">C</tspan> + <tspan x="47.344849" y="1.320312" style="font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">2</tspan> + <tspan x="63.165216" y="1.320312" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="65.290723" y="1.320312" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + </text> + </g> + </g> + <g id="patch_56"> + <path d="M 288 695.198145 +Q 288 700.19972 288 703.915557 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 290 699.915557 +L 288 703.915557 +L 286 699.915557 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="text_39"> + <!-- RMSNorm$_{ff}$ --> + <g style="fill: #3a3a3a" transform="translate(264.479 719.493062)"> + <text> + <tspan x="0" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">R</tspan> + <tspan x="5.992859" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">M</tspan> + <tspan x="13.434448" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">S</tspan> + <tspan x="18.909302" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">N</tspan> + <tspan x="25.361206" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="30.638123" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="34.184143" y="-0.3125" style="font-size: 8.6px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="42.67207" y="1.164062" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="44.797577" y="1.164062" style="font-style: oblique; font-size: 6.02px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + </text> + </g> + </g> + <g id="text_40"> + <text style="font-size: 7.6px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: start; fill: #3a3a3a" x="388.8" y="719.217125" transform="rotate(-0 388.8 719.217125)">◄ norm after the sublayer, again</text> + </g> + <g id="patch_57"> + <path d="M 288 729.042187 +Q 288 734.401592 288 738.475257 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 290 734.475257 +L 288 738.475257 +L 286 734.475257 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="patch_58"> + <path d="M 288 768.24 +C 291.627978 768.24 295.107853 766.798589 297.673221 764.233221 +C 300.238589 761.667853 301.68 758.187978 301.68 754.56 +C 301.68 750.932022 300.238589 747.452147 297.673221 744.886779 +C 295.107853 742.321411 291.627978 740.88 288 740.88 +C 284.372022 740.88 280.892147 742.321411 278.326779 744.886779 +C 275.761411 747.452147 274.32 750.932022 274.32 754.56 +C 274.32 758.187978 275.761411 761.667853 278.326779 764.233221 +C 280.892147 766.798589 284.372022 768.24 288 768.24 +z +" clip-path="url(#pdba9adb895)" style="fill: #ffffff; stroke: #3a3a3a; stroke-width: 1.3; stroke-linejoin: miter"/> + </g> + <g id="text_41"> + <text style="font-weight: 700; font-size: 11px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle; fill: #3a3a3a" x="288" y="757.595312" transform="rotate(-0 288 757.595312)">+</text> + </g> + <g id="patch_59"> + <path d="M 288 768.639902 +Q 288 777.240145 288 784.554649 +" style="fill: none; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + <path d="M 290 780.554649 +L 288 784.554649 +L 286 780.554649 +z +" style="fill: #3a3a3a; stroke: #3a3a3a; stroke-width: 1.15; stroke-linecap: round"/> + </g> + <g id="text_42"> + <!-- output $z^{\prime\prime}$ — one full $f_l$ done --> + <g style="fill: #3a3a3a" transform="translate(212.8075 799.7475)"> + <text> + <tspan x="0" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="6.526611" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="13.289795" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="17.831055" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="24.631348" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="31.394531" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="35.935791" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="48.60599" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="51.913363" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="64.720736" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="68.028109" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="71.335482" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="77.862093" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="84.625277" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="91.068392" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="94.375765" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="98.508822" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="105.272005" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="108.528353" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="111.784701" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="120.557576" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="123.86495" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="130.665243" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="137.191854" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="143.955037" y="-0.40625" style="font-weight: 700; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="39.243164" y="-0.40625" style="font-style: oblique; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="115.092074" y="-0.40625" style="font-style: oblique; font-size: 9.5px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="44.676351" y="-4.234375" style="font-size: 6.65px; font-family: 'cmsy10'; fill: #3a3a3a">0</tspan> + <tspan x="46.504451" y="-4.234375" style="font-size: 6.65px; font-family: 'cmsy10'; fill: #3a3a3a">0</tspan> + <tspan x="55.220736" y="-0.40625" style="font-size: 9.5px; font-family: 'DejaVu Sans Display'; fill: #3a3a3a">—</tspan> + <tspan x="118.436556" y="1.234375" style="font-style: oblique; font-size: 6.65px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + </text> + </g> + </g> + <g id="text_43"> + <!-- weight × activation matmul — 7 logical ($W_{qkv}$ fuses $W_q,W_k,W_v$); crossbar-mappable on analog hardware --> + <g style="fill: #3a3a3a" transform="translate(68.4 840.645)"> + <text> + <tspan x="0" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">w</tspan> + <tspan x="6.747437" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="11.82312" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="14.115234" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">g</tspan> + <tspan x="19.352051" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="24.580811" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="27.815552" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="30.437988" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">×</tspan> + <tspan x="37.350586" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="39.973022" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="45.028564" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="49.564453" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="52.799194" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="55.091309" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">v</tspan> + <tspan x="59.973633" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="65.029175" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="68.263916" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="70.55603" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="75.603516" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="80.832275" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="83.454712" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="91.491211" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="96.546753" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="99.781494" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="107.817993" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="113.046753" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="115.338867" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="126.211304" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="128.83374" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">7</tspan> + <tspan x="134.082642" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="136.705078" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="138.997192" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="144.044678" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">g</tspan> + <tspan x="149.281494" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="151.573608" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="156.109497" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="161.165039" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="163.457153" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="166.07959" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="188.102026" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="190.724463" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="193.628882" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="198.857642" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="203.155859" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="208.231543" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="212.529761" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="227.194067" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">,</tspan> + <tspan x="243.144214" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">,</tspan> + <tspan x="259.167676" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="262.386304" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">;</tspan> + <tspan x="265.165845" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="267.788281" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="272.32417" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="275.716016" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="280.763501" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="285.061719" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="289.359937" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">b</tspan> + <tspan x="294.596753" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="299.652295" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="303.044141" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">-</tspan> + <tspan x="305.896069" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="313.932568" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="318.98811" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="324.224927" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="329.461743" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="334.517285" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">b</tspan> + <tspan x="339.754102" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="342.046216" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="347.121899" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="349.744336" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="354.791821" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="360.020581" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="362.643018" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="367.69856" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="372.927319" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="377.982861" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="380.274976" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="385.322461" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">g</tspan> + <tspan x="390.559277" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="393.181714" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="398.410474" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="403.466016" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="406.857861" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="412.094678" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">w</tspan> + <tspan x="418.842114" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="423.897656" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="427.289502" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="117.961304" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans Display'; fill: #3a3a3a">—</tspan> + <tspan x="169.298218" y="-0.921875" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">W</tspan> + <tspan x="215.152197" y="-0.921875" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">W</tspan> + <tspan x="231.423804" y="-0.921875" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">W</tspan> + <tspan x="247.37395" y="-0.921875" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">W</tspan> + <tspan x="177.455566" y="0.390625" style="font-style: oblique; font-size: 5.74px; font-family: 'DejaVu Sans'; fill: #3a3a3a">q</tspan> + <tspan x="181.121338" y="0.390625" style="font-style: oblique; font-size: 5.74px; font-family: 'DejaVu Sans'; fill: #3a3a3a">k</tspan> + <tspan x="184.465649" y="0.390625" style="font-style: oblique; font-size: 5.74px; font-family: 'DejaVu Sans'; fill: #3a3a3a">v</tspan> + <tspan x="223.309546" y="0.390625" style="font-style: oblique; font-size: 5.74px; font-family: 'DejaVu Sans'; fill: #3a3a3a">q</tspan> + <tspan x="239.581152" y="0.390625" style="font-style: oblique; font-size: 5.74px; font-family: 'DejaVu Sans'; fill: #3a3a3a">k</tspan> + <tspan x="255.531299" y="0.390625" style="font-style: oblique; font-size: 5.74px; font-family: 'DejaVu Sans'; fill: #3a3a3a">v</tspan> + </text> + </g> + </g> + <g id="text_44"> + <!-- activation × activation matmul — 2 ($q\,k^{\top}$ and $A\,v$); computed on the fly: the non-crossbar part of attention --> + <g style="fill: #3a3a3a" transform="translate(68.4 866.027)"> + <text> + <tspan x="0" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="5.055542" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="9.591431" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="12.826172" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="15.118286" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">v</tspan> + <tspan x="20.00061" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="25.056152" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="28.290894" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="30.583008" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="35.630493" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="40.859253" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="43.481689" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">×</tspan> + <tspan x="50.394287" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="53.016724" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="58.072266" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="62.608154" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="65.842896" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="68.13501" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">v</tspan> + <tspan x="73.017334" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="78.072876" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="81.307617" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="83.599731" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="88.647217" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="93.875977" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="96.498413" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="104.534912" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="109.590454" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="112.825195" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="120.861694" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="126.090454" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="128.382568" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="139.255005" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="141.877441" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">2</tspan> + <tspan x="147.126343" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="149.748779" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">(</tspan> + <tspan x="172.41652" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="175.038957" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="180.094499" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="185.323258" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="190.560075" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="205.047956" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">)</tspan> + <tspan x="208.266584" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">;</tspan> + <tspan x="211.046125" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="213.668561" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="218.20445" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="223.251935" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="231.288434" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="236.525251" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="241.75401" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="244.988751" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="250.064435" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="255.301251" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="257.923688" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="262.971173" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="268.199933" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="270.82237" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="274.057111" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="279.285871" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="284.361554" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="286.983991" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="289.88841" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="292.180524" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">y</tspan> + <tspan x="297.062848" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">:</tspan> + <tspan x="299.842389" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="302.464826" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="305.699567" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="310.928327" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="316.00401" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="318.626447" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="323.855207" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="328.902692" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="334.131452" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">-</tspan> + <tspan x="337.10838" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="341.644269" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="345.036115" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="350.0836" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="354.381818" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="358.680036" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">b</tspan> + <tspan x="363.916852" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="368.972394" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="372.36424" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="374.986676" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="380.223493" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="385.279035" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="388.67088" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="391.905622" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="394.528058" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="399.575543" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="402.479962" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="405.102399" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="410.157941" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="413.392682" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="416.627423" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="421.703107" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="426.931867" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="430.166608" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="432.458722" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="437.506208" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="131.005005" y="-0.010938" style="font-size: 8.2px; font-family: 'DejaVu Sans Display'; fill: #3a3a3a">—</tspan> + <tspan x="152.967407" y="-0.010938" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">q</tspan> + <tspan x="159.543667" y="-0.010938" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">k</tspan> + <tspan x="193.182511" y="-0.010938" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">A</tspan> + <tspan x="200.165631" y="-0.010938" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">v</tspan> + <tspan x="166.042094" y="-3.073438" style="font-size: 5.74px; font-family: 'DejaVu Sans'; fill: #3a3a3a">⊤</tspan> + </text> + </g> + </g> + <g id="text_45"> + <text style="font-size: 8.2px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: start; fill: #3a3a3a" x="68.4" y="890.022687" transform="rotate(-0 68.4 890.022687)">RMSNorm (learned gain)</text> + </g> + <g id="text_46"> + <text style="font-size: 8.2px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: start; fill: #3a3a3a" x="306" y="890.022687" transform="rotate(-0 306 890.022687)">elementwise / reshape</text> + </g> + <g id="text_47"> + <!-- No biases anywhere in the network. The $1/\sqrt{h_d}$ scale is folded into $S$; mask and softmax run along the key axis. --> + <g style="fill: #3a3a3a" transform="translate(36 921.052)"> + <text> + <tspan x="0" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">N</tspan> + <tspan x="6.171387" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="11.218872" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="13.841309" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">b</tspan> + <tspan x="19.078125" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="21.370239" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="26.425781" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="30.723999" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="35.799683" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="40.0979" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="42.720337" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="47.775879" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="53.004639" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">y</tspan> + <tspan x="57.886963" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">w</tspan> + <tspan x="64.634399" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="69.863159" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="74.938843" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="78.330688" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="83.406372" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="86.028809" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="88.320923" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="93.549683" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="96.172119" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="99.40686" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="104.63562" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="109.711304" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="112.33374" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="117.5625" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="122.638184" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="125.872925" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">w</tspan> + <tspan x="132.620361" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="137.667847" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="141.059692" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">k</tspan> + <tspan x="145.83728" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">.</tspan> + <tspan x="148.459717" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="151.082153" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="153.70459" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">T</tspan> + <tspan x="158.744019" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="163.972778" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="169.048462" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="171.670898" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">1</tspan> + <tspan x="176.9198" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">/</tspan> + <tspan x="196.808255" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="199.430692" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="203.728909" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="208.264798" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="213.32034" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="215.612454" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="220.688138" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="223.310574" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="225.602689" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="229.900906" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="232.523343" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="235.427762" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="240.475247" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="242.767361" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="248.004178" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="253.079861" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="258.316678" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="260.939114" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="263.231229" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="268.459988" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="271.69473" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="276.742215" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="284.601468" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">;</tspan> + <tspan x="287.381009" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="290.003445" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="298.039944" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="303.095486" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="307.393704" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">k</tspan> + <tspan x="312.171292" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="314.793729" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="319.849271" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="325.07803" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="330.314847" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="332.937283" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="337.235501" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="342.282986" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="345.187405" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="348.422147" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="356.458646" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="361.514188" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">x</tspan> + <tspan x="366.396512" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="369.018948" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="372.410794" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="377.639554" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="382.868314" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="385.49075" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="390.546292" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="392.838406" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="397.885892" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="403.114651" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">g</tspan> + <tspan x="408.351468" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="410.973904" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="414.208646" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="419.437405" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="424.513089" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="427.135526" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">k</tspan> + <tspan x="431.913113" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="436.988797" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">y</tspan> + <tspan x="441.871121" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="444.493558" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="449.5491" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">x</tspan> + <tspan x="454.431424" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="456.723538" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="461.021756" y="-0.818938" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">.</tspan> + <tspan x="179.699341" y="-1.1955" style="font-size: 5.564038px; font-family: 'STIXSizeOneSym'; fill: #3a3a3a">√</tspan> + <tspan x="186.669974" y="-0.409375" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="279.364651" y="-0.818938" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">S</tspan> + <tspan x="191.898734" y="0.903125" style="font-style: oblique; font-size: 5.74px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + </text> + <rect x="185.644974" y="-10.975" width="11.163281" height="0.5125"/> + </g> + </g> + <g id="text_48"> + <!-- $h_{ff} = 1408 \approx 8C/3$, rounded up to a multiple of 64. QK-norm acts on the full width $C$, before the head split. --> + <g style="fill: #3a3a3a" transform="translate(36 938.975)"> + <text> + <tspan x="0" y="-0.921875" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="56.012598" y="-0.921875" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">C</tspan> + <tspan x="337.75332" y="-0.921875" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">C</tspan> + <tspan x="5.22876" y="0.390625" style="font-style: oblique; font-size: 5.74px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="7.261853" y="0.390625" style="font-style: oblique; font-size: 5.74px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="11.120996" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">=</tspan> + <tspan x="19.640894" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">1</tspan> + <tspan x="24.889795" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">4</tspan> + <tspan x="30.138696" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">0</tspan> + <tspan x="35.387598" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">8</tspan> + <tspan x="42.243799" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">≈</tspan> + <tspan x="50.763696" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">8</tspan> + <tspan x="61.773096" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">/</tspan> + <tspan x="64.552637" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">3</tspan> + <tspan x="69.801538" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">,</tspan> + <tspan x="72.423975" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="75.046411" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="78.438257" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="83.485742" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="88.714502" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="93.943262" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="99.180078" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="104.255762" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="109.492578" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="112.115015" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="117.343774" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="122.580591" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="125.203027" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="128.437769" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="133.485254" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="136.10769" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="141.163232" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="143.785669" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="151.822168" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="157.050928" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="159.343042" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="162.577783" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="164.869897" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="170.106714" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="172.398828" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="177.474512" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="180.096948" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="185.144434" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="188.048853" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="190.671289" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">6</tspan> + <tspan x="195.92019" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">4</tspan> + <tspan x="201.169092" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">.</tspan> + <tspan x="203.791528" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="206.413965" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="209.036401" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">Q</tspan> + <tspan x="215.530054" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">K</tspan> + <tspan x="220.940088" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">-</tspan> + <tspan x="223.917017" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="229.145776" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="234.193262" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="237.585107" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="245.621606" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="248.244043" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="253.299585" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="257.835474" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="261.070215" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="265.368433" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="267.990869" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="273.038354" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="278.267114" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="280.889551" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="284.124292" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="289.353052" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="294.428735" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="297.051172" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="299.955591" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="305.184351" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="307.476465" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="309.768579" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="312.391016" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">w</tspan> + <tspan x="319.138452" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="321.430566" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="326.667383" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="329.902124" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="335.130884" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="343.513818" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">,</tspan> + <tspan x="346.136255" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="348.758691" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">b</tspan> + <tspan x="353.995508" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="359.071191" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="361.97561" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="367.023096" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="370.414941" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="375.490625" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="378.113062" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="381.347803" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="386.576563" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="391.652246" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="394.274683" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="399.503442" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="404.579126" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="409.634668" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="414.871484" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="417.493921" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="421.792139" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="427.028955" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="429.321069" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="431.613184" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="434.847925" y="-0.921875" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">.</tspan> + </text> + </g> + </g> + <g id="text_49"> + <!-- The boxed pipeline, input $z$ to output $z^{\prime\prime}$, is exactly $f_l$ on the companion page; the state $z_l$ lives on the residual rail. --> + <g style="fill: #3a3a3a" transform="translate(36 958.497)"> + <text> + <tspan x="0" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">T</tspan> + <tspan x="5.039429" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="10.268188" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="15.343872" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="17.966309" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">b</tspan> + <tspan x="23.203125" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="28.25061" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">x</tspan> + <tspan x="33.132935" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="38.208618" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="43.445435" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="46.067871" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="51.304688" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="53.596802" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="58.833618" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="63.909302" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="66.201416" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="68.49353" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="73.72229" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="78.797974" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">,</tspan> + <tspan x="81.42041" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="84.042847" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="86.334961" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="91.563721" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="96.800537" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="102.029297" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="105.264038" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="112.216919" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="114.839355" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="118.074097" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="123.121582" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="125.744019" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="130.791504" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="136.020264" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="139.255005" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="144.491821" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="149.720581" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="152.955322" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="163.635313" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">,</tspan> + <tspan x="166.257749" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="168.880186" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="171.1723" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="175.470518" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="178.092955" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="183.168638" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">x</tspan> + <tspan x="188.050962" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="193.106504" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="197.642393" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="200.877134" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="203.169248" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">y</tspan> + <tspan x="208.051573" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="215.401658" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="218.024095" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="223.07158" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="228.30034" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="230.922776" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="234.157517" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="239.386277" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="244.461961" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="247.084397" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">c</tspan> + <tspan x="251.620286" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="256.667771" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">m</tspan> + <tspan x="264.70427" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="269.941087" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="274.996629" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="280.225389" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="282.517503" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="287.564988" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="292.793748" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="295.416184" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">p</tspan> + <tspan x="300.653001" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="305.708543" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">g</tspan> + <tspan x="310.945359" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="316.021043" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">;</tspan> + <tspan x="318.800584" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="321.42302" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="324.657762" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="329.886521" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="334.962205" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="337.584642" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="341.882859" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="345.117601" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="350.173142" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="353.407884" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="358.483567" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="367.259678" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="369.882115" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="372.174229" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="374.466343" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">v</tspan> + <tspan x="379.348667" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="384.424351" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="388.722569" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="391.345005" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">o</tspan> + <tspan x="396.392491" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">n</tspan> + <tspan x="401.62125" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="404.243687" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">t</tspan> + <tspan x="407.478428" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">h</tspan> + <tspan x="412.707188" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="417.782872" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="420.405308" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="423.797154" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">e</tspan> + <tspan x="428.872837" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">s</tspan> + <tspan x="433.171055" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="435.463169" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">d</tspan> + <tspan x="440.699986" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">u</tspan> + <tspan x="445.928746" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="450.984288" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="453.276402" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a"> </tspan> + <tspan x="455.898838" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">r</tspan> + <tspan x="459.290684" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">a</tspan> + <tspan x="464.346226" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">i</tspan> + <tspan x="466.63834" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="468.930455" y="-0.732812" style="font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">.</tspan> + <tspan x="107.886475" y="-0.732812" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="155.577759" y="-0.732812" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="210.674009" y="-0.732812" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">f</tspan> + <tspan x="361.106004" y="-0.732812" style="font-style: oblique; font-size: 8.2px; font-family: 'DejaVu Sans'; fill: #3a3a3a">z</tspan> + <tspan x="160.265495" y="-3.795312" style="font-size: 5.74px; font-family: 'cmsy10'; fill: #3a3a3a">0</tspan> + <tspan x="161.841029" y="-3.795312" style="font-size: 5.74px; font-family: 'cmsy10'; fill: #3a3a3a">0</tspan> + <tspan x="213.578428" y="0.579688" style="font-style: oblique; font-size: 5.74px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + <tspan x="365.436448" y="0.579688" style="font-style: oblique; font-size: 5.74px; font-family: 'DejaVu Sans'; fill: #3a3a3a">l</tspan> + </text> + </g> + </g> + </g> + </g> + <defs> + <clipPath id="pdba9adb895"> + <rect x="0" y="0" width="720" height="1022.4"/> + </clipPath> + </defs> +</svg> |