figure: visualize endpoint tracking failure

1 files changed, 160 insertions, 0 deletions

diff --git a/experiments/plot_tracking_failure.py b/experiments/plot_tracking_failure.py
new file mode 100644
index 0000000..1ca888c
--- /dev/null
+++ b/experiments/plot_tracking_failure.py
@@ -0,0 +1,160 @@
+#!/usr/bin/env python3
+"""Render the audited RRM-3 endpoint-alignment failure diagnostic."""
+import argparse
+import hashlib
+import json
+import math
+import os
+
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+from matplotlib.ticker import LogLocator
+
+
+PDF_METADATA = {
+    "Creator": "SDIL audited figure pipeline",
+    "Producer": "Matplotlib",
+    "CreationDate": None,
+    "ModDate": None,
+}
+
+
+def sha256(path):
+    digest = hashlib.sha256()
+    with open(path, "rb") as handle:
+        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--record", default="results/residual_mirror_full/rrm.json")
+    parser.add_argument(
+        "--gate", default="results/residual_mirror_full_gate.json")
+    parser.add_argument(
+        "--out", default="results/figs/figureS_tracking_failure")
+    args = parser.parse_args()
+    with open(args.record) as handle:
+        record = json.load(handle)
+    with open(args.gate) as handle:
+        gate = json.load(handle)
+    if gate.get("protocol") != "residual_response_mirror_full_v1":
+        raise ValueError("unexpected RRM full protocol")
+    if gate.get("status") != "failed":
+        raise ValueError("tracking-failure figure requires the failed frozen gate")
+    if gate["metrics"]["source_commit"] != record["provenance"]["git_commit"]:
+        raise ValueError("record/gate provenance mismatch")
+    if len(record["epochs"]) != 200:
+        raise ValueError("RRM trajectory is incomplete")
+
+    epochs = [int(row["epoch"]) for row in record["epochs"]]
+    losses = [float(row["train_loss"]) for row in record["epochs"]]
+    if not all(math.isfinite(value) and value > 0 for value in losses):
+        raise ValueError("training-loss trajectory is invalid")
+    metrics = gate["metrics"]
+    values = [
+        float(metrics["accuracy"]),
+        float(metrics["early_third_alignment"]),
+        float(metrics["all_layer_alignment"]),
+        float(metrics["mean_feedback_forward_cosine"]),
+    ]
+    references = [float(metrics["bp_accuracy"]), 1.0, 1.0, 1.0]
+
+    plt.rcParams.update({
+        "font.family": "DejaVu Sans",
+        "font.size": 10,
+        "axes.titleweight": "bold",
+        "axes.spines.top": False,
+        "axes.spines.right": False,
+    })
+    fig, (left, right) = plt.subplots(
+        1, 2, figsize=(10.4, 4.25), gridspec_kw={"width_ratios": [1.32, 1]})
+    red = "#C44E52"
+    charcoal = "#3D3D3D"
+    pale_red = "#F5DDDE"
+    pale_blue = "#E6F0F7"
+
+    left.axvspan(1, 100, color=pale_red, alpha=0.65, linewidth=0)
+    left.axvspan(100, 150, color="#F7E9D2", alpha=0.65, linewidth=0)
+    left.axvspan(150, 200, color=pale_blue, alpha=0.75, linewidth=0)
+    left.semilogy(epochs, losses, color=red, linewidth=2.2, zorder=3)
+    left.scatter([epochs[0]], [losses[0]], color=charcoal, s=26, zorder=4)
+    peak_index = max(range(len(losses)), key=losses.__getitem__)
+    left.scatter([epochs[peak_index]], [losses[peak_index]], color=red,
+                 edgecolor="white", linewidth=0.8, s=45, zorder=4)
+    left.annotate(
+        f"peak {losses[peak_index]:.2e}\n(epoch {epochs[peak_index]})",
+        xy=(epochs[peak_index], losses[peak_index]), xytext=(114, 3.0e15),
+        arrowprops={"arrowstyle": "-", "color": red, "linewidth": 1},
+        color=red, fontsize=9, ha="left")
+    left.annotate(
+        f"final train loss\n{losses[-1]:.2e}",
+        xy=(epochs[-1], losses[-1]), xytext=(155, 1.5e11),
+        arrowprops={"arrowstyle": "-", "color": charcoal, "linewidth": 1},
+        fontsize=9, ha="left")
+    left.text(48, 5.0, "LR 0.1", ha="center", color="#87383B", fontsize=9)
+    left.text(125, 5.0, "LR 0.01", ha="center", color="#8A632B", fontsize=9)
+    left.text(175, 5.0, "LR 0.001", ha="center", color="#35698C", fontsize=9)
+    left.set_xlim(1, 200)
+    left.set_ylim(1, 4e16)
+    left.yaxis.set_major_locator(LogLocator(base=10, numticks=7))
+    left.grid(axis="y", which="major", color="#DDDDDD", linewidth=0.7)
+    left.set_xlabel("Training epoch")
+    left.set_ylabel("Mean cross-entropy (log scale)")
+    left.set_title("a   The destructive trajectory", loc="left", pad=10)
+
+    positions = list(range(4))
+    width = 0.34
+    right.bar([value - width / 2 for value in positions], references,
+              width=width, color="#B8B8B8", label="BP / ideal reference")
+    bars = right.bar([value + width / 2 for value in positions], values,
+                     width=width, color=red, label="RRM-3 endpoint")
+    labels = ["Validation\naccuracy", "Early\nalignment",
+              "All-layer\nalignment", "Q/W\ncosine"]
+    right.set_xticks(positions, labels)
+    right.set_ylim(0, 1.08)
+    right.set_ylabel("Normalized endpoint metric")
+    right.grid(axis="y", color="#E2E2E2", linewidth=0.7)
+    right.set_title("b   The reassuring endpoint snapshot", loc="left", pad=10)
+    right.legend(frameon=False, loc="lower right", fontsize=8.5)
+    for bar, value in zip(bars, values):
+        right.text(bar.get_x() + bar.get_width() / 2, value + 0.025,
+                   f"{value:.3f}", ha="center", va="bottom",
+                   color=red, fontsize=8.5, fontweight="bold")
+    right.text(
+        0.98, 0.49,
+        "Final validation loss: NaN\nQ/W tracking recovered; task learning did not",
+        transform=right.transAxes, ha="right", va="center", fontsize=9,
+        color="#6F2B2E",
+        bbox={"boxstyle": "round,pad=0.45", "facecolor": pale_red,
+              "edgecolor": "none"})
+
+    fig.suptitle(
+        "A quiet tail repairs the tracker, not the learner",
+        x=0.5, y=1.015, fontsize=14, fontweight="bold")
+    fig.tight_layout(w_pad=2.7)
+    os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True)
+    png = args.out + ".png"
+    pdf = args.out + ".pdf"
+    fig.savefig(png, dpi=240, bbox_inches="tight", facecolor="white")
+    fig.savefig(pdf, bbox_inches="tight", metadata=PDF_METADATA,
+                facecolor="white")
+    plt.close(fig)
+    output = {
+        "record": args.record,
+        "record_sha256": sha256(args.record),
+        "gate": args.gate,
+        "gate_sha256": sha256(args.gate),
+        "png": png,
+        "png_sha256": sha256(png),
+        "pdf": pdf,
+        "pdf_sha256": sha256(pdf),
+    }
+    print(json.dumps(output, indent=2, sort_keys=True))
+
+
+if __name__ == "__main__":
+    main()