diff options
| -rw-r--r-- | README.md | 2 | ||||
| -rw-r--r-- | RESULTS.md | 3 | ||||
| -rw-r--r-- | experiments/plot_tracking_failure.py | 160 | ||||
| -rw-r--r-- | results/figs/figureS_tracking_failure.pdf | bin | 0 -> 33222 bytes | |||
| -rw-r--r-- | results/figs/figureS_tracking_failure.png | bin | 0 -> 208841 bytes | |||
| -rw-r--r-- | results/figs/figureS_tracking_failure_caption.md | 9 |
6 files changed, 172 insertions, 2 deletions
@@ -85,7 +85,7 @@ somatic statistic rather than arbitrary top-down context. `ROADMAP.md` and - `ORAL_A_V3.md`: frozen vectorizer-space causal-calibration funnel; - `REVIEW_SCORECARD.md`: adversarial ICLR-style score trajectory; - `results/figs/`: deterministic PDF/PNG main figures, captions, and a source - hash manifest. + hash manifest, plus the audited RRM endpoint-tracking failure supplement. The three current main figures show the local-method Pareto frontier, credit assignment versus depth, and the load-bearing innovation ablation. Every final @@ -884,7 +884,8 @@ the ideal tracking error follows `E_(m+1)=(1-eta_M)E_m-U_m`; once task motion slows, the final error can vanish even after the high-rate trajectory has already destabilized the model. This baseline negative does not lower or raise the SDIL score because the separate KP substrate remains under its own frozen -gate. +gate. `results/figs/figureS_tracking_failure.{pdf,png}` renders this mismatch +directly from the frozen record and gate. ### Modified Kolen--Pollack reciprocal baseline diff --git a/experiments/plot_tracking_failure.py b/experiments/plot_tracking_failure.py new file mode 100644 index 0000000..1ca888c --- /dev/null +++ b/experiments/plot_tracking_failure.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +"""Render the audited RRM-3 endpoint-alignment failure diagnostic.""" +import argparse +import hashlib +import json +import math +import os + +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt +from matplotlib.ticker import LogLocator + + +PDF_METADATA = { + "Creator": "SDIL audited figure pipeline", + "Producer": "Matplotlib", + "CreationDate": None, + "ModDate": None, +} + + +def sha256(path): + digest = hashlib.sha256() + with open(path, "rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--record", default="results/residual_mirror_full/rrm.json") + parser.add_argument( + "--gate", default="results/residual_mirror_full_gate.json") + parser.add_argument( + "--out", default="results/figs/figureS_tracking_failure") + args = parser.parse_args() + with open(args.record) as handle: + record = json.load(handle) + with open(args.gate) as handle: + gate = json.load(handle) + if gate.get("protocol") != "residual_response_mirror_full_v1": + raise ValueError("unexpected RRM full protocol") + if gate.get("status") != "failed": + raise ValueError("tracking-failure figure requires the failed frozen gate") + if gate["metrics"]["source_commit"] != record["provenance"]["git_commit"]: + raise ValueError("record/gate provenance mismatch") + if len(record["epochs"]) != 200: + raise ValueError("RRM trajectory is incomplete") + + epochs = [int(row["epoch"]) for row in record["epochs"]] + losses = [float(row["train_loss"]) for row in record["epochs"]] + if not all(math.isfinite(value) and value > 0 for value in losses): + raise ValueError("training-loss trajectory is invalid") + metrics = gate["metrics"] + values = [ + float(metrics["accuracy"]), + float(metrics["early_third_alignment"]), + float(metrics["all_layer_alignment"]), + float(metrics["mean_feedback_forward_cosine"]), + ] + references = [float(metrics["bp_accuracy"]), 1.0, 1.0, 1.0] + + plt.rcParams.update({ + "font.family": "DejaVu Sans", + "font.size": 10, + "axes.titleweight": "bold", + "axes.spines.top": False, + "axes.spines.right": False, + }) + fig, (left, right) = plt.subplots( + 1, 2, figsize=(10.4, 4.25), gridspec_kw={"width_ratios": [1.32, 1]}) + red = "#C44E52" + charcoal = "#3D3D3D" + pale_red = "#F5DDDE" + pale_blue = "#E6F0F7" + + left.axvspan(1, 100, color=pale_red, alpha=0.65, linewidth=0) + left.axvspan(100, 150, color="#F7E9D2", alpha=0.65, linewidth=0) + left.axvspan(150, 200, color=pale_blue, alpha=0.75, linewidth=0) + left.semilogy(epochs, losses, color=red, linewidth=2.2, zorder=3) + left.scatter([epochs[0]], [losses[0]], color=charcoal, s=26, zorder=4) + peak_index = max(range(len(losses)), key=losses.__getitem__) + left.scatter([epochs[peak_index]], [losses[peak_index]], color=red, + edgecolor="white", linewidth=0.8, s=45, zorder=4) + left.annotate( + f"peak {losses[peak_index]:.2e}\n(epoch {epochs[peak_index]})", + xy=(epochs[peak_index], losses[peak_index]), xytext=(114, 3.0e15), + arrowprops={"arrowstyle": "-", "color": red, "linewidth": 1}, + color=red, fontsize=9, ha="left") + left.annotate( + f"final train loss\n{losses[-1]:.2e}", + xy=(epochs[-1], losses[-1]), xytext=(155, 1.5e11), + arrowprops={"arrowstyle": "-", "color": charcoal, "linewidth": 1}, + fontsize=9, ha="left") + left.text(48, 5.0, "LR 0.1", ha="center", color="#87383B", fontsize=9) + left.text(125, 5.0, "LR 0.01", ha="center", color="#8A632B", fontsize=9) + left.text(175, 5.0, "LR 0.001", ha="center", color="#35698C", fontsize=9) + left.set_xlim(1, 200) + left.set_ylim(1, 4e16) + left.yaxis.set_major_locator(LogLocator(base=10, numticks=7)) + left.grid(axis="y", which="major", color="#DDDDDD", linewidth=0.7) + left.set_xlabel("Training epoch") + left.set_ylabel("Mean cross-entropy (log scale)") + left.set_title("a The destructive trajectory", loc="left", pad=10) + + positions = list(range(4)) + width = 0.34 + right.bar([value - width / 2 for value in positions], references, + width=width, color="#B8B8B8", label="BP / ideal reference") + bars = right.bar([value + width / 2 for value in positions], values, + width=width, color=red, label="RRM-3 endpoint") + labels = ["Validation\naccuracy", "Early\nalignment", + "All-layer\nalignment", "Q/W\ncosine"] + right.set_xticks(positions, labels) + right.set_ylim(0, 1.08) + right.set_ylabel("Normalized endpoint metric") + right.grid(axis="y", color="#E2E2E2", linewidth=0.7) + right.set_title("b The reassuring endpoint snapshot", loc="left", pad=10) + right.legend(frameon=False, loc="lower right", fontsize=8.5) + for bar, value in zip(bars, values): + right.text(bar.get_x() + bar.get_width() / 2, value + 0.025, + f"{value:.3f}", ha="center", va="bottom", + color=red, fontsize=8.5, fontweight="bold") + right.text( + 0.98, 0.49, + "Final validation loss: NaN\nQ/W tracking recovered; task learning did not", + transform=right.transAxes, ha="right", va="center", fontsize=9, + color="#6F2B2E", + bbox={"boxstyle": "round,pad=0.45", "facecolor": pale_red, + "edgecolor": "none"}) + + fig.suptitle( + "A quiet tail repairs the tracker, not the learner", + x=0.5, y=1.015, fontsize=14, fontweight="bold") + fig.tight_layout(w_pad=2.7) + os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) + png = args.out + ".png" + pdf = args.out + ".pdf" + fig.savefig(png, dpi=240, bbox_inches="tight", facecolor="white") + fig.savefig(pdf, bbox_inches="tight", metadata=PDF_METADATA, + facecolor="white") + plt.close(fig) + output = { + "record": args.record, + "record_sha256": sha256(args.record), + "gate": args.gate, + "gate_sha256": sha256(args.gate), + "png": png, + "png_sha256": sha256(png), + "pdf": pdf, + "pdf_sha256": sha256(pdf), + } + print(json.dumps(output, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/results/figs/figureS_tracking_failure.pdf b/results/figs/figureS_tracking_failure.pdf Binary files differnew file mode 100644 index 0000000..6943f67 --- /dev/null +++ b/results/figs/figureS_tracking_failure.pdf diff --git a/results/figs/figureS_tracking_failure.png b/results/figs/figureS_tracking_failure.png Binary files differnew file mode 100644 index 0000000..f5344e7 --- /dev/null +++ b/results/figs/figureS_tracking_failure.png diff --git a/results/figs/figureS_tracking_failure_caption.md b/results/figs/figureS_tracking_failure_caption.md new file mode 100644 index 0000000..8a9a500 --- /dev/null +++ b/results/figs/figureS_tracking_failure_caption.md @@ -0,0 +1,9 @@ +**Supplementary Figure — Endpoint alignment is not a trajectory certificate.** +**a,** The sole frozen RRM-3 run becomes unstable before the first learning-rate +drop: mean training loss peaks at `1.5808e16` in epoch 92. Lower rates reduce +the loss to `8.9503e13` by epoch 200 but do not restore the classifier. +**b,** The final snapshot is deceptively reassuring: early/all-layer teaching +alignment is `0.877/0.849`, feedback/forward cosine is `0.999998`, and feedback +norms are controlled, yet validation accuracy is 10% and validation loss is +NaN. BP accuracy (`91.62%`) and unit ideal references are shown in gray. This +is a failed inherited residual-mirror baseline, not positive SDIL evidence. |
