path: root/external/dualprop_patches/0013-analysis-audit-plain-CNN-P1-selector.patch

From 46b74984587f7b76c40f91cfccd3a6939695a121 Mon Sep 17 00:00:00 2001
From: YurenHao0426 <Blackhao0426@gmail.com>
Date: Mon, 27 Jul 2026 13:45:04 -0500
Subject: [PATCH 13/19] analysis: audit plain CNN P1 selector

---
 analyze_p1.py | 241 ++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 241 insertions(+)
 create mode 100644 analyze_p1.py

diff --git a/analyze_p1.py b/analyze_p1.py
new file mode 100644
index 0000000..05dd52d
--- /dev/null
+++ b/analyze_p1.py
@@ -0,0 +1,241 @@
+#!/usr/bin/env python3
+"""Audit and mechanically select the frozen plain-CNN P1 learning rates."""
+import argparse
+import glob
+import hashlib
+import json
+import math
+import os
+
+import numpy as np
+
+
+ROOT = os.path.dirname(os.path.abspath(__file__))
+EXPECTED_SOURCE = {
+    "author_crossover_commit": "90697e8a17ab5e0757a62df40da521766763abe1",
+    "main_commit": "dab65f657ed22ce90fe2b38a360ca62cc8b00e65",
+    "protocol_sha256":
+        "345e0c1245f558a68ea2c212dfa427bdaa30eafd2c8d16c6763691ba354dae61",
+}
+SPECIFICATIONS = (
+    ("bp", 0.025),
+    ("fa", 0.003),
+    ("fa", 0.01),
+    ("fa", 0.03),
+    ("dfa", 0.003),
+    ("dfa", 0.01),
+    ("dfa", 0.03),
+    ("pepita", 0.01),
+    ("ff", 0.03),
+    ("ep", 0.003),
+    ("ep", 0.01),
+    ("ep", 0.03),
+    ("dualprop", 0.025),
+    ("clean_kp", 0.003),
+    ("clean_kp", 0.01),
+    ("clean_kp", 0.03),
+    ("sdil", 0.003),
+    ("sdil", 0.01),
+    ("sdil", 0.03),
+)
+GRIDDED_METHODS = ("fa", "dfa", "ep", "clean_kp", "sdil")
+CLI_METHODS = {
+    "bp": "backprop",
+    "clean_kp": "clean-kp",
+    "dualprop": "dualprop-lagr-ff",
+}
+
+
+def rate_tag(rate):
+    return f"{rate:g}".replace(".", "p")
+
+
+def experiment_name(method, rate):
+    return f"plain-p1-{method}-minicnn-lr{rate_tag(rate)}"
+
+
+def sha256(path):
+    digest = hashlib.sha256()
+    with open(path, "rb") as handle:
+        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def flag(command, name):
+    try:
+        index = command.index(name)
+    except ValueError as error:
+        raise AssertionError(f"missing command flag {name}") from error
+    if index + 1 >= len(command):
+        raise AssertionError(f"missing value for command flag {name}")
+    return command[index + 1]
+
+
+def scalar(value):
+    return float(np.asarray(value))
+
+
+def audit_histogram(path, method):
+    history = np.load(path, allow_pickle=True).item()
+    if method == "ff":
+        assert history["method"] == "ff"
+        assert history["epochs_per_layer"] == 10
+        assert len(history["layers"]) == history["num_layers"]
+        assert all(
+            len(layer["epochs"]) == 10 for layer in history["layers"])
+        final = history["final"]
+        best_accuracy = float(final["validation_accuracy"])
+        final_accuracy = best_accuracy
+        final_loss = float(history["layers"][-1]["epochs"][-1]["loss"])
+        test_values = (final["test_accuracy"], final["test_time"])
+        epochs_completed = sum(
+            len(layer["epochs"]) for layer in history["layers"])
+        best_epoch = None
+        train_seconds = sum(
+            epoch["runtime"] for layer in history["layers"]
+            for epoch in layer["epochs"])
+        validation_seconds = float(final["validation_time"])
+    else:
+        validation_accuracy = np.asarray(
+            history["val_accuracy"], dtype=float)
+        validation_loss = np.asarray(history["val_loss"], dtype=float)
+        assert validation_accuracy.shape == (10,)
+        assert validation_loss.shape == (10,)
+        assert int(history["epochs_completed"]) == 10
+        assert np.all(np.isfinite(validation_accuracy))
+        best_accuracy = float(np.max(validation_accuracy))
+        final_accuracy = float(validation_accuracy[-1])
+        final_loss = float(validation_loss[-1])
+        assert math.isclose(
+            best_accuracy, scalar(history["best_validation_accuracy"]),
+            rel_tol=0, abs_tol=1e-6)
+        test_values = (
+            history["test_accuracy"], history["test_loss"],
+            history["test_top5accuracy"], history["test_time"])
+        epochs_completed = 10
+        best_epoch = int(history["best_epoch"])
+        train_seconds = float(np.sum(history["train_time"]))
+        validation_seconds = float(np.sum(history["val_time"]))
+    assert all(math.isnan(scalar(value)) for value in test_values)
+    return {
+        "best_validation_accuracy": best_accuracy,
+        "final_validation_accuracy": final_accuracy,
+        "final_validation_loss": final_loss,
+        "epochs_completed": epochs_completed,
+        "best_epoch": best_epoch,
+        "train_seconds": train_seconds,
+        "validation_seconds": validation_seconds,
+        "test_metrics_all_nan": True,
+    }
+
+
+def audit_record(method, rate):
+    name = experiment_name(method, rate)
+    run_root = os.path.join(ROOT, "runs", name)
+    histograms = glob.glob(os.path.join(run_root, "*", "hist.npy"))
+    manifests = glob.glob(
+        os.path.join(run_root, "*", "crossover_manifest.json"))
+    assert len(histograms) == 1, f"{name}: found {len(histograms)} histograms"
+    assert len(manifests) == 1, f"{name}: found {len(manifests)} manifests"
+    histogram = os.path.realpath(histograms[0])
+    with open(manifests[0], encoding="utf-8") as handle:
+        manifest = json.load(handle)
+    assert manifest["stage"] == "p1"
+    assert manifest["method"] == method
+    assert manifest["architecture"] == "minicnn"
+    assert math.isclose(float(manifest["rate"]), rate)
+    assert manifest["experiment_name"] == name
+    assert os.path.realpath(manifest["histogram"]) == histogram
+    assert manifest["histogram_sha256"] == sha256(histogram)
+    for key, expected in EXPECTED_SOURCE.items():
+        assert manifest["source"][key] == expected, f"{name}: source drift"
+    command = manifest["command"]
+    assert flag(command, "--model") == "miniCNN"
+    assert flag(command, "--dataset") == "cifar10"
+    assert flag(command, "--num-epochs") == "10"
+    assert flag(command, "--seeds") == "0"
+    assert flag(command, "--feedback-seed") == "1729"
+    assert flag(command, "--test-policy") == "none"
+    assert flag(command, "--early-stop-policy") == "none"
+    assert flag(command, "--gradient-diagnostics") == "none"
+    assert flag(command, "--spectral-diagnostics") == "none"
+    assert flag(command, "--learning-algorithm") == CLI_METHODS.get(
+        method, method)
+    assert math.isclose(float(flag(command, "--learning-rate")), rate)
+    metrics = audit_histogram(histogram, method)
+    return {
+        "method": method,
+        "rate": rate,
+        "experiment_name": name,
+        "manifest": os.path.relpath(manifests[0], ROOT),
+        "histogram_sha256": manifest["histogram_sha256"],
+        "driver_wall_seconds": manifest.get("driver_wall_seconds"),
+        **metrics,
+    }
+
+
+def selected_candidate(candidates):
+    def rank(candidate):
+        loss = candidate["final_validation_loss"]
+        return (
+            candidate["best_validation_accuracy"],
+            int(math.isfinite(loss)),
+            -candidate["rate"],
+        )
+    return max(candidates, key=rank)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--output")
+    args = parser.parse_args()
+    expected_names = {
+        experiment_name(method, rate) for method, rate in SPECIFICATIONS}
+    actual_names = {
+        os.path.basename(path)
+        for path in glob.glob(os.path.join(ROOT, "runs", "plain-p1-*"))
+        if glob.glob(os.path.join(path, "*", "hist.npy"))
+    }
+    assert actual_names == expected_names, {
+        "missing": sorted(expected_names - actual_names),
+        "unexpected": sorted(actual_names - expected_names),
+    }
+    records = [
+        audit_record(method, rate) for method, rate in SPECIFICATIONS]
+    selected = {}
+    for method in dict(SPECIFICATIONS):
+        candidates = [
+            record for record in records if record["method"] == method]
+        chosen = (
+            selected_candidate(candidates)
+            if method in GRIDDED_METHODS else candidates[0])
+        selected[method] = {
+            "rate": chosen["rate"],
+            "best_validation_accuracy":
+                chosen["best_validation_accuracy"],
+            "experiment_name": chosen["experiment_name"],
+        }
+    report = {
+        "gate": "pass",
+        "stage": "plain_cnn_p1",
+        "selection_rule":
+            "maximum best validation accuracy, then finite final validation "
+            "loss, then lower learning rate",
+        "expected_source": EXPECTED_SOURCE,
+        "num_expected_records": len(SPECIFICATIONS),
+        "num_audited_records": len(records),
+        "test_policy": "none",
+        "records": records,
+        "selected": selected,
+    }
+    encoded = json.dumps(report, indent=2, sort_keys=True) + "\n"
+    if args.output:
+        with open(args.output, "w", encoding="utf-8") as handle:
+            handle.write(encoded)
+    else:
+        print(encoded, end="")
+
+
+if __name__ == "__main__":
+    main()
-- 
2.54.0