path: root/external/dualprop_patches/0017-analysis-freeze-complete-P2-audit-gate.patch

From 52f1b56e3a3fc80cc25b9ac9efc935f8d135961e Mon Sep 17 00:00:00 2001
From: YurenHao0426 <Blackhao0426@gmail.com>
Date: Mon, 27 Jul 2026 13:51:49 -0500
Subject: [PATCH 17/19] analysis: freeze complete P2 audit gate

---
 analyze_p2.py | 229 ++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 229 insertions(+)
 create mode 100644 analyze_p2.py

diff --git a/analyze_p2.py b/analyze_p2.py
new file mode 100644
index 0000000..d997ec5
--- /dev/null
+++ b/analyze_p2.py
@@ -0,0 +1,229 @@
+#!/usr/bin/env python3
+"""Audit the complete frozen 27-cell plain-CNN P2 validation panel."""
+import argparse
+import glob
+import hashlib
+import json
+import math
+import os
+
+import numpy as np
+
+import crossover_grid
+
+
+ROOT = os.path.dirname(os.path.abspath(__file__))
+STATUSES = ("completed", "nonzero_exit", "timeout", "missing_histogram")
+
+
+def sha256(path):
+    digest = hashlib.sha256()
+    with open(path, "rb") as handle:
+        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def scalar(value):
+    return float(np.asarray(value))
+
+
+def all_nan(values):
+    return all(math.isnan(scalar(value)) for value in values)
+
+
+def history_metrics(path, job):
+    history = np.load(path, allow_pickle=True).item()
+    method = job["method"]
+    expected_epochs = job["expected_epochs"]
+    if method == "ff":
+        assert history["method"] == "ff"
+        assert history["epochs_per_layer"] == expected_epochs
+        assert len(history["layers"]) == history["num_layers"]
+        layer_epochs = [
+            len(layer["epochs"]) for layer in history["layers"]]
+        assert all(count == expected_epochs for count in layer_epochs)
+        final = history["final"]
+        assert all_nan((final["test_accuracy"], final["test_time"]))
+        layer_losses = [
+            float(epoch["loss"]) for layer in history["layers"]
+            for epoch in layer["epochs"]]
+        nonfinite = [
+            index + 1 for index, value in enumerate(layer_losses)
+            if not math.isfinite(value)]
+        validation_accuracy = float(final["validation_accuracy"])
+        return {
+            "outcome": "nonfinite" if nonfinite else "finite",
+            "best_validation_accuracy": validation_accuracy,
+            "final_validation_accuracy": validation_accuracy,
+            "final_validation_loss": None,
+            "best_epoch": None,
+            "epochs_completed": sum(layer_epochs),
+            "first_nonfinite_step": nonfinite[0] if nonfinite else None,
+            "train_seconds": sum(
+                float(epoch["runtime"]) for layer in history["layers"]
+                for epoch in layer["epochs"]),
+            "validation_seconds": float(final["validation_time"]),
+            "test_metrics_all_nan": True,
+        }
+    completed = int(history["epochs_completed"])
+    assert 1 <= completed <= expected_epochs
+    validation_accuracy = np.asarray(
+        history["val_accuracy"], dtype=float)[:completed]
+    validation_loss = np.asarray(
+        history["val_loss"], dtype=float)[:completed]
+    train_loss = np.asarray(history["train_loss"], dtype=float)[:completed]
+    assert np.asarray(history["val_accuracy"]).shape == (expected_epochs,)
+    assert np.asarray(history["train_loss"]).shape == (expected_epochs,)
+    assert all_nan((
+        history["test_accuracy"], history["test_loss"],
+        history["test_top5accuracy"], history["test_time"]))
+    finite_accuracy = validation_accuracy[np.isfinite(validation_accuracy)]
+    best_accuracy = (
+        float(np.max(finite_accuracy)) if finite_accuracy.size else None)
+    nonfinite_mask = (
+        ~np.isfinite(validation_accuracy)
+        | ~np.isfinite(validation_loss)
+        | ~np.isfinite(train_loss))
+    nonfinite_indices = np.flatnonzero(nonfinite_mask)
+    first_nonfinite = (
+        int(nonfinite_indices[0]) + 1 if nonfinite_indices.size else None)
+    if first_nonfinite is None and completed != expected_epochs:
+        outcome = "premature_finite_stop"
+    elif first_nonfinite is None:
+        outcome = "finite"
+    else:
+        outcome = "nonfinite"
+    if best_accuracy is not None:
+        assert math.isclose(
+            best_accuracy, scalar(history["best_validation_accuracy"]),
+            rel_tol=0, abs_tol=1e-5)
+    return {
+        "outcome": outcome,
+        "best_validation_accuracy": best_accuracy,
+        "final_validation_accuracy": float(validation_accuracy[-1]),
+        "final_validation_loss": float(validation_loss[-1]),
+        "best_epoch": int(history["best_epoch"]),
+        "epochs_completed": completed,
+        "first_nonfinite_step": first_nonfinite,
+        "train_seconds": float(
+            np.sum(np.asarray(history["train_time"])[:completed])),
+        "validation_seconds": float(
+            np.sum(np.asarray(history["val_time"])[:completed])),
+        "test_metrics_all_nan": True,
+    }
+
+
+def record_manifest(experiment_name):
+    paths = glob.glob(os.path.join(
+        ROOT, "runs", experiment_name, "*", "crossover_manifest.json"))
+    assert len(paths) == 1, (
+        f"{experiment_name}: expected one manifest, found {len(paths)}")
+    return paths[0]
+
+
+def audit_record(job, launch):
+    manifest_path = record_manifest(job["experiment_name"])
+    with open(manifest_path, encoding="utf-8") as handle:
+        manifest = json.load(handle)
+    for key in (
+            "stage", "method", "architecture", "rate", "expected_epochs",
+            "experiment_name", "selector_path", "selector_sha256",
+            "timeout_seconds", "command"):
+        assert manifest[key] == job[key], (
+            f"{job['experiment_name']}: registry drift in {key}")
+    assert manifest["source"] == launch["source"]
+    assert manifest["status"] in STATUSES
+    assert manifest["selector_sha256"] == crossover_grid.SELECTOR_SHA256
+    assert manifest["work"] == crossover_grid.p2_work_report(job)
+    hardware = manifest["hardware"]
+    assert hardware["physical_index"] in (5, 7)
+    assert hardware["name"] == "NVIDIA GeForce GTX 1080"
+    assert hardware["peak_process_gpu_memory_mib"] >= 0
+    histogram = manifest["histogram"]
+    if histogram is None:
+        assert manifest["histogram_sha256"] is None
+        metrics = {
+            "outcome": manifest["status"],
+            "best_validation_accuracy": None,
+            "final_validation_accuracy": None,
+            "final_validation_loss": None,
+            "best_epoch": None,
+            "epochs_completed": 0,
+            "first_nonfinite_step": None,
+            "train_seconds": None,
+            "validation_seconds": None,
+            "test_metrics_all_nan": None,
+        }
+    else:
+        assert os.path.isfile(histogram)
+        assert manifest["histogram_sha256"] == sha256(histogram)
+        metrics = history_metrics(histogram, job)
+    return {
+        "method": job["method"],
+        "architecture": job["architecture"],
+        "rate": job["rate"],
+        "status": manifest["status"],
+        "manifest": os.path.relpath(manifest_path, ROOT),
+        "histogram_sha256": manifest["histogram_sha256"],
+        "driver_wall_seconds": manifest["driver_wall_seconds"],
+        "physical_gpu": hardware,
+        "work": manifest["work"],
+        **metrics,
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--output")
+    parser.add_argument("--allow-partial", action="store_true")
+    args = parser.parse_args()
+    launch_path = os.path.join(ROOT, "runs", "plain-p2-launch.json")
+    with open(launch_path, encoding="utf-8") as handle:
+        launch = json.load(handle)
+    jobs = crossover_grid.p2_jobs()
+    assert launch["stage"] == "p2"
+    assert launch["num_registered_cells"] == 27
+    assert launch["registry_sha256"] == crossover_grid.p2_registry_sha256(
+        jobs)
+    assert launch["jobs"] == jobs
+    assert launch["selector_sha256"] == crossover_grid.SELECTOR_SHA256
+    assert launch["source"]["cifar10_tfds_tree"]["num_files"] == 5
+    expected = {job["experiment_name"]: job for job in jobs}
+    actual = {
+        os.path.basename(path)
+        for path in glob.glob(os.path.join(ROOT, "runs", "plain-p2-*"))
+        if os.path.isdir(path) and glob.glob(
+            os.path.join(path, "*", "crossover_manifest.json"))
+    }
+    unexpected = actual - set(expected)
+    missing = set(expected) - actual
+    assert not unexpected, f"unexpected P2 cells: {sorted(unexpected)}"
+    if missing and not args.allow_partial:
+        raise AssertionError(f"missing P2 cells: {sorted(missing)}")
+    records = [
+        audit_record(job, launch) for job in jobs
+        if job["experiment_name"] in actual]
+    report = {
+        "gate": "pass" if not missing else "partial",
+        "stage": "plain_cnn_p2",
+        "launch_record": os.path.relpath(launch_path, ROOT),
+        "source": launch["source"],
+        "selector_sha256": launch["selector_sha256"],
+        "registry_sha256": launch["registry_sha256"],
+        "num_expected_records": len(jobs),
+        "num_audited_records": len(records),
+        "missing_experiments": sorted(missing),
+        "test_policy": "none",
+        "records": records,
+    }
+    encoded = json.dumps(report, indent=2, sort_keys=True) + "\n"
+    if args.output:
+        with open(args.output, "w", encoding="utf-8") as handle:
+            handle.write(encoded)
+    else:
+        print(encoded, end="")
+
+
+if __name__ == "__main__":
+    main()
-- 
2.54.0