From 52f1b56e3a3fc80cc25b9ac9efc935f8d135961e Mon Sep 17 00:00:00 2001 From: YurenHao0426 Date: Mon, 27 Jul 2026 13:51:49 -0500 Subject: [PATCH 17/19] analysis: freeze complete P2 audit gate --- analyze_p2.py | 229 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 229 insertions(+) create mode 100644 analyze_p2.py diff --git a/analyze_p2.py b/analyze_p2.py new file mode 100644 index 0000000..d997ec5 --- /dev/null +++ b/analyze_p2.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python3 +"""Audit the complete frozen 27-cell plain-CNN P2 validation panel.""" +import argparse +import glob +import hashlib +import json +import math +import os + +import numpy as np + +import crossover_grid + + +ROOT = os.path.dirname(os.path.abspath(__file__)) +STATUSES = ("completed", "nonzero_exit", "timeout", "missing_histogram") + + +def sha256(path): + digest = hashlib.sha256() + with open(path, "rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def scalar(value): + return float(np.asarray(value)) + + +def all_nan(values): + return all(math.isnan(scalar(value)) for value in values) + + +def history_metrics(path, job): + history = np.load(path, allow_pickle=True).item() + method = job["method"] + expected_epochs = job["expected_epochs"] + if method == "ff": + assert history["method"] == "ff" + assert history["epochs_per_layer"] == expected_epochs + assert len(history["layers"]) == history["num_layers"] + layer_epochs = [ + len(layer["epochs"]) for layer in history["layers"]] + assert all(count == expected_epochs for count in layer_epochs) + final = history["final"] + assert all_nan((final["test_accuracy"], final["test_time"])) + layer_losses = [ + float(epoch["loss"]) for layer in history["layers"] + for epoch in layer["epochs"]] + nonfinite = [ + index + 1 for index, value in enumerate(layer_losses) + if not math.isfinite(value)] + validation_accuracy = float(final["validation_accuracy"]) + return { + "outcome": "nonfinite" if nonfinite else "finite", + "best_validation_accuracy": validation_accuracy, + "final_validation_accuracy": validation_accuracy, + "final_validation_loss": None, + "best_epoch": None, + "epochs_completed": sum(layer_epochs), + "first_nonfinite_step": nonfinite[0] if nonfinite else None, + "train_seconds": sum( + float(epoch["runtime"]) for layer in history["layers"] + for epoch in layer["epochs"]), + "validation_seconds": float(final["validation_time"]), + "test_metrics_all_nan": True, + } + completed = int(history["epochs_completed"]) + assert 1 <= completed <= expected_epochs + validation_accuracy = np.asarray( + history["val_accuracy"], dtype=float)[:completed] + validation_loss = np.asarray( + history["val_loss"], dtype=float)[:completed] + train_loss = np.asarray(history["train_loss"], dtype=float)[:completed] + assert np.asarray(history["val_accuracy"]).shape == (expected_epochs,) + assert np.asarray(history["train_loss"]).shape == (expected_epochs,) + assert all_nan(( + history["test_accuracy"], history["test_loss"], + history["test_top5accuracy"], history["test_time"])) + finite_accuracy = validation_accuracy[np.isfinite(validation_accuracy)] + best_accuracy = ( + float(np.max(finite_accuracy)) if finite_accuracy.size else None) + nonfinite_mask = ( + ~np.isfinite(validation_accuracy) + | ~np.isfinite(validation_loss) + | ~np.isfinite(train_loss)) + nonfinite_indices = np.flatnonzero(nonfinite_mask) + first_nonfinite = ( + int(nonfinite_indices[0]) + 1 if nonfinite_indices.size else None) + if first_nonfinite is None and completed != expected_epochs: + outcome = "premature_finite_stop" + elif first_nonfinite is None: + outcome = "finite" + else: + outcome = "nonfinite" + if best_accuracy is not None: + assert math.isclose( + best_accuracy, scalar(history["best_validation_accuracy"]), + rel_tol=0, abs_tol=1e-5) + return { + "outcome": outcome, + "best_validation_accuracy": best_accuracy, + "final_validation_accuracy": float(validation_accuracy[-1]), + "final_validation_loss": float(validation_loss[-1]), + "best_epoch": int(history["best_epoch"]), + "epochs_completed": completed, + "first_nonfinite_step": first_nonfinite, + "train_seconds": float( + np.sum(np.asarray(history["train_time"])[:completed])), + "validation_seconds": float( + np.sum(np.asarray(history["val_time"])[:completed])), + "test_metrics_all_nan": True, + } + + +def record_manifest(experiment_name): + paths = glob.glob(os.path.join( + ROOT, "runs", experiment_name, "*", "crossover_manifest.json")) + assert len(paths) == 1, ( + f"{experiment_name}: expected one manifest, found {len(paths)}") + return paths[0] + + +def audit_record(job, launch): + manifest_path = record_manifest(job["experiment_name"]) + with open(manifest_path, encoding="utf-8") as handle: + manifest = json.load(handle) + for key in ( + "stage", "method", "architecture", "rate", "expected_epochs", + "experiment_name", "selector_path", "selector_sha256", + "timeout_seconds", "command"): + assert manifest[key] == job[key], ( + f"{job['experiment_name']}: registry drift in {key}") + assert manifest["source"] == launch["source"] + assert manifest["status"] in STATUSES + assert manifest["selector_sha256"] == crossover_grid.SELECTOR_SHA256 + assert manifest["work"] == crossover_grid.p2_work_report(job) + hardware = manifest["hardware"] + assert hardware["physical_index"] in (5, 7) + assert hardware["name"] == "NVIDIA GeForce GTX 1080" + assert hardware["peak_process_gpu_memory_mib"] >= 0 + histogram = manifest["histogram"] + if histogram is None: + assert manifest["histogram_sha256"] is None + metrics = { + "outcome": manifest["status"], + "best_validation_accuracy": None, + "final_validation_accuracy": None, + "final_validation_loss": None, + "best_epoch": None, + "epochs_completed": 0, + "first_nonfinite_step": None, + "train_seconds": None, + "validation_seconds": None, + "test_metrics_all_nan": None, + } + else: + assert os.path.isfile(histogram) + assert manifest["histogram_sha256"] == sha256(histogram) + metrics = history_metrics(histogram, job) + return { + "method": job["method"], + "architecture": job["architecture"], + "rate": job["rate"], + "status": manifest["status"], + "manifest": os.path.relpath(manifest_path, ROOT), + "histogram_sha256": manifest["histogram_sha256"], + "driver_wall_seconds": manifest["driver_wall_seconds"], + "physical_gpu": hardware, + "work": manifest["work"], + **metrics, + } + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--output") + parser.add_argument("--allow-partial", action="store_true") + args = parser.parse_args() + launch_path = os.path.join(ROOT, "runs", "plain-p2-launch.json") + with open(launch_path, encoding="utf-8") as handle: + launch = json.load(handle) + jobs = crossover_grid.p2_jobs() + assert launch["stage"] == "p2" + assert launch["num_registered_cells"] == 27 + assert launch["registry_sha256"] == crossover_grid.p2_registry_sha256( + jobs) + assert launch["jobs"] == jobs + assert launch["selector_sha256"] == crossover_grid.SELECTOR_SHA256 + assert launch["source"]["cifar10_tfds_tree"]["num_files"] == 5 + expected = {job["experiment_name"]: job for job in jobs} + actual = { + os.path.basename(path) + for path in glob.glob(os.path.join(ROOT, "runs", "plain-p2-*")) + if os.path.isdir(path) and glob.glob( + os.path.join(path, "*", "crossover_manifest.json")) + } + unexpected = actual - set(expected) + missing = set(expected) - actual + assert not unexpected, f"unexpected P2 cells: {sorted(unexpected)}" + if missing and not args.allow_partial: + raise AssertionError(f"missing P2 cells: {sorted(missing)}") + records = [ + audit_record(job, launch) for job in jobs + if job["experiment_name"] in actual] + report = { + "gate": "pass" if not missing else "partial", + "stage": "plain_cnn_p2", + "launch_record": os.path.relpath(launch_path, ROOT), + "source": launch["source"], + "selector_sha256": launch["selector_sha256"], + "registry_sha256": launch["registry_sha256"], + "num_expected_records": len(jobs), + "num_audited_records": len(records), + "missing_experiments": sorted(missing), + "test_policy": "none", + "records": records, + } + encoded = json.dumps(report, indent=2, sort_keys=True) + "\n" + if args.output: + with open(args.output, "w", encoding="utf-8") as handle: + handle.write(encoded) + else: + print(encoded, end="") + + +if __name__ == "__main__": + main() -- 2.54.0