diff options
Diffstat (limited to 'external/dualprop_patches/0017-analysis-freeze-complete-P2-audit-gate.patch')
| -rw-r--r-- | external/dualprop_patches/0017-analysis-freeze-complete-P2-audit-gate.patch | 248 |
1 files changed, 248 insertions, 0 deletions
diff --git a/external/dualprop_patches/0017-analysis-freeze-complete-P2-audit-gate.patch b/external/dualprop_patches/0017-analysis-freeze-complete-P2-audit-gate.patch new file mode 100644 index 0000000..2119d99 --- /dev/null +++ b/external/dualprop_patches/0017-analysis-freeze-complete-P2-audit-gate.patch @@ -0,0 +1,248 @@ +From 52f1b56e3a3fc80cc25b9ac9efc935f8d135961e Mon Sep 17 00:00:00 2001 +From: YurenHao0426 <Blackhao0426@gmail.com> +Date: Mon, 27 Jul 2026 13:51:49 -0500 +Subject: [PATCH 17/19] analysis: freeze complete P2 audit gate + +--- + analyze_p2.py | 229 ++++++++++++++++++++++++++++++++++++++++++++++++++ + 1 file changed, 229 insertions(+) + create mode 100644 analyze_p2.py + +diff --git a/analyze_p2.py b/analyze_p2.py +new file mode 100644 +index 0000000..d997ec5 +--- /dev/null ++++ b/analyze_p2.py +@@ -0,0 +1,229 @@ ++#!/usr/bin/env python3 ++"""Audit the complete frozen 27-cell plain-CNN P2 validation panel.""" ++import argparse ++import glob ++import hashlib ++import json ++import math ++import os ++ ++import numpy as np ++ ++import crossover_grid ++ ++ ++ROOT = os.path.dirname(os.path.abspath(__file__)) ++STATUSES = ("completed", "nonzero_exit", "timeout", "missing_histogram") ++ ++ ++def sha256(path): ++ digest = hashlib.sha256() ++ with open(path, "rb") as handle: ++ for chunk in iter(lambda: handle.read(1024 * 1024), b""): ++ digest.update(chunk) ++ return digest.hexdigest() ++ ++ ++def scalar(value): ++ return float(np.asarray(value)) ++ ++ ++def all_nan(values): ++ return all(math.isnan(scalar(value)) for value in values) ++ ++ ++def history_metrics(path, job): ++ history = np.load(path, allow_pickle=True).item() ++ method = job["method"] ++ expected_epochs = job["expected_epochs"] ++ if method == "ff": ++ assert history["method"] == "ff" ++ assert history["epochs_per_layer"] == expected_epochs ++ assert len(history["layers"]) == history["num_layers"] ++ layer_epochs = [ ++ len(layer["epochs"]) for layer in history["layers"]] ++ assert all(count == expected_epochs for count in layer_epochs) ++ final = history["final"] ++ assert all_nan((final["test_accuracy"], final["test_time"])) ++ layer_losses = [ ++ float(epoch["loss"]) for layer in history["layers"] ++ for epoch in layer["epochs"]] ++ nonfinite = [ ++ index + 1 for index, value in enumerate(layer_losses) ++ if not math.isfinite(value)] ++ validation_accuracy = float(final["validation_accuracy"]) ++ return { ++ "outcome": "nonfinite" if nonfinite else "finite", ++ "best_validation_accuracy": validation_accuracy, ++ "final_validation_accuracy": validation_accuracy, ++ "final_validation_loss": None, ++ "best_epoch": None, ++ "epochs_completed": sum(layer_epochs), ++ "first_nonfinite_step": nonfinite[0] if nonfinite else None, ++ "train_seconds": sum( ++ float(epoch["runtime"]) for layer in history["layers"] ++ for epoch in layer["epochs"]), ++ "validation_seconds": float(final["validation_time"]), ++ "test_metrics_all_nan": True, ++ } ++ completed = int(history["epochs_completed"]) ++ assert 1 <= completed <= expected_epochs ++ validation_accuracy = np.asarray( ++ history["val_accuracy"], dtype=float)[:completed] ++ validation_loss = np.asarray( ++ history["val_loss"], dtype=float)[:completed] ++ train_loss = np.asarray(history["train_loss"], dtype=float)[:completed] ++ assert np.asarray(history["val_accuracy"]).shape == (expected_epochs,) ++ assert np.asarray(history["train_loss"]).shape == (expected_epochs,) ++ assert all_nan(( ++ history["test_accuracy"], history["test_loss"], ++ history["test_top5accuracy"], history["test_time"])) ++ finite_accuracy = validation_accuracy[np.isfinite(validation_accuracy)] ++ best_accuracy = ( ++ float(np.max(finite_accuracy)) if finite_accuracy.size else None) ++ nonfinite_mask = ( ++ ~np.isfinite(validation_accuracy) ++ | ~np.isfinite(validation_loss) ++ | ~np.isfinite(train_loss)) ++ nonfinite_indices = np.flatnonzero(nonfinite_mask) ++ first_nonfinite = ( ++ int(nonfinite_indices[0]) + 1 if nonfinite_indices.size else None) ++ if first_nonfinite is None and completed != expected_epochs: ++ outcome = "premature_finite_stop" ++ elif first_nonfinite is None: ++ outcome = "finite" ++ else: ++ outcome = "nonfinite" ++ if best_accuracy is not None: ++ assert math.isclose( ++ best_accuracy, scalar(history["best_validation_accuracy"]), ++ rel_tol=0, abs_tol=1e-5) ++ return { ++ "outcome": outcome, ++ "best_validation_accuracy": best_accuracy, ++ "final_validation_accuracy": float(validation_accuracy[-1]), ++ "final_validation_loss": float(validation_loss[-1]), ++ "best_epoch": int(history["best_epoch"]), ++ "epochs_completed": completed, ++ "first_nonfinite_step": first_nonfinite, ++ "train_seconds": float( ++ np.sum(np.asarray(history["train_time"])[:completed])), ++ "validation_seconds": float( ++ np.sum(np.asarray(history["val_time"])[:completed])), ++ "test_metrics_all_nan": True, ++ } ++ ++ ++def record_manifest(experiment_name): ++ paths = glob.glob(os.path.join( ++ ROOT, "runs", experiment_name, "*", "crossover_manifest.json")) ++ assert len(paths) == 1, ( ++ f"{experiment_name}: expected one manifest, found {len(paths)}") ++ return paths[0] ++ ++ ++def audit_record(job, launch): ++ manifest_path = record_manifest(job["experiment_name"]) ++ with open(manifest_path, encoding="utf-8") as handle: ++ manifest = json.load(handle) ++ for key in ( ++ "stage", "method", "architecture", "rate", "expected_epochs", ++ "experiment_name", "selector_path", "selector_sha256", ++ "timeout_seconds", "command"): ++ assert manifest[key] == job[key], ( ++ f"{job['experiment_name']}: registry drift in {key}") ++ assert manifest["source"] == launch["source"] ++ assert manifest["status"] in STATUSES ++ assert manifest["selector_sha256"] == crossover_grid.SELECTOR_SHA256 ++ assert manifest["work"] == crossover_grid.p2_work_report(job) ++ hardware = manifest["hardware"] ++ assert hardware["physical_index"] in (5, 7) ++ assert hardware["name"] == "NVIDIA GeForce GTX 1080" ++ assert hardware["peak_process_gpu_memory_mib"] >= 0 ++ histogram = manifest["histogram"] ++ if histogram is None: ++ assert manifest["histogram_sha256"] is None ++ metrics = { ++ "outcome": manifest["status"], ++ "best_validation_accuracy": None, ++ "final_validation_accuracy": None, ++ "final_validation_loss": None, ++ "best_epoch": None, ++ "epochs_completed": 0, ++ "first_nonfinite_step": None, ++ "train_seconds": None, ++ "validation_seconds": None, ++ "test_metrics_all_nan": None, ++ } ++ else: ++ assert os.path.isfile(histogram) ++ assert manifest["histogram_sha256"] == sha256(histogram) ++ metrics = history_metrics(histogram, job) ++ return { ++ "method": job["method"], ++ "architecture": job["architecture"], ++ "rate": job["rate"], ++ "status": manifest["status"], ++ "manifest": os.path.relpath(manifest_path, ROOT), ++ "histogram_sha256": manifest["histogram_sha256"], ++ "driver_wall_seconds": manifest["driver_wall_seconds"], ++ "physical_gpu": hardware, ++ "work": manifest["work"], ++ **metrics, ++ } ++ ++ ++def main(): ++ parser = argparse.ArgumentParser() ++ parser.add_argument("--output") ++ parser.add_argument("--allow-partial", action="store_true") ++ args = parser.parse_args() ++ launch_path = os.path.join(ROOT, "runs", "plain-p2-launch.json") ++ with open(launch_path, encoding="utf-8") as handle: ++ launch = json.load(handle) ++ jobs = crossover_grid.p2_jobs() ++ assert launch["stage"] == "p2" ++ assert launch["num_registered_cells"] == 27 ++ assert launch["registry_sha256"] == crossover_grid.p2_registry_sha256( ++ jobs) ++ assert launch["jobs"] == jobs ++ assert launch["selector_sha256"] == crossover_grid.SELECTOR_SHA256 ++ assert launch["source"]["cifar10_tfds_tree"]["num_files"] == 5 ++ expected = {job["experiment_name"]: job for job in jobs} ++ actual = { ++ os.path.basename(path) ++ for path in glob.glob(os.path.join(ROOT, "runs", "plain-p2-*")) ++ if os.path.isdir(path) and glob.glob( ++ os.path.join(path, "*", "crossover_manifest.json")) ++ } ++ unexpected = actual - set(expected) ++ missing = set(expected) - actual ++ assert not unexpected, f"unexpected P2 cells: {sorted(unexpected)}" ++ if missing and not args.allow_partial: ++ raise AssertionError(f"missing P2 cells: {sorted(missing)}") ++ records = [ ++ audit_record(job, launch) for job in jobs ++ if job["experiment_name"] in actual] ++ report = { ++ "gate": "pass" if not missing else "partial", ++ "stage": "plain_cnn_p2", ++ "launch_record": os.path.relpath(launch_path, ROOT), ++ "source": launch["source"], ++ "selector_sha256": launch["selector_sha256"], ++ "registry_sha256": launch["registry_sha256"], ++ "num_expected_records": len(jobs), ++ "num_audited_records": len(records), ++ "missing_experiments": sorted(missing), ++ "test_policy": "none", ++ "records": records, ++ } ++ encoded = json.dumps(report, indent=2, sort_keys=True) + "\n" ++ if args.output: ++ with open(args.output, "w", encoding="utf-8") as handle: ++ handle.write(encoded) ++ else: ++ print(encoded, end="") ++ ++ ++if __name__ == "__main__": ++ main() +-- +2.54.0 + |