From 46b74984587f7b76c40f91cfccd3a6939695a121 Mon Sep 17 00:00:00 2001 From: YurenHao0426 Date: Mon, 27 Jul 2026 13:45:04 -0500 Subject: [PATCH 13/19] analysis: audit plain CNN P1 selector --- analyze_p1.py | 241 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 241 insertions(+) create mode 100644 analyze_p1.py diff --git a/analyze_p1.py b/analyze_p1.py new file mode 100644 index 0000000..05dd52d --- /dev/null +++ b/analyze_p1.py @@ -0,0 +1,241 @@ +#!/usr/bin/env python3 +"""Audit and mechanically select the frozen plain-CNN P1 learning rates.""" +import argparse +import glob +import hashlib +import json +import math +import os + +import numpy as np + + +ROOT = os.path.dirname(os.path.abspath(__file__)) +EXPECTED_SOURCE = { + "author_crossover_commit": "90697e8a17ab5e0757a62df40da521766763abe1", + "main_commit": "dab65f657ed22ce90fe2b38a360ca62cc8b00e65", + "protocol_sha256": + "345e0c1245f558a68ea2c212dfa427bdaa30eafd2c8d16c6763691ba354dae61", +} +SPECIFICATIONS = ( + ("bp", 0.025), + ("fa", 0.003), + ("fa", 0.01), + ("fa", 0.03), + ("dfa", 0.003), + ("dfa", 0.01), + ("dfa", 0.03), + ("pepita", 0.01), + ("ff", 0.03), + ("ep", 0.003), + ("ep", 0.01), + ("ep", 0.03), + ("dualprop", 0.025), + ("clean_kp", 0.003), + ("clean_kp", 0.01), + ("clean_kp", 0.03), + ("sdil", 0.003), + ("sdil", 0.01), + ("sdil", 0.03), +) +GRIDDED_METHODS = ("fa", "dfa", "ep", "clean_kp", "sdil") +CLI_METHODS = { + "bp": "backprop", + "clean_kp": "clean-kp", + "dualprop": "dualprop-lagr-ff", +} + + +def rate_tag(rate): + return f"{rate:g}".replace(".", "p") + + +def experiment_name(method, rate): + return f"plain-p1-{method}-minicnn-lr{rate_tag(rate)}" + + +def sha256(path): + digest = hashlib.sha256() + with open(path, "rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def flag(command, name): + try: + index = command.index(name) + except ValueError as error: + raise AssertionError(f"missing command flag {name}") from error + if index + 1 >= len(command): + raise AssertionError(f"missing value for command flag {name}") + return command[index + 1] + + +def scalar(value): + return float(np.asarray(value)) + + +def audit_histogram(path, method): + history = np.load(path, allow_pickle=True).item() + if method == "ff": + assert history["method"] == "ff" + assert history["epochs_per_layer"] == 10 + assert len(history["layers"]) == history["num_layers"] + assert all( + len(layer["epochs"]) == 10 for layer in history["layers"]) + final = history["final"] + best_accuracy = float(final["validation_accuracy"]) + final_accuracy = best_accuracy + final_loss = float(history["layers"][-1]["epochs"][-1]["loss"]) + test_values = (final["test_accuracy"], final["test_time"]) + epochs_completed = sum( + len(layer["epochs"]) for layer in history["layers"]) + best_epoch = None + train_seconds = sum( + epoch["runtime"] for layer in history["layers"] + for epoch in layer["epochs"]) + validation_seconds = float(final["validation_time"]) + else: + validation_accuracy = np.asarray( + history["val_accuracy"], dtype=float) + validation_loss = np.asarray(history["val_loss"], dtype=float) + assert validation_accuracy.shape == (10,) + assert validation_loss.shape == (10,) + assert int(history["epochs_completed"]) == 10 + assert np.all(np.isfinite(validation_accuracy)) + best_accuracy = float(np.max(validation_accuracy)) + final_accuracy = float(validation_accuracy[-1]) + final_loss = float(validation_loss[-1]) + assert math.isclose( + best_accuracy, scalar(history["best_validation_accuracy"]), + rel_tol=0, abs_tol=1e-6) + test_values = ( + history["test_accuracy"], history["test_loss"], + history["test_top5accuracy"], history["test_time"]) + epochs_completed = 10 + best_epoch = int(history["best_epoch"]) + train_seconds = float(np.sum(history["train_time"])) + validation_seconds = float(np.sum(history["val_time"])) + assert all(math.isnan(scalar(value)) for value in test_values) + return { + "best_validation_accuracy": best_accuracy, + "final_validation_accuracy": final_accuracy, + "final_validation_loss": final_loss, + "epochs_completed": epochs_completed, + "best_epoch": best_epoch, + "train_seconds": train_seconds, + "validation_seconds": validation_seconds, + "test_metrics_all_nan": True, + } + + +def audit_record(method, rate): + name = experiment_name(method, rate) + run_root = os.path.join(ROOT, "runs", name) + histograms = glob.glob(os.path.join(run_root, "*", "hist.npy")) + manifests = glob.glob( + os.path.join(run_root, "*", "crossover_manifest.json")) + assert len(histograms) == 1, f"{name}: found {len(histograms)} histograms" + assert len(manifests) == 1, f"{name}: found {len(manifests)} manifests" + histogram = os.path.realpath(histograms[0]) + with open(manifests[0], encoding="utf-8") as handle: + manifest = json.load(handle) + assert manifest["stage"] == "p1" + assert manifest["method"] == method + assert manifest["architecture"] == "minicnn" + assert math.isclose(float(manifest["rate"]), rate) + assert manifest["experiment_name"] == name + assert os.path.realpath(manifest["histogram"]) == histogram + assert manifest["histogram_sha256"] == sha256(histogram) + for key, expected in EXPECTED_SOURCE.items(): + assert manifest["source"][key] == expected, f"{name}: source drift" + command = manifest["command"] + assert flag(command, "--model") == "miniCNN" + assert flag(command, "--dataset") == "cifar10" + assert flag(command, "--num-epochs") == "10" + assert flag(command, "--seeds") == "0" + assert flag(command, "--feedback-seed") == "1729" + assert flag(command, "--test-policy") == "none" + assert flag(command, "--early-stop-policy") == "none" + assert flag(command, "--gradient-diagnostics") == "none" + assert flag(command, "--spectral-diagnostics") == "none" + assert flag(command, "--learning-algorithm") == CLI_METHODS.get( + method, method) + assert math.isclose(float(flag(command, "--learning-rate")), rate) + metrics = audit_histogram(histogram, method) + return { + "method": method, + "rate": rate, + "experiment_name": name, + "manifest": os.path.relpath(manifests[0], ROOT), + "histogram_sha256": manifest["histogram_sha256"], + "driver_wall_seconds": manifest.get("driver_wall_seconds"), + **metrics, + } + + +def selected_candidate(candidates): + def rank(candidate): + loss = candidate["final_validation_loss"] + return ( + candidate["best_validation_accuracy"], + int(math.isfinite(loss)), + -candidate["rate"], + ) + return max(candidates, key=rank) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--output") + args = parser.parse_args() + expected_names = { + experiment_name(method, rate) for method, rate in SPECIFICATIONS} + actual_names = { + os.path.basename(path) + for path in glob.glob(os.path.join(ROOT, "runs", "plain-p1-*")) + if glob.glob(os.path.join(path, "*", "hist.npy")) + } + assert actual_names == expected_names, { + "missing": sorted(expected_names - actual_names), + "unexpected": sorted(actual_names - expected_names), + } + records = [ + audit_record(method, rate) for method, rate in SPECIFICATIONS] + selected = {} + for method in dict(SPECIFICATIONS): + candidates = [ + record for record in records if record["method"] == method] + chosen = ( + selected_candidate(candidates) + if method in GRIDDED_METHODS else candidates[0]) + selected[method] = { + "rate": chosen["rate"], + "best_validation_accuracy": + chosen["best_validation_accuracy"], + "experiment_name": chosen["experiment_name"], + } + report = { + "gate": "pass", + "stage": "plain_cnn_p1", + "selection_rule": + "maximum best validation accuracy, then finite final validation " + "loss, then lower learning rate", + "expected_source": EXPECTED_SOURCE, + "num_expected_records": len(SPECIFICATIONS), + "num_audited_records": len(records), + "test_policy": "none", + "records": records, + "selected": selected, + } + encoded = json.dumps(report, indent=2, sort_keys=True) + "\n" + if args.output: + with open(args.output, "w", encoding="utf-8") as handle: + handle.write(encoded) + else: + print(encoded, end="") + + +if __name__ == "__main__": + main() -- 2.54.0