diff options
Diffstat (limited to 'external/dualprop_patches/0013-analysis-audit-plain-CNN-P1-selector.patch')
| -rw-r--r-- | external/dualprop_patches/0013-analysis-audit-plain-CNN-P1-selector.patch | 260 |
1 files changed, 260 insertions, 0 deletions
diff --git a/external/dualprop_patches/0013-analysis-audit-plain-CNN-P1-selector.patch b/external/dualprop_patches/0013-analysis-audit-plain-CNN-P1-selector.patch new file mode 100644 index 0000000..ef2612d --- /dev/null +++ b/external/dualprop_patches/0013-analysis-audit-plain-CNN-P1-selector.patch @@ -0,0 +1,260 @@ +From 46b74984587f7b76c40f91cfccd3a6939695a121 Mon Sep 17 00:00:00 2001 +From: YurenHao0426 <Blackhao0426@gmail.com> +Date: Mon, 27 Jul 2026 13:45:04 -0500 +Subject: [PATCH 13/19] analysis: audit plain CNN P1 selector + +--- + analyze_p1.py | 241 ++++++++++++++++++++++++++++++++++++++++++++++++++ + 1 file changed, 241 insertions(+) + create mode 100644 analyze_p1.py + +diff --git a/analyze_p1.py b/analyze_p1.py +new file mode 100644 +index 0000000..05dd52d +--- /dev/null ++++ b/analyze_p1.py +@@ -0,0 +1,241 @@ ++#!/usr/bin/env python3 ++"""Audit and mechanically select the frozen plain-CNN P1 learning rates.""" ++import argparse ++import glob ++import hashlib ++import json ++import math ++import os ++ ++import numpy as np ++ ++ ++ROOT = os.path.dirname(os.path.abspath(__file__)) ++EXPECTED_SOURCE = { ++ "author_crossover_commit": "90697e8a17ab5e0757a62df40da521766763abe1", ++ "main_commit": "dab65f657ed22ce90fe2b38a360ca62cc8b00e65", ++ "protocol_sha256": ++ "345e0c1245f558a68ea2c212dfa427bdaa30eafd2c8d16c6763691ba354dae61", ++} ++SPECIFICATIONS = ( ++ ("bp", 0.025), ++ ("fa", 0.003), ++ ("fa", 0.01), ++ ("fa", 0.03), ++ ("dfa", 0.003), ++ ("dfa", 0.01), ++ ("dfa", 0.03), ++ ("pepita", 0.01), ++ ("ff", 0.03), ++ ("ep", 0.003), ++ ("ep", 0.01), ++ ("ep", 0.03), ++ ("dualprop", 0.025), ++ ("clean_kp", 0.003), ++ ("clean_kp", 0.01), ++ ("clean_kp", 0.03), ++ ("sdil", 0.003), ++ ("sdil", 0.01), ++ ("sdil", 0.03), ++) ++GRIDDED_METHODS = ("fa", "dfa", "ep", "clean_kp", "sdil") ++CLI_METHODS = { ++ "bp": "backprop", ++ "clean_kp": "clean-kp", ++ "dualprop": "dualprop-lagr-ff", ++} ++ ++ ++def rate_tag(rate): ++ return f"{rate:g}".replace(".", "p") ++ ++ ++def experiment_name(method, rate): ++ return f"plain-p1-{method}-minicnn-lr{rate_tag(rate)}" ++ ++ ++def sha256(path): ++ digest = hashlib.sha256() ++ with open(path, "rb") as handle: ++ for chunk in iter(lambda: handle.read(1024 * 1024), b""): ++ digest.update(chunk) ++ return digest.hexdigest() ++ ++ ++def flag(command, name): ++ try: ++ index = command.index(name) ++ except ValueError as error: ++ raise AssertionError(f"missing command flag {name}") from error ++ if index + 1 >= len(command): ++ raise AssertionError(f"missing value for command flag {name}") ++ return command[index + 1] ++ ++ ++def scalar(value): ++ return float(np.asarray(value)) ++ ++ ++def audit_histogram(path, method): ++ history = np.load(path, allow_pickle=True).item() ++ if method == "ff": ++ assert history["method"] == "ff" ++ assert history["epochs_per_layer"] == 10 ++ assert len(history["layers"]) == history["num_layers"] ++ assert all( ++ len(layer["epochs"]) == 10 for layer in history["layers"]) ++ final = history["final"] ++ best_accuracy = float(final["validation_accuracy"]) ++ final_accuracy = best_accuracy ++ final_loss = float(history["layers"][-1]["epochs"][-1]["loss"]) ++ test_values = (final["test_accuracy"], final["test_time"]) ++ epochs_completed = sum( ++ len(layer["epochs"]) for layer in history["layers"]) ++ best_epoch = None ++ train_seconds = sum( ++ epoch["runtime"] for layer in history["layers"] ++ for epoch in layer["epochs"]) ++ validation_seconds = float(final["validation_time"]) ++ else: ++ validation_accuracy = np.asarray( ++ history["val_accuracy"], dtype=float) ++ validation_loss = np.asarray(history["val_loss"], dtype=float) ++ assert validation_accuracy.shape == (10,) ++ assert validation_loss.shape == (10,) ++ assert int(history["epochs_completed"]) == 10 ++ assert np.all(np.isfinite(validation_accuracy)) ++ best_accuracy = float(np.max(validation_accuracy)) ++ final_accuracy = float(validation_accuracy[-1]) ++ final_loss = float(validation_loss[-1]) ++ assert math.isclose( ++ best_accuracy, scalar(history["best_validation_accuracy"]), ++ rel_tol=0, abs_tol=1e-6) ++ test_values = ( ++ history["test_accuracy"], history["test_loss"], ++ history["test_top5accuracy"], history["test_time"]) ++ epochs_completed = 10 ++ best_epoch = int(history["best_epoch"]) ++ train_seconds = float(np.sum(history["train_time"])) ++ validation_seconds = float(np.sum(history["val_time"])) ++ assert all(math.isnan(scalar(value)) for value in test_values) ++ return { ++ "best_validation_accuracy": best_accuracy, ++ "final_validation_accuracy": final_accuracy, ++ "final_validation_loss": final_loss, ++ "epochs_completed": epochs_completed, ++ "best_epoch": best_epoch, ++ "train_seconds": train_seconds, ++ "validation_seconds": validation_seconds, ++ "test_metrics_all_nan": True, ++ } ++ ++ ++def audit_record(method, rate): ++ name = experiment_name(method, rate) ++ run_root = os.path.join(ROOT, "runs", name) ++ histograms = glob.glob(os.path.join(run_root, "*", "hist.npy")) ++ manifests = glob.glob( ++ os.path.join(run_root, "*", "crossover_manifest.json")) ++ assert len(histograms) == 1, f"{name}: found {len(histograms)} histograms" ++ assert len(manifests) == 1, f"{name}: found {len(manifests)} manifests" ++ histogram = os.path.realpath(histograms[0]) ++ with open(manifests[0], encoding="utf-8") as handle: ++ manifest = json.load(handle) ++ assert manifest["stage"] == "p1" ++ assert manifest["method"] == method ++ assert manifest["architecture"] == "minicnn" ++ assert math.isclose(float(manifest["rate"]), rate) ++ assert manifest["experiment_name"] == name ++ assert os.path.realpath(manifest["histogram"]) == histogram ++ assert manifest["histogram_sha256"] == sha256(histogram) ++ for key, expected in EXPECTED_SOURCE.items(): ++ assert manifest["source"][key] == expected, f"{name}: source drift" ++ command = manifest["command"] ++ assert flag(command, "--model") == "miniCNN" ++ assert flag(command, "--dataset") == "cifar10" ++ assert flag(command, "--num-epochs") == "10" ++ assert flag(command, "--seeds") == "0" ++ assert flag(command, "--feedback-seed") == "1729" ++ assert flag(command, "--test-policy") == "none" ++ assert flag(command, "--early-stop-policy") == "none" ++ assert flag(command, "--gradient-diagnostics") == "none" ++ assert flag(command, "--spectral-diagnostics") == "none" ++ assert flag(command, "--learning-algorithm") == CLI_METHODS.get( ++ method, method) ++ assert math.isclose(float(flag(command, "--learning-rate")), rate) ++ metrics = audit_histogram(histogram, method) ++ return { ++ "method": method, ++ "rate": rate, ++ "experiment_name": name, ++ "manifest": os.path.relpath(manifests[0], ROOT), ++ "histogram_sha256": manifest["histogram_sha256"], ++ "driver_wall_seconds": manifest.get("driver_wall_seconds"), ++ **metrics, ++ } ++ ++ ++def selected_candidate(candidates): ++ def rank(candidate): ++ loss = candidate["final_validation_loss"] ++ return ( ++ candidate["best_validation_accuracy"], ++ int(math.isfinite(loss)), ++ -candidate["rate"], ++ ) ++ return max(candidates, key=rank) ++ ++ ++def main(): ++ parser = argparse.ArgumentParser() ++ parser.add_argument("--output") ++ args = parser.parse_args() ++ expected_names = { ++ experiment_name(method, rate) for method, rate in SPECIFICATIONS} ++ actual_names = { ++ os.path.basename(path) ++ for path in glob.glob(os.path.join(ROOT, "runs", "plain-p1-*")) ++ if glob.glob(os.path.join(path, "*", "hist.npy")) ++ } ++ assert actual_names == expected_names, { ++ "missing": sorted(expected_names - actual_names), ++ "unexpected": sorted(actual_names - expected_names), ++ } ++ records = [ ++ audit_record(method, rate) for method, rate in SPECIFICATIONS] ++ selected = {} ++ for method in dict(SPECIFICATIONS): ++ candidates = [ ++ record for record in records if record["method"] == method] ++ chosen = ( ++ selected_candidate(candidates) ++ if method in GRIDDED_METHODS else candidates[0]) ++ selected[method] = { ++ "rate": chosen["rate"], ++ "best_validation_accuracy": ++ chosen["best_validation_accuracy"], ++ "experiment_name": chosen["experiment_name"], ++ } ++ report = { ++ "gate": "pass", ++ "stage": "plain_cnn_p1", ++ "selection_rule": ++ "maximum best validation accuracy, then finite final validation " ++ "loss, then lower learning rate", ++ "expected_source": EXPECTED_SOURCE, ++ "num_expected_records": len(SPECIFICATIONS), ++ "num_audited_records": len(records), ++ "test_policy": "none", ++ "records": records, ++ "selected": selected, ++ } ++ encoded = json.dumps(report, indent=2, sort_keys=True) + "\n" ++ if args.output: ++ with open(args.output, "w", encoding="utf-8") as handle: ++ handle.write(encoded) ++ else: ++ print(encoded, end="") ++ ++ ++if __name__ == "__main__": ++ main() +-- +2.54.0 + |