diff options
| -rwxr-xr-x | experiments/analyze_verified.py | 153 |
1 files changed, 153 insertions, 0 deletions
diff --git a/experiments/analyze_verified.py b/experiments/analyze_verified.py new file mode 100755 index 0000000..8a1a841 --- /dev/null +++ b/experiments/analyze_verified.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +"""Aggregate the audited multi-seed experiments into compact Markdown tables.""" +import argparse +import glob +import json +import math +import os +import statistics + + +def read_many(patterns): + paths = sorted({path for pattern in patterns for path in glob.glob(pattern)}) + records = [] + for path in paths: + with open(path) as f: + record = json.load(f) + record["_path"] = path + records.append(record) + return records + + +def mean_sd(values): + if not values: + return float("nan"), float("nan") + return statistics.mean(values), statistics.stdev(values) if len(values) > 1 else 0.0 + + +def fmt(values, percent=False, digits=3): + mean, sd = mean_sd(values) + if percent: + mean, sd = 100 * mean, 100 * sd + return f"{mean:.{digits}f} ± {sd:.{digits}f}" + + +def early_layer_alignment(record): + values = (record.get("final", {}).get("cos_r_negg") + or record.get("final", {}).get("cos_fa_negg")) + if not values: + return None + n = max(1, len(values) // 3) + finite = [v for v in values[:n] if math.isfinite(v)] + return statistics.mean(finite) if finite else None + + +def method_name(record): + args = record["args"] + return (args.get("mode") or args.get("method")).upper() + + +def print_scaling(root): + records = read_many([ + os.path.join(root, "scale_v3_cifar10_*.json"), + os.path.join(root, "fa_scale_v2_cifar10_*.json"), + ]) + groups = {} + for record in records: + key = (record["args"]["depth"], method_name(record)) + groups.setdefault(key, []).append(record) + print("## CIFAR-10 depth scaling\n") + print("| depth | method | n | test acc (%) | early-third alignment | wall (s) |") + print("|---:|:---|---:|---:|---:|---:|") + order = {"BP": 0, "FA": 1, "DFA": 2, "SDIL": 3} + for (depth, method), rows in sorted(groups.items(), key=lambda x: (x[0][0], order[x[0][1]])): + aligns = [v for row in rows if (v := early_layer_alignment(row)) is not None] + align = fmt(aligns) if aligns else "—" + print(f"| {depth} | {method} | {len(rows)} | " + f"{fmt([r['final']['test_acc'] for r in rows], percent=True)} | {align} | " + f"{fmt([r['final']['wall_s'] for r in rows], digits=1)} |") + + indexed = {(r["args"]["depth"], r["args"]["seed"], method_name(r)): r for r in records} + for depth in sorted({r["args"]["depth"] for r in records}): + gaps = [] + for seed in range(100): + sdil = indexed.get((depth, seed, "SDIL")) + dfa = indexed.get((depth, seed, "DFA")) + if sdil and dfa: + gaps.append(sdil["final"]["test_acc"] - dfa["final"]["test_acc"]) + if gaps: + print(f"\nPaired SDIL−DFA gap at d{depth}: {fmt(gaps, percent=True)} points (n={len(gaps)}).") + + +def nuisance_label(record): + args = record["args"] + if args["use_residual"]: + return "residual" + if args["raw_scale_control"] == "match_innovation_norm": + return "matched raw" + return "raw" + + +def print_nuisance(root): + records = read_many([os.path.join(root, "nuis_ctrl_v1_*.json")]) + groups = {} + for record in records: + key = (record["args"]["nuis_rho"], nuisance_label(record)) + groups.setdefault(key, []).append(record) + print("\n\n## Somato-dendritic nuisance control\n") + print("| rho | signal | n | test acc (%) |") + print("|---:|:---|---:|---:|") + order = {"raw": 0, "matched raw": 1, "residual": 2} + for (rho, label), rows in sorted(groups.items(), key=lambda x: (x[0][0], order[x[0][1]])): + print(f"| {rho:g} | {label} | {len(rows)} | " + f"{fmt([r['final']['test_acc'] for r in rows], percent=True)} |") + + +def print_local_baselines(root): + records = read_many([ + os.path.join(root, "pepita_tuned_v1_*.json"), + os.path.join(root, "baseline_budget_v1_mnist_ff_*.json"), + os.path.join(root, "ep_original_v2_*.json"), + ]) + groups = {} + for record in records: + args = record["args"] + key = (method_name(record), args["dataset"], args["depth"], args["width"], args["epochs"]) + groups.setdefault(key, []).append(record) + print("\n\n## Native-protocol local baselines\n") + print("| method | dataset | depth × width | budget | n | test acc (%) | wall (s) |") + print("|:---|:---|:---|:---|---:|---:|---:|") + for (method, dataset, depth, width, epochs), rows in sorted(groups.items()): + budget = f"{epochs}/layer" if method == "FF" else str(epochs) + print(f"| {method} | {dataset} | {depth} × {width} | {budget} | {len(rows)} | " + f"{fmt([r['final']['test_acc'] for r in rows], percent=True)} | " + f"{fmt([r['final']['wall_s'] for r in rows], digits=1)} |") + + +def provenance_audit(root): + records = read_many([ + os.path.join(root, "scale_v3_*.json"), + os.path.join(root, "nuis_ctrl_v1_*.json"), + os.path.join(root, "fa_scale_v2_*.json"), + os.path.join(root, "pepita_tuned_v1_*.json"), + os.path.join(root, "baseline_budget_v1_mnist_ff_*.json"), + os.path.join(root, "ep_original_v2_*.json"), + ]) + dirty = [r["_path"] for r in records if r.get("provenance", {}).get("git_dirty") is not False] + print(f"\n\nProvenance: {len(records)} files audited; {len(dirty)} dirty or unknown.") + for path in dirty: + print(f"- {path}") + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--results", default="results") + args = parser.parse_args() + print_scaling(args.results) + print_nuisance(args.results) + print_local_baselines(args.results) + provenance_audit(args.results) + + +if __name__ == "__main__": + main() |