diff options
Diffstat (limited to 'experiments')
| -rw-r--r-- | experiments/analyze_oral_a_hfa_full.py | 75 | ||||
| -rw-r--r-- | experiments/analyze_oral_a_hfa_short.py | 107 | ||||
| -rw-r--r-- | experiments/oral_a_hfa_full_development.py | 50 | ||||
| -rw-r--r-- | experiments/oral_a_hfa_short_screen.py | 51 |
4 files changed, 283 insertions, 0 deletions
diff --git a/experiments/analyze_oral_a_hfa_full.py b/experiments/analyze_oral_a_hfa_full.py new file mode 100644 index 0000000..867d64a --- /dev/null +++ b/experiments/analyze_oral_a_hfa_full.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +"""Audit the frozen HFA-S2 full ResNet-20 validation baseline.""" +import argparse +import json +import math +import os + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--selection", default="results/oral_a_hfa_short_selection.json") + parser.add_argument("--input", default="results/oral_a_hfa_full/hfa.json") + parser.add_argument( + "--out", default="results/oral_a_hfa_full_audit.json") + args = parser.parse_args() + with open(args.selection) as handle: + selection = json.load(handle) + if selection.get("status") != "selected_full_open": + raise ValueError("HFA-S1 did not open HFA-S2") + with open(args.input) as handle: + record = json.load(handle) + run_args = record["args"] + expected = { + "mode": "hfa", "depth": 20, "width": 16, "seed": 0, + "loader_seed": 0, "epochs": 200, "train_limit": 0, + "val_examples": 5000, "split_seed": 2027, + "eval_split": "validation", "eval_every": 0, + "augment_train": 1, "lr_schedule": "step", + "lr_milestones": "100,150", "lr_gamma": 0.1, + "warmup_epochs": 0, "momentum": 0.9, "weight_decay": 1e-4, + "normalization": "batchnorm", "output_lr": 0.1, + "a_scale": 1.0, "alignment_probe": 32, + } + for key, value in expected.items(): + if run_args.get(key) != value: + raise ValueError(f"HFA-S2 {key} drift") + if float(run_args["lr"]) != float(selection["selected"]["lr"]): + raise ValueError("HFA-S2 did not copy the selected hidden rate") + if record["provenance"]["git_tracked_dirty"]: + raise ValueError("tracked-dirty HFA-S2 result") + protocol = record["evaluation_protocol"] + if protocol["test_evaluations"] or protocol["test_used_for_selection"]: + raise ValueError("HFA-S2 touched test") + accuracy = float(record["final"]["accuracy"]) + loss = float(record["final"]["loss"]) + early = float(record["diagnostics"]["early_third_mean"]) + finite = (bool(record["final"]["finite"]) + and math.isfinite(accuracy + loss + early)) + output = { + "protocol": "convolutional_hfa_S2_v1", + "status": ("strong_baseline" if finite and accuracy >= 0.80 + else "weak_or_nonfinite_baseline"), + "accuracy": accuracy, "loss": loss, "finite": finite, + "early_third_alignment": early, + "selected_hidden_lr": float(run_args["lr"]), + "total_macs": int(record["work"]["total_macs_estimate"]), + "logical_batch_loss_queries": int( + record["work"]["logical_batch_loss_queries"]), + "peak_memory_allocated_bytes": int( + record["hardware"]["peak_memory_allocated_bytes"]), + "wall_s": float(record["timing"]["total_timed_wall_s"]), + "source_commit": record["provenance"]["git_commit"], + "confirmation_test_seeds_touched": False, + } + os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) + with open(args.out, "w") as handle: + json.dump(output, handle, indent=2, sort_keys=True) + handle.write("\n") + print(json.dumps(output, indent=2)) + + +if __name__ == "__main__": + main() + diff --git a/experiments/analyze_oral_a_hfa_short.py b/experiments/analyze_oral_a_hfa_short.py new file mode 100644 index 0000000..cc2ee3c --- /dev/null +++ b/experiments/analyze_oral_a_hfa_short.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +"""Audit and select the frozen convolutional-HFA short screen.""" +import argparse +import glob +import json +import math +import os + + +RATES = (0.01, 0.03, 0.1) + + +def read(path): + with open(path) as handle: + record = json.load(handle) + args = record["args"] + expected = { + "mode": "hfa", "depth": 20, "width": 16, "seed": 0, + "loader_seed": 0, "epochs": 20, "train_limit": 10000, + "val_examples": 5000, "split_seed": 2027, + "eval_split": "validation", "eval_every": 0, + "augment_train": 1, "lr_schedule": "cosine", "warmup_epochs": 0, + "momentum": 0.9, "weight_decay": 1e-4, + "normalization": "batchnorm", "output_lr": 0.1, + "a_scale": 1.0, "alignment_probe": 32, + } + for key, value in expected.items(): + if args.get(key) != value: + raise ValueError(f"{path}: {key} drift") + if float(args["lr"]) not in RATES: + raise ValueError(f"{path}: unregistered learning rate") + if record["provenance"]["git_tracked_dirty"]: + raise ValueError(f"tracked-dirty result: {path}") + protocol = record["evaluation_protocol"] + if protocol["test_evaluations"] or protocol["test_used_for_selection"]: + raise ValueError(f"short screen touched test: {path}") + accuracy = float(record["final"]["accuracy"]) + loss = float(record["final"]["loss"]) + diagnostics = record.get("diagnostics") or {} + early = float(diagnostics.get("early_third_mean", float("nan"))) + finite = (bool(record["final"]["finite"]) + and math.isfinite(accuracy + loss + early)) + return { + "path": path, "lr": float(args["lr"]), "accuracy": accuracy, + "loss": loss, "early_third_alignment": early, "finite": finite, + "total_macs": int(record["work"]["total_macs_estimate"]), + "fixed_feedback_parameters": int( + record["architecture"]["fixed_feedback_parameters"]), + "logical_batch_loss_queries": int( + record["work"]["logical_batch_loss_queries"]), + "peak_memory_allocated_bytes": int( + record["hardware"]["peak_memory_allocated_bytes"]), + "wall_s": float(record["timing"]["total_timed_wall_s"]), + "source_commit": record["provenance"]["git_commit"], + } + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--input", default="results/oral_a_hfa_short") + parser.add_argument( + "--out", default="results/oral_a_hfa_short_selection.json") + args = parser.parse_args() + rows = [read(path) for path in sorted( + glob.glob(os.path.join(args.input, "*.json")))] + if len(rows) != len(RATES): + raise ValueError(f"expected {len(RATES)} HFA-S1 runs, found {len(rows)}") + if {row["lr"] for row in rows} != set(RATES): + raise ValueError("HFA-S1 learning-rate grid is incomplete") + if len({row["source_commit"] for row in rows}) != 1: + raise ValueError("HFA-S1 source commits differ") + finite = [row for row in rows if row["finite"]] + if not finite: + selected = None + status = "failed_no_finite_candidate" + else: + selected = sorted( + finite, key=lambda row: ( + -row["accuracy"], row["total_macs"], row["lr"]))[0] + status = ("selected_full_open" if selected["accuracy"] >= 0.50 + else "selected_full_closed") + output = { + "protocol": "convolutional_hfa_S1_v1", "status": status, + "rows": rows, "selected": selected, + "comparators": { + "matched_short_dfa_accuracy": 0.3716, + "matched_short_failed_v1_sdil_accuracy": 0.4198, + "matched_short_bp_accuracy": 0.7494, + }, + "full_validation_threshold": 0.50, + "confirmation_test_seeds_touched": False, + } + os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) + with open(args.out, "w") as handle: + json.dump(output, handle, indent=2, sort_keys=True) + handle.write("\n") + print(json.dumps({ + "status": status, "selected": selected, + "rows": [{key: row[key] for key in ( + "lr", "accuracy", "early_third_alignment", "finite")} + for row in rows], + }, indent=2)) + + +if __name__ == "__main__": + main() + diff --git a/experiments/oral_a_hfa_full_development.py b/experiments/oral_a_hfa_full_development.py new file mode 100644 index 0000000..afd4061 --- /dev/null +++ b/experiments/oral_a_hfa_full_development.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""Run the single frozen HFA-S2 full ResNet-20 validation baseline.""" +import argparse +import json +import os +import subprocess +import sys + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--selection", default="results/oral_a_hfa_short_selection.json") + parser.add_argument("--device", default="cuda") + parser.add_argument("--dry_run", action="store_true") + args = parser.parse_args() + with open(args.selection) as handle: + selection = json.load(handle) + if selection.get("protocol") != "convolutional_hfa_S1_v1": + raise ValueError("unexpected HFA-S1 selection protocol") + if selection.get("status") != "selected_full_open": + raise ValueError("HFA-S1 full-validation gate did not pass") + rate = float(selection["selected"]["lr"]) + if rate not in (0.01, 0.03, 0.1): + raise ValueError("selected HFA learning rate left the frozen grid") + + command = [ + sys.executable, "experiments/conv_run.py", "--mode", "hfa", + "--device", args.device, "--depth", "20", "--width", "16", + "--seed", "0", "--loader_seed", "0", "--batch_size", "128", + "--epochs", "200", "--train_limit", "0", + "--val_examples", "5000", "--split_seed", "2027", + "--eval_split", "validation", "--eval_every", "0", + "--augment_train", "1", "--lr_schedule", "step", + "--lr_milestones", "100,150", "--lr_gamma", "0.1", + "--warmup_epochs", "0", "--momentum", "0.9", + "--weight_decay", "1e-4", "--normalization", "batchnorm", + "--lr", str(rate), "--output_lr", "0.1", "--a_scale", "1", + "--alignment_probe", "32", + "--out", "results/oral_a_hfa_full/hfa.json", + ] + os.makedirs("results/oral_a_hfa_full", exist_ok=True) + print(" ".join(command), flush=True) + if not args.dry_run: + subprocess.run(command, check=True) + + +if __name__ == "__main__": + main() + diff --git a/experiments/oral_a_hfa_short_screen.py b/experiments/oral_a_hfa_short_screen.py new file mode 100644 index 0000000..1a5b9ea --- /dev/null +++ b/experiments/oral_a_hfa_short_screen.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +"""Run a deterministic shard of the frozen convolutional-HFA short screen.""" +import argparse +import os +import subprocess +import sys + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--device", default="cuda") + parser.add_argument("--shard_index", type=int, default=0) + parser.add_argument("--num_shards", type=int, default=1) + parser.add_argument("--dry_run", action="store_true") + args = parser.parse_args() + if not 0 <= args.shard_index < args.num_shards: + raise ValueError("invalid shard index") + + common = [ + sys.executable, "experiments/conv_run.py", "--mode", "hfa", + "--device", args.device, "--depth", "20", "--width", "16", + "--seed", "0", "--loader_seed", "0", "--batch_size", "128", + "--epochs", "20", "--train_limit", "10000", + "--val_examples", "5000", "--split_seed", "2027", + "--eval_split", "validation", "--eval_every", "0", + "--augment_train", "1", "--lr_schedule", "cosine", + "--warmup_epochs", "0", "--momentum", "0.9", + "--weight_decay", "1e-4", "--normalization", "batchnorm", + "--output_lr", "0.1", "--a_scale", "1", + "--alignment_probe", "32", + ] + jobs = [] + for rate in (0.01, 0.03, 0.1): + tag = f"hfa_lr{rate}" + jobs.append((tag, common + [ + "--lr", str(rate), + "--out", f"results/oral_a_hfa_short/{tag}.json", + ])) + + os.makedirs("results/oral_a_hfa_short", exist_ok=True) + selected = [job for index, job in enumerate(jobs) + if index % args.num_shards == args.shard_index] + for tag, command in selected: + print(tag, " ".join(command), flush=True) + if not args.dry_run: + subprocess.run(command, check=True) + + +if __name__ == "__main__": + main() + |