From f37644eb452f0eb3d7f368740f8b39493f11b2aa Mon Sep 17 00:00:00 2001 From: YurenHao0426 Date: Wed, 22 Jul 2026 13:12:33 -0500 Subject: protocol: freeze convolutional HFA baseline funnel --- BASELINES.md | 3 +- HFA_BASELINE.md | 64 +++++++++++++++++ README.md | 2 + experiments/analyze_oral_a_hfa_full.py | 75 ++++++++++++++++++++ experiments/analyze_oral_a_hfa_short.py | 107 +++++++++++++++++++++++++++++ experiments/oral_a_hfa_full_development.py | 50 ++++++++++++++ experiments/oral_a_hfa_short_screen.py | 51 ++++++++++++++ 7 files changed, 351 insertions(+), 1 deletion(-) create mode 100644 HFA_BASELINE.md create mode 100644 experiments/analyze_oral_a_hfa_full.py create mode 100644 experiments/analyze_oral_a_hfa_short.py create mode 100644 experiments/oral_a_hfa_full_development.py create mode 100644 experiments/oral_a_hfa_short_screen.py diff --git a/BASELINES.md b/BASELINES.md index de7dfe8..9800d16 100644 --- a/BASELINES.md +++ b/BASELINES.md @@ -19,7 +19,8 @@ The audited implementations and completed results are in `RESULTS.md`: independent random 3x3 feedback tensors, exact parameter-free shortcut adjoints, and local ReLU/BatchNorm Jacobians; it never copies or reads a downstream forward convolution and is the mandatory baseline for any learned - hierarchical extension; + hierarchical extension. Its frozen validation-only screen is specified in + `HFA_BASELINE.md`; - direct node perturbation uses a causal target on every hidden update and no learned vectorizer; - the no-traffic, `P=0` SDIL backbone is learned direct feedback by node diff --git a/HFA_BASELINE.md b/HFA_BASELINE.md new file mode 100644 index 0000000..1dc5f87 --- /dev/null +++ b/HFA_BASELINE.md @@ -0,0 +1,64 @@ +# Convolutional HFA baseline protocol + +## Status and claim boundary + +This protocol was frozen before observing any convolutional hierarchical-FA +accuracy endpoint. It adds a matched local-learning baseline after the +hierarchical oracle localized the ResNet bottleneck to spatial child-error +fields. HFA is not an SDIL novelty claim: recursive random feedback is prior +art, and a strong HFA result cannot rescue the failed frozen Oral-A A3 gate or +open its sealed A4 test panel. + +The implementation follows the actual option-A residual DAG. Every learned +forward convolution has an independent fixed 3x3 feedback tensor; shortcut +adjoints are parameter-free, and ReLU and BatchNorm Jacobians are local. The +feedback code never reads a downstream forward convolution. In an audit-only +symmetric test, setting the feedback tensors to the forward tensors and the +readout feedback to the negative readout transpose reproduces hidden negative +gradients to below `2e-12` relative error and the exact-BP update to below +`2e-7` absolute error. Actual experiments never make this copy. + +## HFA-S1: bounded short screen + +Use the frozen Oral-A development split and the A2b setting: seed-0 ResNet-20, +the first 10,000 post-split training examples, 5,000 validation examples, +20 epochs, batch 128, ordinary CIFAR augmentation, cosine decay without +warmup, momentum 0.9, weight decay `1e-4`, and final validation evaluation +only. Output learning rate is 0.1. Cross hidden learning rate +`{0.01, 0.03, 0.1}` with feedback scale 1 and the single predeclared feedback +seed derived from model seed 0. A 32-example training-prefix alignment probe is +diagnostic only. + +Select maximum final validation accuracy, then lower estimated MACs, then +lower hidden learning rate. The short screen opens one full validation run only +if the selected trajectory is finite and reaches at least 50% validation +accuracy. For interpretation, the already observed matched short endpoints are +DFA `37.16%`, failed-v1 SDIL `41.98%`, and BP `74.94%`; these are comparisons, +not selectable thresholds. + +## HFA-S2: full validation baseline + +If HFA-S1 passes, train the selected HFA setting for 200 epochs on all 45,000 +development-training examples. Copy the A1 schedule exactly: batch 128, +momentum 0.9, weight decay `1e-4`, output base rate 0.1, and hidden base rate +selected by HFA-S1, with 10x drops at epochs 100 and 150 and no warmup. Evaluate +the same 5,000-example validation split only at the end. No recovery grid is +allowed. + +This is a baseline endpoint, not an advancement test for SDIL. A finite result +at or above 80% is considered a strong matched local baseline that any learned +hierarchical SDIL variant must compare against. A weaker result remains +reportable and cannot trigger additional HFA tuning. Neither outcome permits +test-set evaluation. + +## Accounting and stop rules + +- Record fixed feedback parameters, feedback-convolution MACs, local + correlation MACs, peak memory, wall time, and zero causal queries. +- All three HFA-S1 jobs must use the same clean git commit and must not touch + CIFAR-10 test data. +- No feedback scale, seed, optimizer, architecture, or extra learning rate is + added after the S1 endpoints are read. +- HFA-S2, if opened, copies the selected S1 hidden rate mechanically and does + not select on intermediate validation accuracy. + diff --git a/README.md b/README.md index 64abf25..8ffa678 100644 --- a/README.md +++ b/README.md @@ -120,6 +120,8 @@ random feedback tensors follow the real residual DAG and use local activation Jacobians, but remain independent of forward weights. An audit-only symmetric copy reproduces exact hidden gradients to `5.3e-16` relative error and the BP parameter update to `2.98e-8`; actual HFA never performs that copy. +`HFA_BASELINE.md` freezes its matched ResNet-20 validation screen before any +HFA accuracy endpoint is observed. The subsequent V3 mechanism estimates the required A/G matrix statistics directly by perturbing the vectorizer parameter subspace. It remains diff --git a/experiments/analyze_oral_a_hfa_full.py b/experiments/analyze_oral_a_hfa_full.py new file mode 100644 index 0000000..867d64a --- /dev/null +++ b/experiments/analyze_oral_a_hfa_full.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +"""Audit the frozen HFA-S2 full ResNet-20 validation baseline.""" +import argparse +import json +import math +import os + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--selection", default="results/oral_a_hfa_short_selection.json") + parser.add_argument("--input", default="results/oral_a_hfa_full/hfa.json") + parser.add_argument( + "--out", default="results/oral_a_hfa_full_audit.json") + args = parser.parse_args() + with open(args.selection) as handle: + selection = json.load(handle) + if selection.get("status") != "selected_full_open": + raise ValueError("HFA-S1 did not open HFA-S2") + with open(args.input) as handle: + record = json.load(handle) + run_args = record["args"] + expected = { + "mode": "hfa", "depth": 20, "width": 16, "seed": 0, + "loader_seed": 0, "epochs": 200, "train_limit": 0, + "val_examples": 5000, "split_seed": 2027, + "eval_split": "validation", "eval_every": 0, + "augment_train": 1, "lr_schedule": "step", + "lr_milestones": "100,150", "lr_gamma": 0.1, + "warmup_epochs": 0, "momentum": 0.9, "weight_decay": 1e-4, + "normalization": "batchnorm", "output_lr": 0.1, + "a_scale": 1.0, "alignment_probe": 32, + } + for key, value in expected.items(): + if run_args.get(key) != value: + raise ValueError(f"HFA-S2 {key} drift") + if float(run_args["lr"]) != float(selection["selected"]["lr"]): + raise ValueError("HFA-S2 did not copy the selected hidden rate") + if record["provenance"]["git_tracked_dirty"]: + raise ValueError("tracked-dirty HFA-S2 result") + protocol = record["evaluation_protocol"] + if protocol["test_evaluations"] or protocol["test_used_for_selection"]: + raise ValueError("HFA-S2 touched test") + accuracy = float(record["final"]["accuracy"]) + loss = float(record["final"]["loss"]) + early = float(record["diagnostics"]["early_third_mean"]) + finite = (bool(record["final"]["finite"]) + and math.isfinite(accuracy + loss + early)) + output = { + "protocol": "convolutional_hfa_S2_v1", + "status": ("strong_baseline" if finite and accuracy >= 0.80 + else "weak_or_nonfinite_baseline"), + "accuracy": accuracy, "loss": loss, "finite": finite, + "early_third_alignment": early, + "selected_hidden_lr": float(run_args["lr"]), + "total_macs": int(record["work"]["total_macs_estimate"]), + "logical_batch_loss_queries": int( + record["work"]["logical_batch_loss_queries"]), + "peak_memory_allocated_bytes": int( + record["hardware"]["peak_memory_allocated_bytes"]), + "wall_s": float(record["timing"]["total_timed_wall_s"]), + "source_commit": record["provenance"]["git_commit"], + "confirmation_test_seeds_touched": False, + } + os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) + with open(args.out, "w") as handle: + json.dump(output, handle, indent=2, sort_keys=True) + handle.write("\n") + print(json.dumps(output, indent=2)) + + +if __name__ == "__main__": + main() + diff --git a/experiments/analyze_oral_a_hfa_short.py b/experiments/analyze_oral_a_hfa_short.py new file mode 100644 index 0000000..cc2ee3c --- /dev/null +++ b/experiments/analyze_oral_a_hfa_short.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +"""Audit and select the frozen convolutional-HFA short screen.""" +import argparse +import glob +import json +import math +import os + + +RATES = (0.01, 0.03, 0.1) + + +def read(path): + with open(path) as handle: + record = json.load(handle) + args = record["args"] + expected = { + "mode": "hfa", "depth": 20, "width": 16, "seed": 0, + "loader_seed": 0, "epochs": 20, "train_limit": 10000, + "val_examples": 5000, "split_seed": 2027, + "eval_split": "validation", "eval_every": 0, + "augment_train": 1, "lr_schedule": "cosine", "warmup_epochs": 0, + "momentum": 0.9, "weight_decay": 1e-4, + "normalization": "batchnorm", "output_lr": 0.1, + "a_scale": 1.0, "alignment_probe": 32, + } + for key, value in expected.items(): + if args.get(key) != value: + raise ValueError(f"{path}: {key} drift") + if float(args["lr"]) not in RATES: + raise ValueError(f"{path}: unregistered learning rate") + if record["provenance"]["git_tracked_dirty"]: + raise ValueError(f"tracked-dirty result: {path}") + protocol = record["evaluation_protocol"] + if protocol["test_evaluations"] or protocol["test_used_for_selection"]: + raise ValueError(f"short screen touched test: {path}") + accuracy = float(record["final"]["accuracy"]) + loss = float(record["final"]["loss"]) + diagnostics = record.get("diagnostics") or {} + early = float(diagnostics.get("early_third_mean", float("nan"))) + finite = (bool(record["final"]["finite"]) + and math.isfinite(accuracy + loss + early)) + return { + "path": path, "lr": float(args["lr"]), "accuracy": accuracy, + "loss": loss, "early_third_alignment": early, "finite": finite, + "total_macs": int(record["work"]["total_macs_estimate"]), + "fixed_feedback_parameters": int( + record["architecture"]["fixed_feedback_parameters"]), + "logical_batch_loss_queries": int( + record["work"]["logical_batch_loss_queries"]), + "peak_memory_allocated_bytes": int( + record["hardware"]["peak_memory_allocated_bytes"]), + "wall_s": float(record["timing"]["total_timed_wall_s"]), + "source_commit": record["provenance"]["git_commit"], + } + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--input", default="results/oral_a_hfa_short") + parser.add_argument( + "--out", default="results/oral_a_hfa_short_selection.json") + args = parser.parse_args() + rows = [read(path) for path in sorted( + glob.glob(os.path.join(args.input, "*.json")))] + if len(rows) != len(RATES): + raise ValueError(f"expected {len(RATES)} HFA-S1 runs, found {len(rows)}") + if {row["lr"] for row in rows} != set(RATES): + raise ValueError("HFA-S1 learning-rate grid is incomplete") + if len({row["source_commit"] for row in rows}) != 1: + raise ValueError("HFA-S1 source commits differ") + finite = [row for row in rows if row["finite"]] + if not finite: + selected = None + status = "failed_no_finite_candidate" + else: + selected = sorted( + finite, key=lambda row: ( + -row["accuracy"], row["total_macs"], row["lr"]))[0] + status = ("selected_full_open" if selected["accuracy"] >= 0.50 + else "selected_full_closed") + output = { + "protocol": "convolutional_hfa_S1_v1", "status": status, + "rows": rows, "selected": selected, + "comparators": { + "matched_short_dfa_accuracy": 0.3716, + "matched_short_failed_v1_sdil_accuracy": 0.4198, + "matched_short_bp_accuracy": 0.7494, + }, + "full_validation_threshold": 0.50, + "confirmation_test_seeds_touched": False, + } + os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) + with open(args.out, "w") as handle: + json.dump(output, handle, indent=2, sort_keys=True) + handle.write("\n") + print(json.dumps({ + "status": status, "selected": selected, + "rows": [{key: row[key] for key in ( + "lr", "accuracy", "early_third_alignment", "finite")} + for row in rows], + }, indent=2)) + + +if __name__ == "__main__": + main() + diff --git a/experiments/oral_a_hfa_full_development.py b/experiments/oral_a_hfa_full_development.py new file mode 100644 index 0000000..afd4061 --- /dev/null +++ b/experiments/oral_a_hfa_full_development.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""Run the single frozen HFA-S2 full ResNet-20 validation baseline.""" +import argparse +import json +import os +import subprocess +import sys + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--selection", default="results/oral_a_hfa_short_selection.json") + parser.add_argument("--device", default="cuda") + parser.add_argument("--dry_run", action="store_true") + args = parser.parse_args() + with open(args.selection) as handle: + selection = json.load(handle) + if selection.get("protocol") != "convolutional_hfa_S1_v1": + raise ValueError("unexpected HFA-S1 selection protocol") + if selection.get("status") != "selected_full_open": + raise ValueError("HFA-S1 full-validation gate did not pass") + rate = float(selection["selected"]["lr"]) + if rate not in (0.01, 0.03, 0.1): + raise ValueError("selected HFA learning rate left the frozen grid") + + command = [ + sys.executable, "experiments/conv_run.py", "--mode", "hfa", + "--device", args.device, "--depth", "20", "--width", "16", + "--seed", "0", "--loader_seed", "0", "--batch_size", "128", + "--epochs", "200", "--train_limit", "0", + "--val_examples", "5000", "--split_seed", "2027", + "--eval_split", "validation", "--eval_every", "0", + "--augment_train", "1", "--lr_schedule", "step", + "--lr_milestones", "100,150", "--lr_gamma", "0.1", + "--warmup_epochs", "0", "--momentum", "0.9", + "--weight_decay", "1e-4", "--normalization", "batchnorm", + "--lr", str(rate), "--output_lr", "0.1", "--a_scale", "1", + "--alignment_probe", "32", + "--out", "results/oral_a_hfa_full/hfa.json", + ] + os.makedirs("results/oral_a_hfa_full", exist_ok=True) + print(" ".join(command), flush=True) + if not args.dry_run: + subprocess.run(command, check=True) + + +if __name__ == "__main__": + main() + diff --git a/experiments/oral_a_hfa_short_screen.py b/experiments/oral_a_hfa_short_screen.py new file mode 100644 index 0000000..1a5b9ea --- /dev/null +++ b/experiments/oral_a_hfa_short_screen.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +"""Run a deterministic shard of the frozen convolutional-HFA short screen.""" +import argparse +import os +import subprocess +import sys + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--device", default="cuda") + parser.add_argument("--shard_index", type=int, default=0) + parser.add_argument("--num_shards", type=int, default=1) + parser.add_argument("--dry_run", action="store_true") + args = parser.parse_args() + if not 0 <= args.shard_index < args.num_shards: + raise ValueError("invalid shard index") + + common = [ + sys.executable, "experiments/conv_run.py", "--mode", "hfa", + "--device", args.device, "--depth", "20", "--width", "16", + "--seed", "0", "--loader_seed", "0", "--batch_size", "128", + "--epochs", "20", "--train_limit", "10000", + "--val_examples", "5000", "--split_seed", "2027", + "--eval_split", "validation", "--eval_every", "0", + "--augment_train", "1", "--lr_schedule", "cosine", + "--warmup_epochs", "0", "--momentum", "0.9", + "--weight_decay", "1e-4", "--normalization", "batchnorm", + "--output_lr", "0.1", "--a_scale", "1", + "--alignment_probe", "32", + ] + jobs = [] + for rate in (0.01, 0.03, 0.1): + tag = f"hfa_lr{rate}" + jobs.append((tag, common + [ + "--lr", str(rate), + "--out", f"results/oral_a_hfa_short/{tag}.json", + ])) + + os.makedirs("results/oral_a_hfa_short", exist_ok=True) + selected = [job for index, job in enumerate(jobs) + if index % args.num_shards == args.shard_index] + for tag, command in selected: + print(tag, " ".join(command), flush=True) + if not args.dry_run: + subprocess.run(command, check=True) + + +if __name__ == "__main__": + main() + -- cgit v1.2.3