From b329856d15b91a6f31fb75f010f0a550cddd0ee8 Mon Sep 17 00:00:00 2001 From: YurenHao0426 Date: Wed, 22 Jul 2026 05:31:43 -0500 Subject: baselines: audit native author runs --- BASELINES.md | 5 +- experiments/baseline_smoke.py | 33 +++ experiments/import_native_baseline.py | 349 +++++++++++++++++++++++++++++ experiments/native_baseline_protocols.json | 114 ++++++++++ 4 files changed, 500 insertions(+), 1 deletion(-) create mode 100644 experiments/import_native_baseline.py create mode 100644 experiments/native_baseline_protocols.json diff --git a/BASELINES.md b/BASELINES.md index 34e9f58..cfdb3f5 100644 --- a/BASELINES.md +++ b/BASELINES.md @@ -57,7 +57,10 @@ model.Q.mode=sym_Y The inherited configuration supplies three convolutional layers plus a 1500-unit fully connected layer, batch size 32, 400 epochs, label smoothing 0.05, SGD momentum 0.9, weight decay `1e-4`, two warmup epochs and cosine -decay. Y and Q use learning rate `0.001`. The paper reports +decay. The resolved author config uses softmax outputs with MSE loss; Y and Q +use learning rate `0.001`. The current author loop interprets `n_epochs=400` +as an epoch-0 evaluation followed by 399 optimization epochs, and the audit +records both counts rather than silently relabelling them. The paper reports `17.03 ± 0.21%` top-1 error (`82.97 ± 0.21%` accuracy) for symmetric-plastic BurstCCN. diff --git a/experiments/baseline_smoke.py b/experiments/baseline_smoke.py index c656a2f..324e333 100644 --- a/experiments/baseline_smoke.py +++ b/experiments/baseline_smoke.py @@ -8,6 +8,8 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from sdil.data import onehot from sdil.local_baselines import FANet, PEPITANet, FFNet, EPNet from sdil import probes +from experiments.import_native_baseline import (summarize_burst_history, + summarize_dualprop_hist) def check_fa_residual_transport(): @@ -133,9 +135,40 @@ def check_ep_energy_dynamics(): print("EP free particles persist across presentations") +def check_native_result_selection(): + burst_rows = [ + {"epoch": 0, "epoch/top1_error/test": 30.0, + "epoch/top1_error/val": 28.0, "_runtime": 10.0}, + {"epoch": 1, "epoch/top1_error/test": 20.0, + "epoch/top1_error/val": 22.0, "_runtime": 20.0}, + {"epoch": 2, "epoch/top1_error/test": 21.0, + "epoch/top1_error/val": 18.0, "_runtime": 30.0}, + ] + burst = summarize_burst_history(burst_rows, 3) + assert burst["primary_final_test_accuracy_percent"] == 79.0 + assert burst["validation_selected_epoch"] == 2 + assert burst["validation_selected_test_accuracy_percent"] == 79.0 + assert burst["test_selected_epoch"] == 1 + assert burst["test_selected_test_accuracy_percent"] == 80.0 + + dual = summarize_dualprop_hist({ + "train_loss": [2.0, 1.0], "train_accuracy": [30.0, 70.0], + "train_time": [4.0, 5.0], "val_loss": [1.5, 1.2], + "val_accuracy": [60.0, 65.0], "val_top5accuracy": [90.0, 92.0], + "val_time": [1.0, 1.5], "test_loss": 1.1, + "test_accuracy": 64.0, "test_top5accuracy": 93.0, "test_time": 2.0, + }, 2) + assert dual["validation_selected_epoch"] == 2 + assert dual["primary_test_accuracy_percent"] == 64.0 + assert dual["test_evaluations"] == 1 + assert dual["wall_s"] == 13.5 + print("native final/validation-selected/test-selected semantics: exact") + + if __name__ == "__main__": check_fa_residual_transport() check_pepita_output_rule() check_ff_local_optimization() check_ep_energy_dynamics() + check_native_result_selection() print("ALL BASELINE SMOKE CHECKS PASSED") diff --git a/experiments/import_native_baseline.py b/experiments/import_native_baseline.py new file mode 100644 index 0000000..ae61727 --- /dev/null +++ b/experiments/import_native_baseline.py @@ -0,0 +1,349 @@ +#!/usr/bin/env python3 +"""Import completed author-code runs into deterministic, audited JSON records. + +Run this script with the baseline's own Python environment. BurstCCN needs the +``wandb`` package to decode its offline event file; Dual Prop needs ``numpy`` +to read the author's ``hist.npy``. The importer never evaluates or retrains a +model and refuses incomplete histories, dirty source trees, revision drift, +non-finite metrics, and protocol/config mismatches. +""" +import argparse +import hashlib +import json +import math +import os +import subprocess +import sys + + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +DEFAULT_PROTOCOLS = os.path.join( + ROOT, "experiments", "native_baseline_protocols.json") + + +def file_record(path): + digest = hashlib.sha256() + with open(path, "rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return { + "path": os.path.abspath(path), + "bytes": os.path.getsize(path), + "sha256": digest.hexdigest(), + } + + +def run_text(command, cwd=None): + return subprocess.run( + command, cwd=cwd, check=True, capture_output=True, text=True).stdout.strip() + + +def git_record(path): + return { + "path": os.path.abspath(path), + "revision": run_text(["git", "rev-parse", "HEAD"], cwd=path), + "tracked_dirty": bool(run_text( + ["git", "status", "--porcelain", "--untracked-files=no"], cwd=path)), + } + + +def finite_number(value, label): + value = float(value) + if not math.isfinite(value): + raise ValueError(f"{label} is non-finite: {value!r}") + return value + + +def finite_list(values, label): + return [finite_number(value, f"{label}[{index}]") + for index, value in enumerate(values)] + + +def summarize_burst_history(rows, expected_epochs): + by_epoch = {} + for row in rows: + if "epoch" not in row: + continue + epoch = int(row["epoch"]) + if "epoch/top1_error/test" in row: + by_epoch[epoch] = row + expected = list(range(expected_epochs)) + observed = sorted(by_epoch) + if observed != expected: + raise ValueError( + f"BurstCCN history incomplete: expected epochs 0..{expected_epochs - 1}, " + f"found {len(observed)} rows ending at {observed[-1] if observed else None}") + + test_errors = finite_list( + [by_epoch[e]["epoch/top1_error/test"] for e in expected], "test top1 error") + val_errors = finite_list( + [by_epoch[e]["epoch/top1_error/val"] for e in expected], "val top1 error") + final = expected[-1] + best_test = min(range(expected_epochs), key=lambda e: test_errors[e]) + best_val = min(range(expected_epochs), key=lambda e: val_errors[e]) + runtime = finite_number(by_epoch[final].get("_runtime"), "W&B runtime") + return { + "epochs_observed": expected_epochs, + "optimization_epochs": expected_epochs - 1, + "initial_evaluation_epoch": 0, + "primary_final_test_accuracy_percent": 100.0 - test_errors[final], + "primary_final_epoch": final, + "final_validation_accuracy_percent": 100.0 - val_errors[final], + "validation_selected_epoch": best_val, + "validation_selected_validation_accuracy_percent": 100.0 - val_errors[best_val], + "validation_selected_test_accuracy_percent": 100.0 - test_errors[best_val], + "test_selected_epoch": best_test, + "test_selected_test_accuracy_percent": 100.0 - test_errors[best_test], + "test_evaluations": expected_epochs, + "wall_s": runtime, + "wall_definition": "W&B runtime from run initialization through the final epoch log", + } + + +def read_burst_history(path): + try: + from wandb.proto import wandb_internal_pb2 + from wandb.sdk.internal.datastore import DataStore + except ImportError as exc: + raise RuntimeError( + "BurstCCN import must run in an environment containing wandb") from exc + + store = DataStore() + store.open_for_scan(path) + rows = [] + while True: + data = store.scan_data() + if data is None: + break + record = wandb_internal_pb2.Record() + record.ParseFromString(data) + if record.WhichOneof("record_type") != "history": + continue + row = {} + for item in record.history.item: + key = item.key or "/".join(item.nested_key) + if key: + row[key] = json.loads(item.value_json) + rows.append(row) + return rows + + +def summarize_dualprop_hist(hist, expected_epochs): + required_vectors = ( + "train_loss", "train_accuracy", "train_time", "val_loss", + "val_accuracy", "val_top5accuracy", "val_time") + vectors = {} + for key in required_vectors: + if key not in hist: + raise ValueError(f"Dual Prop history lacks {key!r}") + values = list(hist[key]) + if len(values) != expected_epochs: + raise ValueError( + f"Dual Prop {key} has {len(values)} epochs, expected {expected_epochs}") + vectors[key] = finite_list(values, key) + for key in ("test_loss", "test_accuracy", "test_top5accuracy", "test_time"): + if key not in hist: + raise ValueError(f"Dual Prop history lacks {key!r}") + test_accuracy = finite_number(hist["test_accuracy"], "test_accuracy") + test_top5 = finite_number(hist["test_top5accuracy"], "test_top5accuracy") + test_loss = finite_number(hist["test_loss"], "test_loss") + test_time = finite_number(hist["test_time"], "test_time") + best_val = max(range(expected_epochs), key=lambda e: vectors["val_accuracy"][e]) + component_wall = (sum(vectors["train_time"]) + sum(vectors["val_time"]) + + test_time) + return { + "epochs_observed": expected_epochs, + "final_epoch": expected_epochs, + "final_train_accuracy_percent": vectors["train_accuracy"][-1], + "final_validation_accuracy_percent": vectors["val_accuracy"][-1], + "validation_selected_epoch": best_val + 1, + "validation_selected_validation_accuracy_percent": vectors["val_accuracy"][best_val], + "validation_selected_test_accuracy_percent": test_accuracy, + "primary_test_accuracy_percent": test_accuracy, + "primary_test_top5_accuracy_percent": test_top5, + "primary_test_loss": test_loss, + "test_evaluations": 1, + "test_selected_test_accuracy_percent": None, + "wall_s": component_wall, + "training_wall_s": sum(vectors["train_time"]), + "validation_wall_s": sum(vectors["val_time"]), + "test_wall_s": test_time, + "wall_definition": ( + "sum of author-measured train, validation, and final test runtimes; " + "excludes setup, checkpoint I/O, and diagnostic plotting"), + } + + +def read_dualprop_hist(path): + try: + import numpy as np + except ImportError as exc: + raise RuntimeError( + "Dual Prop import must run in an environment containing numpy") from exc + value = np.load(path, allow_pickle=True) + if value.shape != (): + raise ValueError(f"expected scalar object hist.npy, found shape {value.shape}") + hist = value.item() + if not isinstance(hist, dict): + raise ValueError(f"expected dict in hist.npy, found {type(hist).__name__}") + return hist + + +def nested_value(mapping, dotted_key): + value = mapping + for key in dotted_key.split("."): + if not isinstance(value, dict) or key not in value: + raise ValueError(f"resolved config lacks {dotted_key!r}") + value = value[key] + return value + + +def read_and_validate_config(path, required): + try: + from omegaconf import OmegaConf + except ImportError as exc: + raise RuntimeError( + "resolved Hydra config import must run in the BurstCCN environment") from exc + config = OmegaConf.to_container(OmegaConf.load(path), resolve=True) + mismatches = [] + for key, expected in required.items(): + actual = nested_value(config, key) + if actual != expected: + mismatches.append(f"{key}: expected {expected!r}, found {actual!r}") + if mismatches: + raise ValueError("resolved config mismatch:\n" + "\n".join(mismatches)) + return config + + +def environment_record(include_freeze=True): + record = { + "python_executable": os.path.abspath(sys.executable), + "python_version": sys.version, + } + if include_freeze: + freeze = run_text([sys.executable, "-m", "pip", "freeze"]).splitlines() + freeze = sorted(line for line in freeze if line.strip()) + encoded = ("\n".join(freeze) + "\n").encode() + record["pip_freeze"] = freeze + record["pip_freeze_sha256"] = hashlib.sha256(encoded).hexdigest() + return record + + +def gpu_record(index): + query = run_text([ + "nvidia-smi", f"--id={index}", + "--query-gpu=index,name,uuid,driver_version,memory.total", + "--format=csv,noheader,nounits", + ]) + fields = [value.strip() for value in query.split(",")] + if len(fields) != 5: + raise ValueError(f"unexpected nvidia-smi record: {query!r}") + return dict(zip(("physical_index", "name", "uuid", "driver_version", + "memory_total_mib"), fields)) + + +def load_protocol(path, protocol_id): + with open(path) as handle: + ledger = json.load(handle) + if ledger.get("schema_version") != 1: + raise ValueError("unsupported native protocol schema") + try: + return ledger["protocols"][protocol_id] + except KeyError as exc: + raise ValueError(f"unknown protocol {protocol_id!r}") from exc + + +def importer_provenance(): + return git_record(ROOT) + + +def import_result(args): + protocol = load_protocol(args.protocols, args.protocol) + source = git_record(args.source_dir) + if source["revision"] != protocol["source_revision"]: + raise ValueError( + f"source revision mismatch: expected {protocol['source_revision']}, " + f"found {source['revision']}") + if source["tracked_dirty"]: + raise ValueError("author source tree has tracked modifications") + local_source = importer_provenance() + if local_source["tracked_dirty"]: + raise ValueError("SDIL importer source tree has tracked modifications") + + if protocol["method"] == "BurstCCN": + if not args.resolved_config: + raise ValueError("BurstCCN requires --resolved-config") + resolved = read_and_validate_config( + args.resolved_config, protocol["required_config"]) + history = read_burst_history(args.artifact) + metrics = summarize_burst_history(history, protocol["expected_epochs"]) + elif protocol["method"] == "Dual Propagation": + if args.resolved_config: + raise ValueError("Dual Prop protocol is frozen directly in the ledger") + resolved = protocol["resolved_config"] + history = read_dualprop_hist(args.artifact) + metrics = summarize_dualprop_hist(history, protocol["expected_epochs"]) + else: + raise ValueError(f"unsupported native method {protocol['method']!r}") + + artifacts = [file_record(args.artifact)] + if args.resolved_config: + artifacts.append(file_record(args.resolved_config)) + dataset_artifacts = [file_record(path) for path in args.dataset_artifact] + result = { + "schema_version": 1, + "protocol_id": args.protocol, + "method": protocol["method"], + "dataset": protocol["dataset"], + "seed": protocol["seed"], + "protocol": protocol, + "resolved_config": resolved, + "metrics": metrics, + "audit": { + "complete": True, + "finite": True, + "expected_epochs": protocol["expected_epochs"], + "source_revision_match": True, + "source_tracked_clean": True, + "selection_semantics_frozen": True, + }, + "source_provenance": source, + "importer_provenance": local_source, + "runtime_environment": environment_record(not args.skip_package_freeze), + "gpu": gpu_record(args.gpu_index), + "dataset_artifacts": dataset_artifacts, + "run_artifacts": artifacts, + } + if os.path.exists(args.output) and not args.overwrite: + raise FileExistsError(f"refusing to overwrite {args.output}") + os.makedirs(os.path.dirname(os.path.abspath(args.output)), exist_ok=True) + with open(args.output, "w") as handle: + json.dump(result, handle, indent=2, sort_keys=True) + handle.write("\n") + print(json.dumps({ + "output": os.path.abspath(args.output), + "method": result["method"], + "metrics": result["metrics"], + }, indent=2, sort_keys=True)) + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--protocol", required=True) + parser.add_argument("--protocols", default=DEFAULT_PROTOCOLS) + parser.add_argument("--source-dir", required=True) + parser.add_argument("--artifact", required=True, + help="BurstCCN run-*.wandb or Dual Prop hist.npy") + parser.add_argument("--resolved-config", + help="BurstCCN's .hydra/config.yaml") + parser.add_argument("--dataset-artifact", action="append", default=[]) + parser.add_argument("--gpu-index", required=True, type=int) + parser.add_argument("--output", required=True) + parser.add_argument("--skip-package-freeze", action="store_true", + help="development only; final audited imports retain the freeze") + parser.add_argument("--overwrite", action="store_true") + return parser.parse_args() + + +if __name__ == "__main__": + import_result(parse_args()) diff --git a/experiments/native_baseline_protocols.json b/experiments/native_baseline_protocols.json new file mode 100644 index 0000000..9edba86 --- /dev/null +++ b/experiments/native_baseline_protocols.json @@ -0,0 +1,114 @@ +{ + "schema_version": 1, + "protocols": { + "burstccn_cifar10_symplastic_paper_seed0": { + "method": "BurstCCN", + "dataset": "CIFAR-10", + "seed": 0, + "expected_epochs": 400, + "repository_url": "https://github.com/neuralml/BurstCCN-journal", + "source_revision": "9170e110b4d18729dc5bc5d517fc52eefcd04d69", + "paper_url": "https://www.biorxiv.org/content/10.64898/2026.06.16.732595v1", + "published_accuracy_percent_mean": 82.97, + "published_accuracy_percent_sd": 0.21, + "selection_semantics": { + "primary": "final epoch test accuracy", + "validation_selected": "test accuracy at the epoch with minimum validation top-1 error", + "test_selected": "maximum test accuracy over all epochs; reported separately" + }, + "required_config": { + "model.loss_type": "mse", + "model.hidden_activation_function": "relu_rms", + "model.output_activation_function": "softmax", + "model.W.optimiser.lr": 0.002, + "model.W.optimiser.momentum": 0.9, + "model.W.optimiser.weight_decay": 0.0001, + "model.Y.mode": "sym_W", + "model.Y.grad_type": "kp", + "model.Y.optimiser.lr": 0.001, + "model.Q.mode": "sym_Y", + "model.Q.grad_type": "kp", + "model.Q.optimiser.lr": 0.001, + "dataset.dataset_name": "cifar10", + "dataset.use_validation": true, + "training.seed": 0, + "training.n_epochs": 400, + "training.batch_size": 32, + "training.label_smoothing": 0.05 + }, + "command": [ + "python", + "train.py", + "--config-name=cifar10_burstccn_kp", + "dataset.root=/home/yurenh2/sdrn/data", + "training.seed=0", + "model.hidden_activation_function=relu_rms", + "model.output_activation_function=softmax", + "model.W.optimiser.lr=0.002", + "model.Y.mode=sym_W", + "model.Q.mode=sym_Y", + "hydra.run.dir=/scratch/yurenh2/burstccn-native/cifar10_symplastic_paper_seed0" + ] + }, + "dualprop_cifar10_vgg16_a0_b0p1_seed1988": { + "method": "Dual Propagation", + "dataset": "CIFAR-10", + "seed": 1988, + "expected_epochs": 130, + "repository_url": "https://github.com/Rasmuskh/dualprop_icml_2024", + "source_revision": "7b2595b34421e1483a721dbfdeff8cdabda3a1ff", + "paper_url": "https://arxiv.org/abs/2402.08573", + "published_accuracy_percent_mean": 92.41, + "published_accuracy_percent_sd": 0.07, + "selection_semantics": { + "primary": "single test evaluation from the best-validation checkpoint", + "validation_selected": "same as primary", + "test_selected": "not available; test is evaluated only once" + }, + "resolved_config": { + "model": "VGG16", + "alpha": 0.0, + "beta": 0.1, + "learning_algorithm": "dualprop-lagr-ff", + "inference_sequence": "fwK", + "inference_passes_nudged": 16, + "batch_size": 100, + "learning_rate": 0.025, + "warmup_epochs": 10, + "decay_epochs": 120, + "momentum": 0.9, + "weight_decay": 0.0005, + "percent_train": 90, + "percent_val": 10 + }, + "command": [ + "python", + "train.py", + "--model", "VGG16", + "--dataset", "cifar10", + "--num-epochs", "130", + "--batch-size", "100", + "--alpha", "0.0", + "--beta", "0.1", + "--loss", "sce", + "--inference-sequence", "fwK", + "--inference-passes-nudged", "16", + "--learning-rate", "0.025", + "--learning-rate-final", "0.000002", + "--warmup-learning-rate", "0.001", + "--learning-algorithm", "dualprop-lagr-ff", + "--activation", "relu", + "--decay-epochs", "120", + "--warmup-epochs", "10", + "--momentum", "0.9", + "--weight-decay", "5e-4", + "--dtype", "float32", + "--param-dtype", "float32", + "--percent-train", "90", + "--percent-val", "10", + "--seeds", "1988", + "--experiment-name", "sdil-dp-full-a0-b0p1-seed1988-v1" + ] + } + } +} -- cgit v1.2.3