diff options
| -rw-r--r-- | RAIN_EP_BIAS_CONFIRMATION.md | 54 | ||||
| -rw-r--r-- | experiments/analyze_rain_ep_bias_c1.py | 178 | ||||
| -rw-r--r-- | experiments/rain_ep_bias_c1.py | 84 |
3 files changed, 316 insertions, 0 deletions
diff --git a/RAIN_EP_BIAS_CONFIRMATION.md b/RAIN_EP_BIAS_CONFIRMATION.md new file mode 100644 index 0000000..9aa22c1 --- /dev/null +++ b/RAIN_EP_BIAS_CONFIRMATION.md @@ -0,0 +1,54 @@ +# Frozen Rain EP layer-state confirmation + +This protocol was frozen after the single-seed S1 development screen in +`results/ep_bias/s1_summary.json`. S1 selected the only promoted corruption +ratio, `0.01`; stronger `0.1` and `4.0` conditions remain recorded failures. + +## Fixed protocol + +- Author implementation: `rain-neuromorphics/energy-based-learning`, revision + `6b253fd8a5d267535f58ab79992256ef10031ceb`. +- Model and learner: author `ConvHopfieldEnergy28` 32--64--10 with positive EP, + nudging `0.25`, 12 training relaxation iterations, 30 inference iterations, + author local parameter rules and SGD settings. +- Data: FashionMNIST training set only. Data seed `6100` fixes disjoint 10,000 + training and 2,000 validation examples. The test set is not evaluated. +- Model/order seeds: `1989, 1990, 1991, 1992, 1993`; batch size 128; three + epochs; no epoch selection. +- Conditions: clean, raw structured bias, same-RMS zero-mean noise, constant + predictor, affine innovation predictor, and oracle subtraction. +- Bias: per-neuron affine function of the local first-phase state, normalized + by the experimenter to `0.01` times the initial clean layer-state-difference + RMS. This normalization is not visible to either predictor. +- Predictor: normalized local LMS at rate `0.2`. Constant and innovation see + the same 128 instruction-off observations in the existing first EP phase of + the first training minibatch. Both are then frozen. There is no extra + equilibrium phase and no backpropagation. +- Hardware: all six conditions for a seed run sequentially on one physical + GPU. Different seeds may run on GPUs 5 and 7 in parallel. + +The injected bias is deliberately inside the affine predictor class. Passing +therefore confirms correction, causality and transfer to an EP implementation; +it is not independent evidence that a real device exposes the same feature. + +## Frozen gate + +Across all five paired seeds: + +1. clean, noise, constant, innovation and oracle complete three finite epochs; +2. mean clean validation accuracy is at least 70%; +3. raw loses at least 10 accuracy points relative to clean in every seed; +4. innovation beats raw and constant in every seed, with 95% lower bounds of + at least 10 and 5 accuracy points respectively; +5. the 95% upper bounds on clean-minus-innovation, absolute + innovation-minus-oracle, and absolute noise-minus-clean are each below + three accuracy points; +6. innovation has lower final residual/clean state-difference RMS than constant + in every seed; +7. constant and innovation each use exactly 128 neutral observations; +8. innovation/clean mean wall-time ratio is at most `1.15`; +9. all records share one author revision, one SDIL revision, the fixed data + split and `autodiff_used_for_learning=false`. + +Failure closes this exact confirmation. It does not authorize a new ratio, +predictor rate, seed replacement or extra calibration on the same holdout. diff --git a/experiments/analyze_rain_ep_bias_c1.py b/experiments/analyze_rain_ep_bias_c1.py new file mode 100644 index 0000000..77d9b92 --- /dev/null +++ b/experiments/analyze_rain_ep_bias_c1.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +"""Audit the frozen five-seed Rain EP layer-state confirmation.""" + +from __future__ import annotations + +import json +import math +from pathlib import Path +import statistics + +from rain_ep_bias_c1 import CONDITIONS, RESULT_ROOT, SEEDS + + +ROOT = Path(__file__).resolve().parents[1] +T95 = 2.131846786326649 + + +def read(path: Path) -> dict: + with path.open(encoding="utf-8") as handle: + return json.load(handle) + + +def path_for(seed: int, file_mode: str) -> Path: + return RESULT_ROOT / f"rain-ep-c1-s{seed}-{file_mode}.json" + + +def interval(values: list[float], absolute_mean: bool = False) -> dict: + mean = statistics.fmean(values) + sem = statistics.stdev(values) / math.sqrt(len(values)) + center = abs(mean) if absolute_mean else mean + return { + "mean": mean, + "sem": sem, + "lower_95": mean - T95 * sem, + "upper_95": center + T95 * sem, + } + + +def main() -> None: + records = {} + missing = [] + for seed in SEEDS: + for mode, file_mode in CONDITIONS: + path = path_for(seed, file_mode) + if not path.is_file(): + missing.append(f"s{seed}:{mode}") + else: + records[(seed, mode)] = read(path) + if missing: + raise RuntimeError("missing C1 cells: " + ", ".join(missing)) + + rows = [] + gain_raw = [] + gain_constant = [] + deficit_clean = [] + difference_oracle = [] + difference_noise = [] + wall_ratios = [] + for seed in SEEDS: + final = { + mode: records[(seed, mode)]["final"] + for mode, _ in CONDITIONS + } + accuracy = { + mode: final[mode]["test_accuracy"] for mode, _ in CONDITIONS + } + gain_raw.append(accuracy["innovation"] - accuracy["raw"]) + gain_constant.append( + accuracy["innovation"] - accuracy["constant"]) + deficit_clean.append(accuracy["clean"] - accuracy["innovation"]) + difference_oracle.append( + accuracy["innovation"] - accuracy["oracle"]) + difference_noise.append( + accuracy["same_rms_noise"] - accuracy["clean"]) + wall_ratios.append( + final["innovation"]["wall_seconds"] / final["clean"]["wall_seconds"]) + rows.append({ + "seed": seed, + **{mode: value for mode, value in accuracy.items()}, + "innovation_minus_raw": gain_raw[-1], + "innovation_minus_constant": gain_constant[-1], + "clean_minus_innovation": deficit_clean[-1], + "innovation_minus_oracle": difference_oracle[-1], + "noise_minus_clean": difference_noise[-1], + "innovation_wall_over_clean": wall_ratios[-1], + }) + + clean_mean = statistics.fmean(row["clean"] for row in rows) + required_finite = ( + "clean", "same_rms_noise", "constant", "innovation", "oracle") + finite_complete = all( + len(records[(seed, mode)]["metrics"]) == 3 + and all(metric["finite"] for metric in records[(seed, mode)]["metrics"]) + for seed in SEEDS for mode in required_finite + ) + innovation_residual_below_constant = all( + records[(seed, "innovation")]["final"]["corrector"] + ["residual_to_clean_state_difference_rms"] + < records[(seed, "constant")]["final"]["corrector"] + ["residual_to_clean_state_difference_rms"] + for seed in SEEDS + ) + neutral_matched = all( + records[(seed, mode)]["final"]["corrector"]["neutral_observations"] + == 128 + for seed in SEEDS for mode in ("constant", "innovation") + ) + source_revisions = { + record["sdil"]["revision"] for record in records.values() + } + author_revisions = { + record["author"]["revision"] for record in records.values() + } + protocol_fixed = all( + record["protocol"]["evaluation_split"] == "train_holdout" + and record["protocol"]["data_seed"] == 6100 + and record["protocol"]["bias_ratio"] == 0.01 + and record["protocol"]["adapter"] == "layer" + and record["protocol"]["autodiff_used_for_learning"] is False + and record["protocol"]["extra_equilibrium_phases_for_predictor"] == 0 + for record in records.values() + ) + stats = { + "innovation_minus_raw": interval(gain_raw), + "innovation_minus_constant": interval(gain_constant), + "clean_minus_innovation": interval(deficit_clean), + "innovation_minus_oracle": interval( + difference_oracle, absolute_mean=True), + "noise_minus_clean": interval(difference_noise, absolute_mean=True), + "innovation_wall_over_clean": interval(wall_ratios), + } + checks = { + "finite_complete_required_conditions": finite_complete, + "mean_clean_validation_at_least_70": clean_mean >= 0.70, + "raw_damage_at_least_10_points_every_seed": all( + row["clean"] - row["raw"] >= 0.10 for row in rows), + "innovation_beats_raw_every_seed": all(value > 0 for value in gain_raw), + "innovation_raw_lower_95_at_least_10_points": ( + stats["innovation_minus_raw"]["lower_95"] >= 0.10), + "innovation_beats_constant_every_seed": all( + value > 0 for value in gain_constant), + "innovation_constant_lower_95_at_least_5_points": ( + stats["innovation_minus_constant"]["lower_95"] >= 0.05), + "clean_deficit_upper_95_below_3_points": ( + stats["clean_minus_innovation"]["upper_95"] < 0.03), + "oracle_abs_mean_upper_95_below_3_points": ( + stats["innovation_minus_oracle"]["upper_95"] < 0.03), + "noise_abs_mean_upper_95_below_3_points": ( + stats["noise_minus_clean"]["upper_95"] < 0.03), + "innovation_residual_below_constant_every_seed": ( + innovation_residual_below_constant), + "matched_128_neutral_observations": neutral_matched, + "mean_wall_overhead_at_most_15_percent": ( + stats["innovation_wall_over_clean"]["mean"] <= 1.15), + "single_sdil_revision": len(source_revisions) == 1, + "single_author_revision": len(author_revisions) == 1, + "protocol_fixed_and_bp_free": protocol_fixed, + } + report = { + "stage": "rain_ep_bias_c1", + "gate": "pass" if all(checks.values()) else "fail", + "checks": checks, + "rows": rows, + "statistics": stats, + "mean_clean_validation_accuracy": clean_mean, + "num_records": len(records), + "sdil_revision": next(iter(source_revisions)), + "author_revision": next(iter(author_revisions)), + "test_policy": "test_set_never_loaded; fixed epoch-3 train holdout", + } + output = RESULT_ROOT.parent / "c1_gate.json" + output.write_text( + json.dumps(report, indent=2, sort_keys=True, allow_nan=False) + "\n") + print(json.dumps(report, indent=2, sort_keys=True, allow_nan=False)) + + +if __name__ == "__main__": + main() diff --git a/experiments/rain_ep_bias_c1.py b/experiments/rain_ep_bias_c1.py new file mode 100644 index 0000000..67567de --- /dev/null +++ b/experiments/rain_ep_bias_c1.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""Run one same-GPU seed of the frozen Rain EP bias confirmation.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import subprocess +import sys +import time + + +ROOT = Path(__file__).resolve().parents[1] +ENDPOINT = ROOT / "experiments" / "rain_ep_bias_train.py" +RESULT_ROOT = ROOT / "results" / "ep_bias" / "c1" +AUTHOR_REVISION = "6b253fd8a5d267535f58ab79992256ef10031ceb" +SEEDS = (1989, 1990, 1991, 1992, 1993) +CONDITIONS = ( + ("clean", "clean"), + ("raw", "raw"), + ("same_rms_noise", "noise"), + ("constant", "constant"), + ("innovation", "innovation"), + ("oracle", "oracle"), +) + + +def revision(path: Path) -> str: + return subprocess.check_output( + ["git", "-C", str(path), "rev-parse", "HEAD"], text=True).strip() + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--author-root", type=Path, required=True) + parser.add_argument("--seed", type=int, choices=SEEDS, required=True) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + author_root = args.author_root.resolve() + if revision(author_root) != AUTHOR_REVISION: + raise ValueError("Rain author revision changed") + RESULT_ROOT.mkdir(parents=True, exist_ok=True) + started = time.time() + outputs = [] + for mode, file_mode in CONDITIONS: + output = RESULT_ROOT / f"rain-ep-c1-s{args.seed}-{file_mode}.json" + if output.exists(): + raise FileExistsError(output) + command = [ + sys.executable, str(ENDPOINT), + "--author-root", str(author_root), + "--device", "cuda", "--adapter", "layer", "--mode", mode, + "--bias-ratio", "0.01", "--predictor-rate", "0.2", + "--layer-calibration-steps", "1", "--epochs", "3", + "--train-limit", "10000", "--test-limit", "2000", + "--evaluation-split", "train_holdout", "--data-seed", "6100", + "--batch-size", "128", "--training-iterations", "12", + "--inference-iterations", "30", "--seed", str(args.seed), + "--output", str(output), + ] + subprocess.run(command, cwd=author_root, check=True) + outputs.append(str(output.relative_to(ROOT))) + launch = { + "stage": "rain_ep_bias_c1", + "seed": args.seed, + "conditions": [condition for condition, _ in CONDITIONS], + "outputs": outputs, + "author_revision": revision(author_root), + "sdil_revision": revision(ROOT), + "cuda_visible_devices": os.environ.get("CUDA_VISIBLE_DEVICES"), + "wall_seconds": time.time() - started, + } + path = RESULT_ROOT / f"launch-s{args.seed}.json" + path.write_text(json.dumps(launch, indent=2, sort_keys=True) + "\n") + print(json.dumps(launch, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() |