experiment: freeze Rain EP bias confirmation

3 files changed, 316 insertions, 0 deletions

diff --git a/RAIN_EP_BIAS_CONFIRMATION.md b/RAIN_EP_BIAS_CONFIRMATION.md
new file mode 100644
index 0000000..9aa22c1
--- /dev/null
+++ b/RAIN_EP_BIAS_CONFIRMATION.md
@@ -0,0 +1,54 @@
+# Frozen Rain EP layer-state confirmation
+
+This protocol was frozen after the single-seed S1 development screen in
+`results/ep_bias/s1_summary.json`. S1 selected the only promoted corruption
+ratio, `0.01`; stronger `0.1` and `4.0` conditions remain recorded failures.
+
+## Fixed protocol
+
+- Author implementation: `rain-neuromorphics/energy-based-learning`, revision
+  `6b253fd8a5d267535f58ab79992256ef10031ceb`.
+- Model and learner: author `ConvHopfieldEnergy28` 32--64--10 with positive EP,
+  nudging `0.25`, 12 training relaxation iterations, 30 inference iterations,
+  author local parameter rules and SGD settings.
+- Data: FashionMNIST training set only. Data seed `6100` fixes disjoint 10,000
+  training and 2,000 validation examples. The test set is not evaluated.
+- Model/order seeds: `1989, 1990, 1991, 1992, 1993`; batch size 128; three
+  epochs; no epoch selection.
+- Conditions: clean, raw structured bias, same-RMS zero-mean noise, constant
+  predictor, affine innovation predictor, and oracle subtraction.
+- Bias: per-neuron affine function of the local first-phase state, normalized
+  by the experimenter to `0.01` times the initial clean layer-state-difference
+  RMS. This normalization is not visible to either predictor.
+- Predictor: normalized local LMS at rate `0.2`. Constant and innovation see
+  the same 128 instruction-off observations in the existing first EP phase of
+  the first training minibatch. Both are then frozen. There is no extra
+  equilibrium phase and no backpropagation.
+- Hardware: all six conditions for a seed run sequentially on one physical
+  GPU. Different seeds may run on GPUs 5 and 7 in parallel.
+
+The injected bias is deliberately inside the affine predictor class. Passing
+therefore confirms correction, causality and transfer to an EP implementation;
+it is not independent evidence that a real device exposes the same feature.
+
+## Frozen gate
+
+Across all five paired seeds:
+
+1. clean, noise, constant, innovation and oracle complete three finite epochs;
+2. mean clean validation accuracy is at least 70%;
+3. raw loses at least 10 accuracy points relative to clean in every seed;
+4. innovation beats raw and constant in every seed, with 95% lower bounds of
+   at least 10 and 5 accuracy points respectively;
+5. the 95% upper bounds on clean-minus-innovation, absolute
+   innovation-minus-oracle, and absolute noise-minus-clean are each below
+   three accuracy points;
+6. innovation has lower final residual/clean state-difference RMS than constant
+   in every seed;
+7. constant and innovation each use exactly 128 neutral observations;
+8. innovation/clean mean wall-time ratio is at most `1.15`;
+9. all records share one author revision, one SDIL revision, the fixed data
+   split and `autodiff_used_for_learning=false`.
+
+Failure closes this exact confirmation. It does not authorize a new ratio,
+predictor rate, seed replacement or extra calibration on the same holdout.
diff --git a/experiments/analyze_rain_ep_bias_c1.py b/experiments/analyze_rain_ep_bias_c1.py
new file mode 100644
index 0000000..77d9b92
--- /dev/null
+++ b/experiments/analyze_rain_ep_bias_c1.py
@@ -0,0 +1,178 @@
+#!/usr/bin/env python3
+"""Audit the frozen five-seed Rain EP layer-state confirmation."""
+
+from __future__ import annotations
+
+import json
+import math
+from pathlib import Path
+import statistics
+
+from rain_ep_bias_c1 import CONDITIONS, RESULT_ROOT, SEEDS
+
+
+ROOT = Path(__file__).resolve().parents[1]
+T95 = 2.131846786326649
+
+
+def read(path: Path) -> dict:
+    with path.open(encoding="utf-8") as handle:
+        return json.load(handle)
+
+
+def path_for(seed: int, file_mode: str) -> Path:
+    return RESULT_ROOT / f"rain-ep-c1-s{seed}-{file_mode}.json"
+
+
+def interval(values: list[float], absolute_mean: bool = False) -> dict:
+    mean = statistics.fmean(values)
+    sem = statistics.stdev(values) / math.sqrt(len(values))
+    center = abs(mean) if absolute_mean else mean
+    return {
+        "mean": mean,
+        "sem": sem,
+        "lower_95": mean - T95 * sem,
+        "upper_95": center + T95 * sem,
+    }
+
+
+def main() -> None:
+    records = {}
+    missing = []
+    for seed in SEEDS:
+        for mode, file_mode in CONDITIONS:
+            path = path_for(seed, file_mode)
+            if not path.is_file():
+                missing.append(f"s{seed}:{mode}")
+            else:
+                records[(seed, mode)] = read(path)
+    if missing:
+        raise RuntimeError("missing C1 cells: " + ", ".join(missing))
+
+    rows = []
+    gain_raw = []
+    gain_constant = []
+    deficit_clean = []
+    difference_oracle = []
+    difference_noise = []
+    wall_ratios = []
+    for seed in SEEDS:
+        final = {
+            mode: records[(seed, mode)]["final"]
+            for mode, _ in CONDITIONS
+        }
+        accuracy = {
+            mode: final[mode]["test_accuracy"] for mode, _ in CONDITIONS
+        }
+        gain_raw.append(accuracy["innovation"] - accuracy["raw"])
+        gain_constant.append(
+            accuracy["innovation"] - accuracy["constant"])
+        deficit_clean.append(accuracy["clean"] - accuracy["innovation"])
+        difference_oracle.append(
+            accuracy["innovation"] - accuracy["oracle"])
+        difference_noise.append(
+            accuracy["same_rms_noise"] - accuracy["clean"])
+        wall_ratios.append(
+            final["innovation"]["wall_seconds"] / final["clean"]["wall_seconds"])
+        rows.append({
+            "seed": seed,
+            **{mode: value for mode, value in accuracy.items()},
+            "innovation_minus_raw": gain_raw[-1],
+            "innovation_minus_constant": gain_constant[-1],
+            "clean_minus_innovation": deficit_clean[-1],
+            "innovation_minus_oracle": difference_oracle[-1],
+            "noise_minus_clean": difference_noise[-1],
+            "innovation_wall_over_clean": wall_ratios[-1],
+        })
+
+    clean_mean = statistics.fmean(row["clean"] for row in rows)
+    required_finite = (
+        "clean", "same_rms_noise", "constant", "innovation", "oracle")
+    finite_complete = all(
+        len(records[(seed, mode)]["metrics"]) == 3
+        and all(metric["finite"] for metric in records[(seed, mode)]["metrics"])
+        for seed in SEEDS for mode in required_finite
+    )
+    innovation_residual_below_constant = all(
+        records[(seed, "innovation")]["final"]["corrector"]
+        ["residual_to_clean_state_difference_rms"]
+        < records[(seed, "constant")]["final"]["corrector"]
+        ["residual_to_clean_state_difference_rms"]
+        for seed in SEEDS
+    )
+    neutral_matched = all(
+        records[(seed, mode)]["final"]["corrector"]["neutral_observations"]
+        == 128
+        for seed in SEEDS for mode in ("constant", "innovation")
+    )
+    source_revisions = {
+        record["sdil"]["revision"] for record in records.values()
+    }
+    author_revisions = {
+        record["author"]["revision"] for record in records.values()
+    }
+    protocol_fixed = all(
+        record["protocol"]["evaluation_split"] == "train_holdout"
+        and record["protocol"]["data_seed"] == 6100
+        and record["protocol"]["bias_ratio"] == 0.01
+        and record["protocol"]["adapter"] == "layer"
+        and record["protocol"]["autodiff_used_for_learning"] is False
+        and record["protocol"]["extra_equilibrium_phases_for_predictor"] == 0
+        for record in records.values()
+    )
+    stats = {
+        "innovation_minus_raw": interval(gain_raw),
+        "innovation_minus_constant": interval(gain_constant),
+        "clean_minus_innovation": interval(deficit_clean),
+        "innovation_minus_oracle": interval(
+            difference_oracle, absolute_mean=True),
+        "noise_minus_clean": interval(difference_noise, absolute_mean=True),
+        "innovation_wall_over_clean": interval(wall_ratios),
+    }
+    checks = {
+        "finite_complete_required_conditions": finite_complete,
+        "mean_clean_validation_at_least_70": clean_mean >= 0.70,
+        "raw_damage_at_least_10_points_every_seed": all(
+            row["clean"] - row["raw"] >= 0.10 for row in rows),
+        "innovation_beats_raw_every_seed": all(value > 0 for value in gain_raw),
+        "innovation_raw_lower_95_at_least_10_points": (
+            stats["innovation_minus_raw"]["lower_95"] >= 0.10),
+        "innovation_beats_constant_every_seed": all(
+            value > 0 for value in gain_constant),
+        "innovation_constant_lower_95_at_least_5_points": (
+            stats["innovation_minus_constant"]["lower_95"] >= 0.05),
+        "clean_deficit_upper_95_below_3_points": (
+            stats["clean_minus_innovation"]["upper_95"] < 0.03),
+        "oracle_abs_mean_upper_95_below_3_points": (
+            stats["innovation_minus_oracle"]["upper_95"] < 0.03),
+        "noise_abs_mean_upper_95_below_3_points": (
+            stats["noise_minus_clean"]["upper_95"] < 0.03),
+        "innovation_residual_below_constant_every_seed": (
+            innovation_residual_below_constant),
+        "matched_128_neutral_observations": neutral_matched,
+        "mean_wall_overhead_at_most_15_percent": (
+            stats["innovation_wall_over_clean"]["mean"] <= 1.15),
+        "single_sdil_revision": len(source_revisions) == 1,
+        "single_author_revision": len(author_revisions) == 1,
+        "protocol_fixed_and_bp_free": protocol_fixed,
+    }
+    report = {
+        "stage": "rain_ep_bias_c1",
+        "gate": "pass" if all(checks.values()) else "fail",
+        "checks": checks,
+        "rows": rows,
+        "statistics": stats,
+        "mean_clean_validation_accuracy": clean_mean,
+        "num_records": len(records),
+        "sdil_revision": next(iter(source_revisions)),
+        "author_revision": next(iter(author_revisions)),
+        "test_policy": "test_set_never_loaded; fixed epoch-3 train holdout",
+    }
+    output = RESULT_ROOT.parent / "c1_gate.json"
+    output.write_text(
+        json.dumps(report, indent=2, sort_keys=True, allow_nan=False) + "\n")
+    print(json.dumps(report, indent=2, sort_keys=True, allow_nan=False))
+
+
+if __name__ == "__main__":
+    main()
diff --git a/experiments/rain_ep_bias_c1.py b/experiments/rain_ep_bias_c1.py
new file mode 100644
index 0000000..67567de
--- /dev/null
+++ b/experiments/rain_ep_bias_c1.py
@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+"""Run one same-GPU seed of the frozen Rain EP bias confirmation."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+from pathlib import Path
+import subprocess
+import sys
+import time
+
+
+ROOT = Path(__file__).resolve().parents[1]
+ENDPOINT = ROOT / "experiments" / "rain_ep_bias_train.py"
+RESULT_ROOT = ROOT / "results" / "ep_bias" / "c1"
+AUTHOR_REVISION = "6b253fd8a5d267535f58ab79992256ef10031ceb"
+SEEDS = (1989, 1990, 1991, 1992, 1993)
+CONDITIONS = (
+    ("clean", "clean"),
+    ("raw", "raw"),
+    ("same_rms_noise", "noise"),
+    ("constant", "constant"),
+    ("innovation", "innovation"),
+    ("oracle", "oracle"),
+)
+
+
+def revision(path: Path) -> str:
+    return subprocess.check_output(
+        ["git", "-C", str(path), "rev-parse", "HEAD"], text=True).strip()
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--author-root", type=Path, required=True)
+    parser.add_argument("--seed", type=int, choices=SEEDS, required=True)
+    return parser.parse_args()
+
+
+def main() -> None:
+    args = parse_args()
+    author_root = args.author_root.resolve()
+    if revision(author_root) != AUTHOR_REVISION:
+        raise ValueError("Rain author revision changed")
+    RESULT_ROOT.mkdir(parents=True, exist_ok=True)
+    started = time.time()
+    outputs = []
+    for mode, file_mode in CONDITIONS:
+        output = RESULT_ROOT / f"rain-ep-c1-s{args.seed}-{file_mode}.json"
+        if output.exists():
+            raise FileExistsError(output)
+        command = [
+            sys.executable, str(ENDPOINT),
+            "--author-root", str(author_root),
+            "--device", "cuda", "--adapter", "layer", "--mode", mode,
+            "--bias-ratio", "0.01", "--predictor-rate", "0.2",
+            "--layer-calibration-steps", "1", "--epochs", "3",
+            "--train-limit", "10000", "--test-limit", "2000",
+            "--evaluation-split", "train_holdout", "--data-seed", "6100",
+            "--batch-size", "128", "--training-iterations", "12",
+            "--inference-iterations", "30", "--seed", str(args.seed),
+            "--output", str(output),
+        ]
+        subprocess.run(command, cwd=author_root, check=True)
+        outputs.append(str(output.relative_to(ROOT)))
+    launch = {
+        "stage": "rain_ep_bias_c1",
+        "seed": args.seed,
+        "conditions": [condition for condition, _ in CONDITIONS],
+        "outputs": outputs,
+        "author_revision": revision(author_root),
+        "sdil_revision": revision(ROOT),
+        "cuda_visible_devices": os.environ.get("CUDA_VISIBLE_DEVICES"),
+        "wall_seconds": time.time() - started,
+    }
+    path = RESULT_ROOT / f"launch-s{args.seed}.json"
+    path.write_text(json.dumps(launch, indent=2, sort_keys=True) + "\n")
+    print(json.dumps(launch, indent=2, sort_keys=True))
+
+
+if __name__ == "__main__":
+    main()