results: validate beta-independent Rain bias

1 files changed, 92 insertions, 0 deletions

diff --git a/experiments/analyze_rain_ep_s3.py b/experiments/analyze_rain_ep_s3.py
new file mode 100644
index 0000000..7e735d6
--- /dev/null
+++ b/experiments/analyze_rain_ep_s3.py
@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+"""Summarize the beta-independent Rain hardware-bias development screen."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[1]
+RESULT_ROOT = ROOT / "results" / "ep_bias" / "s3"
+FILES = {
+    "clean": "clean-e3-s1988.json",
+    "raw": "raw-free2e-4-e3-s1988.json",
+    "constant": "constant-free2e-4-e3-s1988.json",
+    "innovation": "innovation-free2e-4-e3-s1988.json",
+    "random_beta": "random-beta-free2e-4-e3-s1988.json",
+    "random_beta_zero_bias": "random-beta-zero-bias-e3-s1988.json",
+    "fixed_negative_zero_bias": "fixed-negative-zero-bias-e3-s1988.json",
+}
+
+
+def read(name: str) -> dict:
+    with (RESULT_ROOT / FILES[name]).open(encoding="utf-8") as handle:
+        return json.load(handle)
+
+
+def main() -> None:
+    records = {name: read(name) for name in FILES}
+    rows = {
+        name: {
+            "final_validation_accuracy": record["final"]["test_accuracy"],
+            "wall_seconds": record["final"]["wall_seconds"],
+            "finite": all(metric["finite"] for metric in record["metrics"]),
+            "final_corrector": record["final"].get("corrector", {}),
+            "beta_sign_counts": record.get("beta_sign_counts", {}),
+        }
+        for name, record in records.items()
+    }
+    clean = rows["clean"]["final_validation_accuracy"]
+    raw = rows["raw"]["final_validation_accuracy"]
+    constant = rows["constant"]["final_validation_accuracy"]
+    innovation = rows["innovation"]["final_validation_accuracy"]
+    report = {
+        "stage": "rain_ep_beta_independent_hardware_bias_s3",
+        "status": "positive_single_seed_development_negative_beta_unavailable",
+        "bias_model": {
+            "normalization": "initial_free_layer_state_rms",
+            "ratio": 0.0002,
+            "depends_on_beta_sign": False,
+            "depends_on_clean_task_difference": False,
+        },
+        "rows": rows,
+        "effects": {
+            "innovation_minus_raw_accuracy_points": 100.0 * (
+                innovation - raw),
+            "innovation_minus_constant_accuracy_points": 100.0 * (
+                innovation - constant),
+            "clean_minus_innovation_accuracy_points": 100.0 * (
+                clean - innovation),
+            "innovation_wall_over_clean": (
+                rows["innovation"]["wall_seconds"]
+                / rows["clean"]["wall_seconds"]),
+            "innovation_final_residual_to_clean_state_difference_rms": (
+                rows["innovation"]["final_corrector"]
+                ["residual_to_clean_state_difference_rms"]),
+            "constant_final_residual_to_clean_state_difference_rms": (
+                rows["constant"]["final_corrector"]
+                ["residual_to_clean_state_difference_rms"]),
+        },
+        "random_beta_boundary": {
+            "random_sign_fails_with_zero_bias": (
+                rows["random_beta_zero_bias"]["final_validation_accuracy"] < 0.2),
+            "fixed_negative_fails_with_zero_bias": (
+                rows["fixed_negative_zero_bias"]["final_validation_accuracy"] < 0.2),
+            "interpretation": (
+                "Negative-beta relaxation is not a valid drop-in baseline under "
+                "the pinned author's positive-EP hyperparameters. Random-sign "
+                "beta requires separate stability work before comparison; this "
+                "screen is not an SDIL win over a functioning random-beta method."
+            ),
+        },
+        "test_policy": "new development train holdout data seed 6200",
+    }
+    output = RESULT_ROOT.parent / "s3_summary.json"
+    output.write_text(
+        json.dumps(report, indent=2, sort_keys=True, allow_nan=False) + "\n")
+    print(json.dumps(report, indent=2, sort_keys=True, allow_nan=False))
+
+
+if __name__ == "__main__":
+    main()