#!/usr/bin/env python3 """Summarize the beta-independent Rain hardware-bias development screen.""" from __future__ import annotations import json from pathlib import Path ROOT = Path(__file__).resolve().parents[1] RESULT_ROOT = ROOT / "results" / "ep_bias" / "s3" FILES = { "clean": "clean-e3-s1988.json", "raw": "raw-free2e-4-e3-s1988.json", "constant": "constant-free2e-4-e3-s1988.json", "innovation": "innovation-free2e-4-e3-s1988.json", "random_beta": "random-beta-free2e-4-e3-s1988.json", "random_beta_zero_bias": "random-beta-zero-bias-e3-s1988.json", "fixed_negative_zero_bias": "fixed-negative-zero-bias-e3-s1988.json", } def read(name: str) -> dict: with (RESULT_ROOT / FILES[name]).open(encoding="utf-8") as handle: return json.load(handle) def main() -> None: records = {name: read(name) for name in FILES} rows = { name: { "final_validation_accuracy": record["final"]["test_accuracy"], "wall_seconds": record["final"]["wall_seconds"], "finite": all(metric["finite"] for metric in record["metrics"]), "final_corrector": record["final"].get("corrector", {}), "beta_sign_counts": record.get("beta_sign_counts", {}), } for name, record in records.items() } clean = rows["clean"]["final_validation_accuracy"] raw = rows["raw"]["final_validation_accuracy"] constant = rows["constant"]["final_validation_accuracy"] innovation = rows["innovation"]["final_validation_accuracy"] report = { "stage": "rain_ep_beta_independent_hardware_bias_s3", "status": "positive_single_seed_development_negative_beta_unavailable", "bias_model": { "normalization": "initial_free_layer_state_rms", "ratio": 0.0002, "depends_on_beta_sign": False, "depends_on_clean_task_difference": False, }, "rows": rows, "effects": { "innovation_minus_raw_accuracy_points": 100.0 * ( innovation - raw), "innovation_minus_constant_accuracy_points": 100.0 * ( innovation - constant), "clean_minus_innovation_accuracy_points": 100.0 * ( clean - innovation), "innovation_wall_over_clean": ( rows["innovation"]["wall_seconds"] / rows["clean"]["wall_seconds"]), "innovation_final_residual_to_clean_state_difference_rms": ( rows["innovation"]["final_corrector"] ["residual_to_clean_state_difference_rms"]), "constant_final_residual_to_clean_state_difference_rms": ( rows["constant"]["final_corrector"] ["residual_to_clean_state_difference_rms"]), }, "random_beta_boundary": { "random_sign_fails_with_zero_bias": ( rows["random_beta_zero_bias"]["final_validation_accuracy"] < 0.2), "fixed_negative_fails_with_zero_bias": ( rows["fixed_negative_zero_bias"]["final_validation_accuracy"] < 0.2), "interpretation": ( "Negative-beta relaxation is not a valid drop-in baseline under " "the pinned author's positive-EP hyperparameters. Random-sign " "beta requires separate stability work before comparison; this " "screen is not an SDIL win over a functioning random-beta method." ), }, "test_policy": "new development train holdout data seed 6200", } output = RESULT_ROOT.parent / "s3_summary.json" output.write_text( json.dumps(report, indent=2, sort_keys=True, allow_nan=False) + "\n") print(json.dumps(report, indent=2, sort_keys=True, allow_nan=False)) if __name__ == "__main__": main()