1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
|
#!/usr/bin/env python3
"""Summarize the beta-independent Rain hardware-bias development screen."""
from __future__ import annotations
import json
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
RESULT_ROOT = ROOT / "results" / "ep_bias" / "s3"
FILES = {
"clean": "clean-e3-s1988.json",
"raw": "raw-free2e-4-e3-s1988.json",
"constant": "constant-free2e-4-e3-s1988.json",
"innovation": "innovation-free2e-4-e3-s1988.json",
"random_beta": "random-beta-free2e-4-e3-s1988.json",
"random_beta_zero_bias": "random-beta-zero-bias-e3-s1988.json",
"fixed_negative_zero_bias": "fixed-negative-zero-bias-e3-s1988.json",
}
def read(name: str) -> dict:
with (RESULT_ROOT / FILES[name]).open(encoding="utf-8") as handle:
return json.load(handle)
def main() -> None:
records = {name: read(name) for name in FILES}
rows = {
name: {
"final_validation_accuracy": record["final"]["test_accuracy"],
"wall_seconds": record["final"]["wall_seconds"],
"finite": all(metric["finite"] for metric in record["metrics"]),
"final_corrector": record["final"].get("corrector", {}),
"beta_sign_counts": record.get("beta_sign_counts", {}),
}
for name, record in records.items()
}
clean = rows["clean"]["final_validation_accuracy"]
raw = rows["raw"]["final_validation_accuracy"]
constant = rows["constant"]["final_validation_accuracy"]
innovation = rows["innovation"]["final_validation_accuracy"]
report = {
"stage": "rain_ep_beta_independent_hardware_bias_s3",
"status": "positive_single_seed_development_negative_beta_unavailable",
"bias_model": {
"normalization": "initial_free_layer_state_rms",
"ratio": 0.0002,
"depends_on_beta_sign": False,
"depends_on_clean_task_difference": False,
},
"rows": rows,
"effects": {
"innovation_minus_raw_accuracy_points": 100.0 * (
innovation - raw),
"innovation_minus_constant_accuracy_points": 100.0 * (
innovation - constant),
"clean_minus_innovation_accuracy_points": 100.0 * (
clean - innovation),
"innovation_wall_over_clean": (
rows["innovation"]["wall_seconds"]
/ rows["clean"]["wall_seconds"]),
"innovation_final_residual_to_clean_state_difference_rms": (
rows["innovation"]["final_corrector"]
["residual_to_clean_state_difference_rms"]),
"constant_final_residual_to_clean_state_difference_rms": (
rows["constant"]["final_corrector"]
["residual_to_clean_state_difference_rms"]),
},
"random_beta_boundary": {
"random_sign_fails_with_zero_bias": (
rows["random_beta_zero_bias"]["final_validation_accuracy"] < 0.2),
"fixed_negative_fails_with_zero_bias": (
rows["fixed_negative_zero_bias"]["final_validation_accuracy"] < 0.2),
"interpretation": (
"Negative-beta relaxation is not a valid drop-in baseline under "
"the pinned author's positive-EP hyperparameters. Random-sign "
"beta requires separate stability work before comparison; this "
"screen is not an SDIL win over a functioning random-beta method."
),
},
"test_policy": "new development train holdout data seed 6200",
}
output = RESULT_ROOT.parent / "s3_summary.json"
output.write_text(
json.dumps(report, indent=2, sort_keys=True, allow_nan=False) + "\n")
print(json.dumps(report, indent=2, sort_keys=True, allow_nan=False))
if __name__ == "__main__":
main()
|
