experiments: sweep neutral predictor timescales

1 files changed, 52 insertions, 0 deletions

diff --git a/experiments/analyze_traffic_timescale.py b/experiments/analyze_traffic_timescale.py
new file mode 100644
index 0000000..e266d8f
--- /dev/null
+++ b/experiments/analyze_traffic_timescale.py
@@ -0,0 +1,52 @@
+"""Audit predictor-timescale validation development runs."""
+import glob
+import json
+import math
+import os
+import statistics
+
+
+ROOT = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "results")
+
+
+def finite_mean(values):
+    values = [value for value in values if value is not None and math.isfinite(value)]
+    return statistics.mean(values) if values else float("nan")
+
+
+def main():
+    rows = []
+    for path in sorted(glob.glob(os.path.join(ROOT, "traffic_time_dev_v1_*.json"))):
+        with open(path) as handle:
+            row = json.load(handle)
+        if row.get("final", {}).get("eval_split") != "validation":
+            raise RuntimeError(f"non-validation timescale result: {path}")
+        if row.get("provenance", {}).get("git_dirty") is not False:
+            raise RuntimeError(f"dirty or unknown provenance: {path}")
+        if not row.get("split", {}).get("validation_index_sha256"):
+            raise RuntimeError(f"missing validation hash: {path}")
+        rows.append(row)
+
+    print("| scale | eta_P | warmup | n | last val (%) | best val (%) | initial traffic R2 | final traffic R2 |")
+    print("|---:|---:|---:|---:|---:|---:|---:|---:|")
+    groups = {}
+    for row in rows:
+        args = row["args"]
+        key = (args["nuis_rho"], args["eta_P"], args["p_warmup_steps"])
+        groups.setdefault(key, []).append(row)
+
+    for key in sorted(groups):
+        group = groups[key]
+        last = [100 * row["final"]["val_acc"] for row in group]
+        best = [100 * max(step["val_acc"] for step in row["steps"] if "val_acc" in step)
+                for row in group]
+        initial_r2 = [finite_mean(next(step["traffic_r2"] for step in row["steps"]
+                                       if "traffic_r2" in step)) for row in group]
+        final_r2 = [finite_mean(row["final"]["traffic_r2"]) for row in group]
+        print(f"| {key[0]:g} | {key[1]:g} | {key[2]} | {len(group)} | "
+              f"{statistics.mean(last):.3f} | {statistics.mean(best):.3f} | "
+              f"{statistics.mean(initial_r2):.3f} | {statistics.mean(final_r2):.3f} |")
+
+
+if __name__ == "__main__":
+    main()