experiments: add useful-depth validation screen

1 files changed, 51 insertions, 0 deletions

diff --git a/experiments/analyze_depth_tasks.py b/experiments/analyze_depth_tasks.py
new file mode 100644
index 0000000..f1a8e19
--- /dev/null
+++ b/experiments/analyze_depth_tasks.py
@@ -0,0 +1,51 @@
+"""Audit validation-only useful-depth screening runs."""
+import glob
+import json
+import os
+import statistics
+
+
+ROOT = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "results")
+
+
+def main():
+    groups = {}
+    for path in sorted(glob.glob(os.path.join(ROOT, "depthtask_dev_v1_*.json"))):
+        with open(path) as handle:
+            row = json.load(handle)
+        if row.get("final", {}).get("eval_split") != "validation":
+            raise RuntimeError(f"test result in development screen: {path}")
+        if row.get("provenance", {}).get("git_dirty") is not False:
+            raise RuntimeError(f"dirty or unknown provenance: {path}")
+        split = row.get("split", {})
+        if not split.get("validation_index_sha256"):
+            raise RuntimeError(f"missing synthetic validation hash: {path}")
+        args = row["args"]
+        key = (args["dataset"], args["mode"], args["width"], args["depth"])
+        groups.setdefault(key, []).append(row)
+
+    print("| task | method | width | depth | task/model runs | validation acc (%) |")
+    print("|:---|:---|---:|---:|---:|---:|")
+    by_curve = {}
+    for key in sorted(groups):
+        rows = groups[key]
+        values = [100 * row["final"]["val_acc"] for row in rows]
+        mean = statistics.mean(values)
+        sd = statistics.stdev(values) if len(values) > 1 else None
+        value = f"{mean:.3f}" if sd is None else f"{mean:.3f} ± {sd:.3f}"
+        print(f"| {key[0]} | {key[1].upper()} | {key[2]} | {key[3]} | {len(rows)} | {value} |")
+        by_curve.setdefault(key[:3], {})[key[3]] = mean
+
+    print()
+    for key, curve in sorted(by_curve.items()):
+        if len(curve) < 2:
+            continue
+        shallow, deep = min(curve), max(curve)
+        gain = curve[deep] - curve[shallow]
+        verdict = "PASS" if gain >= 5.0 else "FAIL"
+        print(f"{key[0]} {key[1].upper()} w{key[2]}: d{shallow}->d{deep} "
+              f"gain {gain:+.3f} points [{verdict} C2 BP-screen threshold]")
+
+
+if __name__ == "__main__":
+    main()