blob: f1a8e197e2b00603007eec92731799204ff2c909 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
|
"""Audit validation-only useful-depth screening runs."""
import glob
import json
import os
import statistics
ROOT = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "results")
def main():
groups = {}
for path in sorted(glob.glob(os.path.join(ROOT, "depthtask_dev_v1_*.json"))):
with open(path) as handle:
row = json.load(handle)
if row.get("final", {}).get("eval_split") != "validation":
raise RuntimeError(f"test result in development screen: {path}")
if row.get("provenance", {}).get("git_dirty") is not False:
raise RuntimeError(f"dirty or unknown provenance: {path}")
split = row.get("split", {})
if not split.get("validation_index_sha256"):
raise RuntimeError(f"missing synthetic validation hash: {path}")
args = row["args"]
key = (args["dataset"], args["mode"], args["width"], args["depth"])
groups.setdefault(key, []).append(row)
print("| task | method | width | depth | task/model runs | validation acc (%) |")
print("|:---|:---|---:|---:|---:|---:|")
by_curve = {}
for key in sorted(groups):
rows = groups[key]
values = [100 * row["final"]["val_acc"] for row in rows]
mean = statistics.mean(values)
sd = statistics.stdev(values) if len(values) > 1 else None
value = f"{mean:.3f}" if sd is None else f"{mean:.3f} ± {sd:.3f}"
print(f"| {key[0]} | {key[1].upper()} | {key[2]} | {key[3]} | {len(rows)} | {value} |")
by_curve.setdefault(key[:3], {})[key[3]] = mean
print()
for key, curve in sorted(by_curve.items()):
if len(curve) < 2:
continue
shallow, deep = min(curve), max(curve)
gain = curve[deep] - curve[shallow]
verdict = "PASS" if gain >= 5.0 else "FAIL"
print(f"{key[0]} {key[1].upper()} w{key[2]}: d{shallow}->d{deep} "
f"gain {gain:+.3f} points [{verdict} C2 BP-screen threshold]")
if __name__ == "__main__":
main()
|
