1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
|
"""Audit validation-only useful-depth screening runs."""
import glob
import json
import os
import statistics
ROOT = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "results")
def main():
groups = {}
for path in sorted(glob.glob(os.path.join(ROOT, "depthtask_dev_v1_*.json"))):
with open(path) as handle:
row = json.load(handle)
if row.get("final", {}).get("eval_split") != "validation":
raise RuntimeError(f"test result in development screen: {path}")
if row.get("provenance", {}).get("git_dirty") is not False:
raise RuntimeError(f"dirty or unknown provenance: {path}")
split = row.get("split", {})
if not split.get("validation_index_sha256"):
raise RuntimeError(f"missing synthetic validation hash: {path}")
args = row["args"]
key = (args["dataset"], args["mode"], args["width"], args["depth"])
groups.setdefault(key, []).append(row)
print("| task | method | width | depth | task/model runs | last val (%) | best val (%) |")
print("|:---|:---|---:|---:|---:|---:|---:|")
by_curve = {}
for key in sorted(groups):
rows = groups[key]
last_values = [100 * row["final"]["val_acc"] for row in rows]
best_values = [100 * max(step["val_acc"] for step in row["steps"]
if "val_acc" in step) for row in rows]
last_mean = statistics.mean(last_values)
best_mean = statistics.mean(best_values)
last_sd = statistics.stdev(last_values) if len(last_values) > 1 else None
best_sd = statistics.stdev(best_values) if len(best_values) > 1 else None
last_text = (f"{last_mean:.3f}" if last_sd is None
else f"{last_mean:.3f} ± {last_sd:.3f}")
best_text = (f"{best_mean:.3f}" if best_sd is None
else f"{best_mean:.3f} ± {best_sd:.3f}")
print(f"| {key[0]} | {key[1].upper()} | {key[2]} | {key[3]} | {len(rows)} | "
f"{last_text} | {best_text} |")
by_curve.setdefault(key[:3], {})[key[3]] = (last_mean, best_mean)
print()
for key, curve in sorted(by_curve.items()):
if len(curve) < 2:
continue
shallow, deep = min(curve), max(curve)
last_gain = curve[deep][0] - curve[shallow][0]
best_gain = curve[deep][1] - curve[shallow][1]
verdict = "PASS" if best_gain >= 5.0 else "FAIL"
print(f"{key[0]} {key[1].upper()} w{key[2]}: d{shallow}->d{deep} "
f"last gain {last_gain:+.3f}, best-epoch gain {best_gain:+.3f} points "
f"[{verdict} C2 BP-screen threshold]")
if __name__ == "__main__":
main()
|
