"""Audit validation-only useful-depth screening runs."""
import glob
import json
import os
import statistics


ROOT = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "results")


def main():
    groups = {}
    for path in sorted(glob.glob(os.path.join(ROOT, "depthtask_dev_v1_*.json"))):
        with open(path) as handle:
            row = json.load(handle)
        if row.get("final", {}).get("eval_split") != "validation":
            raise RuntimeError(f"test result in development screen: {path}")
        if row.get("provenance", {}).get("git_dirty") is not False:
            raise RuntimeError(f"dirty or unknown provenance: {path}")
        split = row.get("split", {})
        if not split.get("validation_index_sha256"):
            raise RuntimeError(f"missing synthetic validation hash: {path}")
        args = row["args"]
        key = (args["dataset"], args["mode"], args["width"], args["depth"])
        groups.setdefault(key, []).append(row)

    print("| task | method | width | depth | task/model runs | validation acc (%) |")
    print("|:---|:---|---:|---:|---:|---:|")
    by_curve = {}
    for key in sorted(groups):
        rows = groups[key]
        values = [100 * row["final"]["val_acc"] for row in rows]
        mean = statistics.mean(values)
        sd = statistics.stdev(values) if len(values) > 1 else None
        value = f"{mean:.3f}" if sd is None else f"{mean:.3f} ± {sd:.3f}"
        print(f"| {key[0]} | {key[1].upper()} | {key[2]} | {key[3]} | {len(rows)} | {value} |")
        by_curve.setdefault(key[:3], {})[key[3]] = mean

    print()
    for key, curve in sorted(by_curve.items()):
        if len(curve) < 2:
            continue
        shallow, deep = min(curve), max(curve)
        gain = curve[deep] - curve[shallow]
        verdict = "PASS" if gain >= 5.0 else "FAIL"
        print(f"{key[0]} {key[1].upper()} w{key[2]}: d{shallow}->d{deep} "
              f"gain {gain:+.3f} points [{verdict} C2 BP-screen threshold]")


if __name__ == "__main__":
    main()