"""Audit validation-only useful-depth screening runs."""
import glob
import json
import os
import statistics


ROOT = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "results")


def main():
    groups = {}
    for path in sorted(glob.glob(os.path.join(ROOT, "depthtask_dev_v1_*.json"))):
        with open(path) as handle:
            row = json.load(handle)
        if row.get("final", {}).get("eval_split") != "validation":
            raise RuntimeError(f"test result in development screen: {path}")
        if row.get("provenance", {}).get("git_dirty") is not False:
            raise RuntimeError(f"dirty or unknown provenance: {path}")
        split = row.get("split", {})
        if not split.get("validation_index_sha256"):
            raise RuntimeError(f"missing synthetic validation hash: {path}")
        args = row["args"]
        key = (args["dataset"], args["mode"], args["width"], args["depth"])
        groups.setdefault(key, []).append(row)

    print("| task | method | width | depth | task/model runs | last val (%) | best val (%) |")
    print("|:---|:---|---:|---:|---:|---:|---:|")
    by_curve = {}
    for key in sorted(groups):
        rows = groups[key]
        last_values = [100 * row["final"]["val_acc"] for row in rows]
        best_values = [100 * max(step["val_acc"] for step in row["steps"]
                                 if "val_acc" in step) for row in rows]
        last_mean = statistics.mean(last_values)
        best_mean = statistics.mean(best_values)
        last_sd = statistics.stdev(last_values) if len(last_values) > 1 else None
        best_sd = statistics.stdev(best_values) if len(best_values) > 1 else None
        last_text = (f"{last_mean:.3f}" if last_sd is None
                     else f"{last_mean:.3f} ± {last_sd:.3f}")
        best_text = (f"{best_mean:.3f}" if best_sd is None
                     else f"{best_mean:.3f} ± {best_sd:.3f}")
        print(f"| {key[0]} | {key[1].upper()} | {key[2]} | {key[3]} | {len(rows)} | "
              f"{last_text} | {best_text} |")
        by_curve.setdefault(key[:3], {})[key[3]] = (last_mean, best_mean)

    print()
    for key, curve in sorted(by_curve.items()):
        if len(curve) < 2:
            continue
        shallow, deep = min(curve), max(curve)
        last_gain = curve[deep][0] - curve[shallow][0]
        best_gain = curve[deep][1] - curve[shallow][1]
        verdict = "PASS" if best_gain >= 5.0 else "FAIL"
        print(f"{key[0]} {key[1].upper()} w{key[2]}: d{shallow}->d{deep} "
              f"last gain {last_gain:+.3f}, best-epoch gain {best_gain:+.3f} points "
              f"[{verdict} C2 BP-screen threshold]")


if __name__ == "__main__":
    main()