experiments: predeclare low-query calibration screen

2 files changed, 103 insertions, 0 deletions

diff --git a/experiments/analyze_query_budget.py b/experiments/analyze_query_budget.py
new file mode 100644
index 0000000..7f6344f
--- /dev/null
+++ b/experiments/analyze_query_budget.py
@@ -0,0 +1,63 @@
+"""Audit the validation-only causal-calibration query screen."""
+import glob
+import json
+import math
+import os
+
+
+ROOT = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "results")
+
+
+def main():
+    paths = sorted(glob.glob(os.path.join(ROOT, "query_dev_v1_*.json")))
+    if not paths:
+        raise RuntimeError("no query_dev_v1 results")
+    rows = []
+    commits = set()
+    split_hashes = set()
+    for path in paths:
+        with open(path) as handle:
+            row = json.load(handle)
+        if row.get("final", {}).get("eval_split") != "validation":
+            raise RuntimeError(f"non-validation result: {path}")
+        if row.get("provenance", {}).get("git_dirty") is not False:
+            raise RuntimeError(f"dirty or unknown provenance: {path}")
+        commits.add(row["provenance"]["git_commit"])
+        split_hashes.add(row["split"]["validation_index_sha256"])
+        rows.append(row)
+    if len(commits) != 1 or len(split_hashes) != 1:
+        raise RuntimeError(f"mixed provenance/splits: commits={commits}, splits={split_hashes}")
+
+    dfa = [row for row in rows if row["args"]["mode"] == "dfa"]
+    ref = [row for row in rows if row["args"]["mode"] == "sdil"
+           and row["args"]["pert_ndirs"] == 16 and row["args"]["pert_every"] == 4]
+    if len(dfa) != 1 or len(ref) != 1:
+        raise RuntimeError(f"need exactly one DFA and k16/e4 reference; got {len(dfa)}, {len(ref)}")
+    dfa_acc = dfa[0]["final"]["val_acc"]
+    ref_acc = ref[0]["final"]["val_acc"]
+    ref_gain = ref_acc - dfa_acc
+    ref_queries = ref[0]["cost"]["calibration_batch_loss_evaluations"]
+    ref_work = ref[0]["cost"]["calibration_forward_equivalent_examples"]
+
+    print(f"commit={next(iter(commits))} split={next(iter(split_hashes))}")
+    print("| method | K | every | val (%) | batch loss queries | query reduction | "
+          "forward-eq reduction | gain retention | wall (s) |")
+    print("|:---|---:|---:|---:|---:|---:|---:|---:|---:|")
+    for row in sorted(rows, key=lambda r: (r["args"]["mode"] == "dfa",
+                                           r["args"]["pert_ndirs"],
+                                           r["args"]["pert_every"])):
+        args = row["args"]
+        cost = row["cost"]
+        queries = cost["calibration_batch_loss_evaluations"]
+        work = cost["calibration_forward_equivalent_examples"]
+        q_reduction = ref_queries / queries if queries else math.inf
+        w_reduction = ref_work / work if work else math.inf
+        retention = ((row["final"]["val_acc"] - dfa_acc) / ref_gain
+                     if args["mode"] == "sdil" and ref_gain else float("nan"))
+        print(f"| {args['mode']} | {args['pert_ndirs']} | {args['pert_every']} | "
+              f"{100 * row['final']['val_acc']:.3f} | {queries} | {q_reduction:.1f}x | "
+              f"{w_reduction:.1f}x | {retention:.3f} | {row['final']['wall_s']:.1f} |")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/experiments/query_budget_validation_sweep.sh b/experiments/query_budget_validation_sweep.sh
new file mode 100755
index 0000000..b91386f
--- /dev/null
+++ b/experiments/query_budget_validation_sweep.sh
@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+# Validation-only screen for amortized calibration query budgets.
+# Usage: query_budget_validation_sweep.sh <gpu> "<mode:k:every> ..." [seeds]
+# Example: query_budget_validation_sweep.sh 5 "sdil:16:4 sdil:1:4" 0
+set -eu
+
+cd "$(dirname "$0")/.."
+GPU="${1:?GPU index required}"
+SPECS="${2:?space-separated mode:k:every specs required}"
+SEEDS="${3:-0}"
+PYTHON="${PYTHON:-/home/yurenh2/miniconda3/envs/ep_pascal/bin/python3}"
+
+for seed in $SEEDS; do
+  for spec in $SPECS; do
+    IFS=: read -r mode ndirs every <<EOF
+$spec
+EOF
+    if [ "$mode" = dfa ]; then
+      tag="query_dev_v1_cifar10_d20_dfa_s${seed}"
+    else
+      tag="query_dev_v1_cifar10_d20_k${ndirs}_e${every}_s${seed}"
+    fi
+    out="results/${tag}.json"
+    if [ -s "$out" ]; then
+      echo "[$tag] exists; skipping"
+      continue
+    fi
+    CUDA_VISIBLE_DEVICES="$GPU" "$PYTHON" experiments/run.py \
+      --mode "$mode" --dataset cifar10 --device cuda \
+      --depth 20 --width 64 --residual 1 --act tanh \
+      --epochs 5 --batch_size 128 --eta 0.05 --momentum 0.9 \
+      --eta_A 0.02 --pert_sigma 0.01 --pert_ndirs "$ndirs" \
+      --pert_every "$every" --pert_mode simultaneous \
+      --traffic_mode none --nuis_rho 0 --use_residual 0 \
+      --learn_A 1 --learn_P 0 --p_warmup_steps 0 \
+      --val_examples 5000 --split_seed 2027 --eval_split validation \
+      --diagnostics none --eval_every 0 --seed "$seed" \
+      --log_every 100000 --tag "$tag"
+  done
+done