diff options
| -rw-r--r-- | ROADMAP.md | 5 | ||||
| -rw-r--r-- | experiments/analyze_query_budget.py | 63 | ||||
| -rwxr-xr-x | experiments/query_budget_validation_sweep.sh | 40 |
3 files changed, 108 insertions, 0 deletions
@@ -62,6 +62,11 @@ fixed epoch and fixed query/FLOP budgets. 16-directions-every-4-steps setting must preserve at least 90% of its gain over DFA. Pareto claims must hold in a hardware-independent cost coordinate as well as GTX-1080 wall time. +The validation screen is frozen to CIFAR-10 d20/w64, five epochs, seed 0, and compares the current +`K=16/every=4` reference with `K/every` pairs `1/4`, `2/8`, `4/16`, `1/8`, `1/16`, and `2/32`, plus +DFA. Selection uses final validation accuracy and logical batch loss queries; the test split is not +evaluated. The chosen low-query protocol then receives a multi-seed frozen confirmation. + ### C4. The comparison set contains the closest alternatives The main comparison must include exact BP, FA, DFA, direct/unamortized node perturbation, learned diff --git a/experiments/analyze_query_budget.py b/experiments/analyze_query_budget.py new file mode 100644 index 0000000..7f6344f --- /dev/null +++ b/experiments/analyze_query_budget.py @@ -0,0 +1,63 @@ +"""Audit the validation-only causal-calibration query screen.""" +import glob +import json +import math +import os + + +ROOT = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "results") + + +def main(): + paths = sorted(glob.glob(os.path.join(ROOT, "query_dev_v1_*.json"))) + if not paths: + raise RuntimeError("no query_dev_v1 results") + rows = [] + commits = set() + split_hashes = set() + for path in paths: + with open(path) as handle: + row = json.load(handle) + if row.get("final", {}).get("eval_split") != "validation": + raise RuntimeError(f"non-validation result: {path}") + if row.get("provenance", {}).get("git_dirty") is not False: + raise RuntimeError(f"dirty or unknown provenance: {path}") + commits.add(row["provenance"]["git_commit"]) + split_hashes.add(row["split"]["validation_index_sha256"]) + rows.append(row) + if len(commits) != 1 or len(split_hashes) != 1: + raise RuntimeError(f"mixed provenance/splits: commits={commits}, splits={split_hashes}") + + dfa = [row for row in rows if row["args"]["mode"] == "dfa"] + ref = [row for row in rows if row["args"]["mode"] == "sdil" + and row["args"]["pert_ndirs"] == 16 and row["args"]["pert_every"] == 4] + if len(dfa) != 1 or len(ref) != 1: + raise RuntimeError(f"need exactly one DFA and k16/e4 reference; got {len(dfa)}, {len(ref)}") + dfa_acc = dfa[0]["final"]["val_acc"] + ref_acc = ref[0]["final"]["val_acc"] + ref_gain = ref_acc - dfa_acc + ref_queries = ref[0]["cost"]["calibration_batch_loss_evaluations"] + ref_work = ref[0]["cost"]["calibration_forward_equivalent_examples"] + + print(f"commit={next(iter(commits))} split={next(iter(split_hashes))}") + print("| method | K | every | val (%) | batch loss queries | query reduction | " + "forward-eq reduction | gain retention | wall (s) |") + print("|:---|---:|---:|---:|---:|---:|---:|---:|---:|") + for row in sorted(rows, key=lambda r: (r["args"]["mode"] == "dfa", + r["args"]["pert_ndirs"], + r["args"]["pert_every"])): + args = row["args"] + cost = row["cost"] + queries = cost["calibration_batch_loss_evaluations"] + work = cost["calibration_forward_equivalent_examples"] + q_reduction = ref_queries / queries if queries else math.inf + w_reduction = ref_work / work if work else math.inf + retention = ((row["final"]["val_acc"] - dfa_acc) / ref_gain + if args["mode"] == "sdil" and ref_gain else float("nan")) + print(f"| {args['mode']} | {args['pert_ndirs']} | {args['pert_every']} | " + f"{100 * row['final']['val_acc']:.3f} | {queries} | {q_reduction:.1f}x | " + f"{w_reduction:.1f}x | {retention:.3f} | {row['final']['wall_s']:.1f} |") + + +if __name__ == "__main__": + main() diff --git a/experiments/query_budget_validation_sweep.sh b/experiments/query_budget_validation_sweep.sh new file mode 100755 index 0000000..b91386f --- /dev/null +++ b/experiments/query_budget_validation_sweep.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Validation-only screen for amortized calibration query budgets. +# Usage: query_budget_validation_sweep.sh <gpu> "<mode:k:every> ..." [seeds] +# Example: query_budget_validation_sweep.sh 5 "sdil:16:4 sdil:1:4" 0 +set -eu + +cd "$(dirname "$0")/.." +GPU="${1:?GPU index required}" +SPECS="${2:?space-separated mode:k:every specs required}" +SEEDS="${3:-0}" +PYTHON="${PYTHON:-/home/yurenh2/miniconda3/envs/ep_pascal/bin/python3}" + +for seed in $SEEDS; do + for spec in $SPECS; do + IFS=: read -r mode ndirs every <<EOF +$spec +EOF + if [ "$mode" = dfa ]; then + tag="query_dev_v1_cifar10_d20_dfa_s${seed}" + else + tag="query_dev_v1_cifar10_d20_k${ndirs}_e${every}_s${seed}" + fi + out="results/${tag}.json" + if [ -s "$out" ]; then + echo "[$tag] exists; skipping" + continue + fi + CUDA_VISIBLE_DEVICES="$GPU" "$PYTHON" experiments/run.py \ + --mode "$mode" --dataset cifar10 --device cuda \ + --depth 20 --width 64 --residual 1 --act tanh \ + --epochs 5 --batch_size 128 --eta 0.05 --momentum 0.9 \ + --eta_A 0.02 --pert_sigma 0.01 --pert_ndirs "$ndirs" \ + --pert_every "$every" --pert_mode simultaneous \ + --traffic_mode none --nuis_rho 0 --use_residual 0 \ + --learn_A 1 --learn_P 0 --p_warmup_steps 0 \ + --val_examples 5000 --split_seed 2027 --eval_split validation \ + --diagnostics none --eval_every 0 --seed "$seed" \ + --log_every 100000 --tag "$tag" + done +done |