diff options
Diffstat (limited to 'experiments')
| -rw-r--r-- | experiments/analyze_query_confirm.py | 96 | ||||
| -rwxr-xr-x | experiments/query_budget_confirm.sh | 38 |
2 files changed, 134 insertions, 0 deletions
diff --git a/experiments/analyze_query_confirm.py b/experiments/analyze_query_confirm.py new file mode 100644 index 0000000..3a3bfd3 --- /dev/null +++ b/experiments/analyze_query_confirm.py @@ -0,0 +1,96 @@ +"""Audit frozen multi-seed confirmation of the low-query protocol.""" +import glob +import json +import math +import os +import statistics + + +ROOT = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "results") + + +def mean_sd(values): + return statistics.mean(values), statistics.stdev(values) if len(values) > 1 else 0.0 + + +def key(row): + args = row["args"] + if args["mode"] == "dfa": + return "DFA" + return f"K{args['pert_ndirs']}/e{args['pert_every']}" + + +def main(): + paths = sorted(glob.glob(os.path.join(ROOT, "query_confirm_v1_*.json"))) + rows = [] + commits = set() + for path in paths: + with open(path) as handle: + row = json.load(handle) + if row.get("final", {}).get("eval_split") != "test": + raise RuntimeError(f"non-test confirmation result: {path}") + if row.get("provenance", {}).get("git_dirty") is not False: + raise RuntimeError(f"dirty or unknown provenance: {path}") + if "hardware" not in row or "training_forward_equivalent_examples" not in row.get("cost", {}): + raise RuntimeError(f"missing cost/memory audit: {path}") + commits.add(row["provenance"]["git_commit"]) + rows.append(row) + if len(commits) != 1: + raise RuntimeError(f"mixed commits: {commits}") + + groups = {} + for row in rows: + groups.setdefault(key(row), []).append(row) + expected = {"DFA", "K16/e4", "K1/e4"} + if set(groups) != expected or any(len(group) != 5 for group in groups.values()): + raise RuntimeError(f"need five rows for {expected}; got " + f"{ {name: len(group) for name, group in groups.items()} }") + for group in groups.values(): + group.sort(key=lambda row: row["args"]["seed"]) + seeds = [[row["args"]["seed"] for row in groups[name]] for name in sorted(groups)] + if len({tuple(value) for value in seeds}) != 1: + raise RuntimeError(f"unpaired seeds: {seeds}") + + print(f"commit={next(iter(commits))} seeds={seeds[0]}") + print("| method | n | test (%) | batch queries | train forward-eq examples | " + "peak allocated (MiB) | wall (s) |") + print("|:---|---:|---:|---:|---:|---:|---:|") + for name in ("DFA", "K16/e4", "K1/e4"): + group = groups[name] + acc = mean_sd([100 * row["final"]["test_acc"] for row in group]) + queries = mean_sd([row["cost"]["calibration_batch_loss_evaluations"] for row in group]) + work = mean_sd([row["cost"]["training_forward_equivalent_examples"] for row in group]) + memory = mean_sd([row["hardware"]["peak_memory_allocated_bytes"] / 2**20 for row in group]) + wall = mean_sd([row["final"]["wall_s"] for row in group]) + print(f"| {name} | {len(group)} | {acc[0]:.3f} +/- {acc[1]:.3f} | " + f"{queries[0]:.0f} | {work[0]:.0f} | {memory[0]:.1f} +/- {memory[1]:.1f} | " + f"{wall[0]:.1f} +/- {wall[1]:.1f} |") + + dfa = [row["final"]["test_acc"] for row in groups["DFA"]] + ref = [row["final"]["test_acc"] for row in groups["K16/e4"]] + low = [row["final"]["test_acc"] for row in groups["K1/e4"]] + ref_gains = [a - b for a, b in zip(ref, dfa)] + low_gains = [a - b for a, b in zip(low, dfa)] + retention = statistics.mean(low_gains) / statistics.mean(ref_gains) + query_reduction = (groups["K16/e4"][0]["cost"]["calibration_batch_loss_evaluations"] + / groups["K1/e4"][0]["cost"]["calibration_batch_loss_evaluations"]) + calibration_work_reduction = ( + groups["K16/e4"][0]["cost"]["calibration_forward_equivalent_examples"] + / groups["K1/e4"][0]["cost"]["calibration_forward_equivalent_examples"]) + total_work_reduction = ( + groups["K16/e4"][0]["cost"]["training_forward_equivalent_examples"] + / groups["K1/e4"][0]["cost"]["training_forward_equivalent_examples"]) + print(f"paired reference gain: {100 * statistics.mean(ref_gains):.3f} points") + print(f"paired low-query gain: {100 * statistics.mean(low_gains):.3f} points") + print(f"gain retention: {retention:.3f}") + print(f"query reduction: {query_reduction:.1f}x") + print(f"calibration/total forward-eq reduction: " + f"{calibration_work_reduction:.1f}x / {total_work_reduction:.1f}x") + passed = query_reduction >= 10 and retention >= 0.9 + print(f"C3 confirmation gate: {'PASS' if passed else 'FAIL'}") + if not passed: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/experiments/query_budget_confirm.sh b/experiments/query_budget_confirm.sh new file mode 100755 index 0000000..a36dc16 --- /dev/null +++ b/experiments/query_budget_confirm.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Frozen five-seed test confirmation of the validation-selected K=1/every=4 protocol. +# Usage: query_budget_confirm.sh <gpu> "<seeds>" +set -eu + +cd "$(dirname "$0")/.." +GPU="${1:?GPU index required}" +SEEDS="${2:-0 1 2 3 4}" +PYTHON="${PYTHON:-/home/yurenh2/miniconda3/envs/ep_pascal/bin/python3}" + +for seed in $SEEDS; do + for spec in dfa:1 sdil:16 sdil:1; do + IFS=: read -r mode ndirs <<EOF +$spec +EOF + if [ "$mode" = dfa ]; then + tag="query_confirm_v1_cifar10_d20_dfa_s${seed}" + else + tag="query_confirm_v1_cifar10_d20_k${ndirs}_e4_s${seed}" + fi + out="results/${tag}.json" + if [ -s "$out" ]; then + echo "[$tag] exists; skipping" + continue + fi + CUDA_VISIBLE_DEVICES="$GPU" "$PYTHON" experiments/run.py \ + --mode "$mode" --dataset cifar10 --device cuda \ + --depth 20 --width 64 --residual 1 --act tanh \ + --epochs 5 --batch_size 128 --eta 0.05 --momentum 0.9 \ + --eta_A 0.02 --pert_sigma 0.01 --pert_ndirs "$ndirs" \ + --pert_every 4 --pert_mode simultaneous \ + --traffic_mode none --nuis_rho 0 --use_residual 0 \ + --learn_A 1 --learn_P 0 --p_warmup_steps 0 \ + --val_examples 0 --eval_split test \ + --diagnostics none --eval_every 0 --seed "$seed" \ + --log_every 100000 --tag "$tag" + done +done |