From 66899949bcbb11664a63669c2d27bd4c12284955 Mon Sep 17 00:00:00 2001 From: YurenHao0426 Date: Wed, 22 Jul 2026 02:21:01 -0500 Subject: experiments: freeze five-seed low-query confirmation --- experiments/analyze_query_confirm.py | 96 ++++++++++++++++++++++++++++++++++++ experiments/query_budget_confirm.sh | 38 ++++++++++++++ 2 files changed, 134 insertions(+) create mode 100644 experiments/analyze_query_confirm.py create mode 100755 experiments/query_budget_confirm.sh (limited to 'experiments') diff --git a/experiments/analyze_query_confirm.py b/experiments/analyze_query_confirm.py new file mode 100644 index 0000000..3a3bfd3 --- /dev/null +++ b/experiments/analyze_query_confirm.py @@ -0,0 +1,96 @@ +"""Audit frozen multi-seed confirmation of the low-query protocol.""" +import glob +import json +import math +import os +import statistics + + +ROOT = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "results") + + +def mean_sd(values): + return statistics.mean(values), statistics.stdev(values) if len(values) > 1 else 0.0 + + +def key(row): + args = row["args"] + if args["mode"] == "dfa": + return "DFA" + return f"K{args['pert_ndirs']}/e{args['pert_every']}" + + +def main(): + paths = sorted(glob.glob(os.path.join(ROOT, "query_confirm_v1_*.json"))) + rows = [] + commits = set() + for path in paths: + with open(path) as handle: + row = json.load(handle) + if row.get("final", {}).get("eval_split") != "test": + raise RuntimeError(f"non-test confirmation result: {path}") + if row.get("provenance", {}).get("git_dirty") is not False: + raise RuntimeError(f"dirty or unknown provenance: {path}") + if "hardware" not in row or "training_forward_equivalent_examples" not in row.get("cost", {}): + raise RuntimeError(f"missing cost/memory audit: {path}") + commits.add(row["provenance"]["git_commit"]) + rows.append(row) + if len(commits) != 1: + raise RuntimeError(f"mixed commits: {commits}") + + groups = {} + for row in rows: + groups.setdefault(key(row), []).append(row) + expected = {"DFA", "K16/e4", "K1/e4"} + if set(groups) != expected or any(len(group) != 5 for group in groups.values()): + raise RuntimeError(f"need five rows for {expected}; got " + f"{ {name: len(group) for name, group in groups.items()} }") + for group in groups.values(): + group.sort(key=lambda row: row["args"]["seed"]) + seeds = [[row["args"]["seed"] for row in groups[name]] for name in sorted(groups)] + if len({tuple(value) for value in seeds}) != 1: + raise RuntimeError(f"unpaired seeds: {seeds}") + + print(f"commit={next(iter(commits))} seeds={seeds[0]}") + print("| method | n | test (%) | batch queries | train forward-eq examples | " + "peak allocated (MiB) | wall (s) |") + print("|:---|---:|---:|---:|---:|---:|---:|") + for name in ("DFA", "K16/e4", "K1/e4"): + group = groups[name] + acc = mean_sd([100 * row["final"]["test_acc"] for row in group]) + queries = mean_sd([row["cost"]["calibration_batch_loss_evaluations"] for row in group]) + work = mean_sd([row["cost"]["training_forward_equivalent_examples"] for row in group]) + memory = mean_sd([row["hardware"]["peak_memory_allocated_bytes"] / 2**20 for row in group]) + wall = mean_sd([row["final"]["wall_s"] for row in group]) + print(f"| {name} | {len(group)} | {acc[0]:.3f} +/- {acc[1]:.3f} | " + f"{queries[0]:.0f} | {work[0]:.0f} | {memory[0]:.1f} +/- {memory[1]:.1f} | " + f"{wall[0]:.1f} +/- {wall[1]:.1f} |") + + dfa = [row["final"]["test_acc"] for row in groups["DFA"]] + ref = [row["final"]["test_acc"] for row in groups["K16/e4"]] + low = [row["final"]["test_acc"] for row in groups["K1/e4"]] + ref_gains = [a - b for a, b in zip(ref, dfa)] + low_gains = [a - b for a, b in zip(low, dfa)] + retention = statistics.mean(low_gains) / statistics.mean(ref_gains) + query_reduction = (groups["K16/e4"][0]["cost"]["calibration_batch_loss_evaluations"] + / groups["K1/e4"][0]["cost"]["calibration_batch_loss_evaluations"]) + calibration_work_reduction = ( + groups["K16/e4"][0]["cost"]["calibration_forward_equivalent_examples"] + / groups["K1/e4"][0]["cost"]["calibration_forward_equivalent_examples"]) + total_work_reduction = ( + groups["K16/e4"][0]["cost"]["training_forward_equivalent_examples"] + / groups["K1/e4"][0]["cost"]["training_forward_equivalent_examples"]) + print(f"paired reference gain: {100 * statistics.mean(ref_gains):.3f} points") + print(f"paired low-query gain: {100 * statistics.mean(low_gains):.3f} points") + print(f"gain retention: {retention:.3f}") + print(f"query reduction: {query_reduction:.1f}x") + print(f"calibration/total forward-eq reduction: " + f"{calibration_work_reduction:.1f}x / {total_work_reduction:.1f}x") + passed = query_reduction >= 10 and retention >= 0.9 + print(f"C3 confirmation gate: {'PASS' if passed else 'FAIL'}") + if not passed: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/experiments/query_budget_confirm.sh b/experiments/query_budget_confirm.sh new file mode 100755 index 0000000..a36dc16 --- /dev/null +++ b/experiments/query_budget_confirm.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Frozen five-seed test confirmation of the validation-selected K=1/every=4 protocol. +# Usage: query_budget_confirm.sh "" +set -eu + +cd "$(dirname "$0")/.." +GPU="${1:?GPU index required}" +SEEDS="${2:-0 1 2 3 4}" +PYTHON="${PYTHON:-/home/yurenh2/miniconda3/envs/ep_pascal/bin/python3}" + +for seed in $SEEDS; do + for spec in dfa:1 sdil:16 sdil:1; do + IFS=: read -r mode ndirs <