diff options
Diffstat (limited to 'experiments/oral_a_confirmation.py')
| -rwxr-xr-x | experiments/oral_a_confirmation.py | 84 |
1 files changed, 84 insertions, 0 deletions
diff --git a/experiments/oral_a_confirmation.py b/experiments/oral_a_confirmation.py new file mode 100755 index 0000000..e8b0dba --- /dev/null +++ b/experiments/oral_a_confirmation.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""Run a deterministic shard of the frozen A4 independent test panel.""" +import argparse +import json +import os +import subprocess +import sys + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--gate", default="results/oral_a_full_gate.json") + parser.add_argument("--bp_selection", default="results/oral_a_bp_selection.json") + parser.add_argument("--short_selection", default="results/oral_a_short_selection.json") + parser.add_argument("--device", default="cuda") + parser.add_argument("--shard_index", type=int, default=0) + parser.add_argument("--num_shards", type=int, default=1) + parser.add_argument("--dry_run", action="store_true") + args = parser.parse_args() + if not 0 <= args.shard_index < args.num_shards: + raise ValueError("invalid shard index") + with open(args.gate) as handle: + gate = json.load(handle) + with open(args.bp_selection) as handle: + bp_selection = json.load(handle) + with open(args.short_selection) as handle: + short = json.load(handle) + if gate["status"] != "passed": + raise ValueError("A3 did not pass; A4 test confirmation is prohibited") + if short["status"] != "selected" or not bp_selection["status"].startswith("passed_"): + raise ValueError("development selections are incomplete") + dfa = short["selected_dfa"] + sdil = short["selected_sdil"] + bp_variant = bp_selection["selected"]["variant"] + jobs = [] + for depth in (20, 32, 56): + for seed in range(10, 15): + common = [ + sys.executable, "experiments/conv_run.py", "--device", args.device, + "--depth", str(depth), "--width", "16", "--seed", str(seed), + "--loader_seed", str(seed), "--perturb_seed", str(1000 + seed), + "--batch_size", "128", "--epochs", "200", "--val_examples", "0", + "--eval_split", "test", "--eval_every", "0", "--augment_train", "1", + "--momentum", "0.9", "--normalization", "batchnorm", + ] + if bp_variant == "primary": + bp_schedule = [ + "--lr", "0.1", "--lr_schedule", "step", + "--lr_milestones", "100,150", "--lr_gamma", "0.1", + "--warmup_epochs", "0", "--weight_decay", "1e-4"] + else: + bp_schedule = [ + "--lr", "0.1", "--lr_schedule", "cosine", + "--warmup_epochs", "5", "--weight_decay", "5e-4"] + jobs.append((f"bp_d{depth}_s{seed}", common + [ + "--mode", "bp", *bp_schedule, + "--out", f"results/oral_a_confirm/bp_d{depth}_s{seed}.json"])) + local_schedule = [ + "--output_lr", "0.1", "--lr_schedule", "step", + "--lr_milestones", "100,150", "--lr_gamma", "0.1", + "--warmup_epochs", "0", "--weight_decay", "1e-4", + "--alignment_probe", "32"] + jobs.append((f"dfa_d{depth}_s{seed}", common + local_schedule + [ + "--mode", "dfa", "--lr", str(dfa["lr"]), "--a_scale", "1", + "--vectorizer_mode", "spatial_template", + "--out", f"results/oral_a_confirm/dfa_d{depth}_s{seed}.json"])) + jobs.append((f"sdil_d{depth}_s{seed}", common + local_schedule + [ + "--mode", "sdil", "--lr", str(sdil["lr"]), + "--vectorizer_mode", sdil["vectorizer_mode"], + "--a_scale", str(sdil["a_scale"]), "--eta_A", str(sdil["eta_A"]), + "--a_warmup_steps", "100", "--pert_sigma", "0.01", + "--pert_directions", "1", "--pert_every", "4", + "--out", f"results/oral_a_confirm/sdil_d{depth}_s{seed}.json"])) + os.makedirs("results/oral_a_confirm", exist_ok=True) + for index, (tag, command) in enumerate(jobs): + if index % args.num_shards != args.shard_index: + continue + print(tag, " ".join(command), flush=True) + if not args.dry_run: + subprocess.run(command, check=True) + + +if __name__ == "__main__": + main() |