diff options
Diffstat (limited to 'experiments')
| -rwxr-xr-x | experiments/analyze_oral_a_apical.py | 100 | ||||
| -rwxr-xr-x | experiments/analyze_oral_a_bp.py | 72 | ||||
| -rwxr-xr-x | experiments/analyze_oral_a_short.py | 84 | ||||
| -rw-r--r-- | experiments/conv_run.py | 16 | ||||
| -rwxr-xr-x | experiments/oral_a_apical_screen.sh | 25 | ||||
| -rwxr-xr-x | experiments/oral_a_bp_reference.sh | 31 | ||||
| -rwxr-xr-x | experiments/oral_a_short_screen.py | 65 |
7 files changed, 391 insertions, 2 deletions
diff --git a/experiments/analyze_oral_a_apical.py b/experiments/analyze_oral_a_apical.py new file mode 100755 index 0000000..1d10edb --- /dev/null +++ b/experiments/analyze_oral_a_apical.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +"""Apply the frozen A2a selector to convolutional apical-screen records.""" +import argparse +import glob +import json +import math +import os + + +MODES = ("spatial_template", "channel_gated") +EXPECTED_SCALES = (0.0, 1.0) +EXPECTED_RATES = (0.001, 0.01, 0.1) + + +def load(path): + with open(path) as handle: + record = json.load(handle) + args = record["args"] + if record["provenance"]["git_tracked_dirty"]: + raise ValueError(f"tracked-dirty result: {path}") + expected = { + "mode": "sdil", "depth": 20, "width": 16, "seed": 0, + "epochs": 0, "train_limit": 10000, "val_examples": 5000, + "a_warmup_steps": 100, "pert_directions": 1, "pert_every": 4, + "normalization": "batchnorm", + } + for key, value in expected.items(): + if args.get(key) != value: + raise ValueError(f"{path}: {key}={args.get(key)!r}, expected {value!r}") + if record["split"]["validation_index_sha256"] != ( + "8328b206a97c420e49e54e3eca4abe3274c4756b084355784ea3fb8059e4515b"): + raise ValueError(f"split drift: {path}") + diagnostics = record.get("diagnostics") + warmup = record.get("apical_warmup", {}).get("last") + if diagnostics is None or warmup is None: + raise ValueError(f"missing diagnostics/warmup: {path}") + values = diagnostics["teaching_negative_gradient_cosine"] + early_count = max(1, len(values) // 3) + metrics = { + "early_third_alignment": sum(values[:early_count]) / early_count, + "all_layer_alignment": sum(values) / len(values), + "calibration_mse": warmup["calibration_mse"], + "prediction_target_cosine": warmup["prediction_target_cosine"], + } + finite = (record["final"]["finite"] + and all(math.isfinite(value) for value in metrics.values())) + return { + "path": path, + "source_commit": record["provenance"]["git_commit"], + "vectorizer_mode": args["vectorizer_mode"], + "a_scale": float(args["a_scale"]), + "eta_A": float(args["eta_A"]), + "metrics": metrics, + "eligible": finite and metrics["early_third_alignment"] > 0.0, + "vectorizer_parameters": record["architecture"]["vectorizer_parameters"], + } + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--input", default="results/oral_a_apical_screen") + parser.add_argument("--out", default="results/oral_a_apical_selection.json") + args = parser.parse_args() + paths = sorted(glob.glob(os.path.join(args.input, "*.json"))) + rows = [load(path) for path in paths] + observed = {(row["vectorizer_mode"], row["a_scale"], row["eta_A"]) + for row in rows} + expected = {(mode, scale, rate) for mode in MODES + for scale in EXPECTED_SCALES for rate in EXPECTED_RATES} + if observed != expected or len(rows) != len(expected): + raise ValueError(f"incomplete A2a grid: missing={expected-observed}, extra={observed-expected}") + if len({row["source_commit"] for row in rows}) != 1: + raise ValueError("A2a source commits differ") + selected = {} + for mode in MODES: + eligible = [row for row in rows + if row["vectorizer_mode"] == mode and row["eligible"]] + if eligible: + eligible.sort(key=lambda row: ( + -row["metrics"]["early_third_alignment"], + -row["metrics"]["all_layer_alignment"], + row["eta_A"], row["a_scale"])) + selected[mode] = eligible[0] + output = { + "protocol": "oral_a_A2a_v1", + "status": ("selected" if len(selected) == len(MODES) + else "failed_no_eligible_family"), + "rows": rows, + "selected": selected, + "confirmation_test_seeds_touched": False, + } + os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) + with open(args.out, "w") as handle: + json.dump(output, handle, indent=2, sort_keys=True) + handle.write("\n") + print(json.dumps({"status": output["status"], "selected": selected}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/experiments/analyze_oral_a_bp.py b/experiments/analyze_oral_a_bp.py new file mode 100755 index 0000000..d309273 --- /dev/null +++ b/experiments/analyze_oral_a_bp.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""Apply the frozen A1 exact-BP viability gate.""" +import argparse +import json +import math +import os + + +def read(path, variant): + with open(path) as handle: + record = json.load(handle) + args = record["args"] + expected = { + "mode": "bp", "depth": 20, "width": 16, "seed": 0, + "epochs": 200, "val_examples": 5000, "eval_split": "validation", + "normalization": "batchnorm", + } + for key, value in expected.items(): + if args.get(key) != value: + raise ValueError(f"{path}: {key}={args.get(key)!r}, expected {value!r}") + if record["provenance"]["git_tracked_dirty"]: + raise ValueError(f"tracked-dirty result: {path}") + if record["split"]["validation_index_sha256"] != ( + "8328b206a97c420e49e54e3eca4abe3274c4756b084355784ea3fb8059e4515b"): + raise ValueError(f"split drift: {path}") + accuracy = float(record["final"]["accuracy"]) + loss = float(record["final"]["loss"]) + return { + "variant": variant, "path": path, + "source_commit": record["provenance"]["git_commit"], + "accuracy": accuracy, "loss": loss, + "finite": record["final"]["finite"] and math.isfinite(accuracy + loss), + "test_evaluations": record["evaluation_protocol"]["test_evaluations"], + } + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--primary", default="results/oral_a_dev/bp_reference_primary.json") + parser.add_argument("--recovery", default="results/oral_a_dev/bp_reference_recovery.json") + parser.add_argument("--out", default="results/oral_a_bp_selection.json") + args = parser.parse_args() + primary = read(args.primary, "primary") + rows = [primary] + if primary["finite"] and primary["accuracy"] >= 0.90: + status = "passed_primary" + selected = primary + elif not os.path.exists(args.recovery): + status = "recovery_required" + selected = None + else: + recovery = read(args.recovery, "recovery") + rows.append(recovery) + selected = max(rows, key=lambda row: row["accuracy"] if row["finite"] else -1.0) + status = ("passed_recovery" if selected["finite"] and selected["accuracy"] >= 0.90 + else "failed_no_viable_reference") + if any(row["test_evaluations"] for row in rows): + raise ValueError("A1 must not evaluate test") + output = { + "protocol": "oral_a_A1_v1", "status": status, + "rows": rows, "selected": selected, + "confirmation_test_seeds_touched": False, + } + os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) + with open(args.out, "w") as handle: + json.dump(output, handle, indent=2, sort_keys=True) + handle.write("\n") + print(json.dumps(output, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/experiments/analyze_oral_a_short.py b/experiments/analyze_oral_a_short.py new file mode 100755 index 0000000..375718a --- /dev/null +++ b/experiments/analyze_oral_a_short.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""Apply the frozen A2b selector and advancement gate.""" +import argparse +import glob +import json +import math +import os + + +def read(path): + with open(path) as handle: + record = json.load(handle) + args = record["args"] + expected = { + "depth": 20, "width": 16, "seed": 0, "epochs": 20, + "train_limit": 10000, "val_examples": 5000, + "eval_split": "validation", "normalization": "batchnorm", + } + for key, value in expected.items(): + if args.get(key) != value: + raise ValueError(f"{path}: {key} drift") + if record["provenance"]["git_tracked_dirty"]: + raise ValueError(f"tracked-dirty result: {path}") + if record["evaluation_protocol"]["test_evaluations"]: + raise ValueError(f"short screen touched test: {path}") + accuracy = float(record["final"]["accuracy"]) + loss = float(record["final"]["loss"]) + return { + "path": path, "mode": args["mode"], "lr": float(args["lr"]), + "vectorizer_mode": args.get("vectorizer_mode"), + "a_scale": float(args.get("a_scale", 0.0)), + "eta_A": float(args.get("eta_A", 0.0)), + "accuracy": accuracy, "loss": loss, + "finite": record["final"]["finite"] and math.isfinite(accuracy + loss), + "total_macs": record["work"]["total_macs_estimate"], + "source_commit": record["provenance"]["git_commit"], + } + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--input", default="results/oral_a_short") + parser.add_argument("--out", default="results/oral_a_short_selection.json") + args = parser.parse_args() + rows = [read(path) for path in sorted(glob.glob(os.path.join(args.input, "*.json")))] + if len(rows) != 10: + raise ValueError(f"expected 10 A2b runs, found {len(rows)}") + if len({row["source_commit"] for row in rows}) != 1: + raise ValueError("A2b source commits differ") + bp = [row for row in rows if row["mode"] == "bp"] + dfa = [row for row in rows if row["mode"] == "dfa"] + sdil = [row for row in rows if row["mode"] == "sdil"] + if len(bp) != 1 or len(dfa) != 3 or len(sdil) != 6: + raise ValueError("A2b method grid is incomplete") + selected_dfa = sorted( + dfa, key=lambda row: (-row["accuracy"], row["total_macs"], row["lr"]))[0] + best_accuracy = max(row["accuracy"] for row in sdil if row["finite"]) + gated_near_best = [row for row in sdil if row["finite"] + and row["vectorizer_mode"] == "channel_gated" + and row["accuracy"] >= best_accuracy - 0.005] + pool = gated_near_best or [row for row in sdil if row["finite"]] + selected_sdil = sorted( + pool, key=lambda row: (-row["accuracy"], row["total_macs"], row["lr"]))[0] + passed = (bp[0]["finite"] and selected_dfa["finite"] and selected_sdil["finite"] + and bp[0]["accuracy"] >= 0.50 + and selected_sdil["accuracy"] >= 0.35 + and selected_sdil["accuracy"] >= selected_dfa["accuracy"] - 0.10) + output = { + "protocol": "oral_a_A2b_v1", + "status": "selected" if passed else "failed_advancement_gate", + "rows": rows, "selected_bp": bp[0], + "selected_dfa": selected_dfa, "selected_sdil": selected_sdil, + "confirmation_test_seeds_touched": False, + } + os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) + with open(args.out, "w") as handle: + json.dump(output, handle, indent=2, sort_keys=True) + handle.write("\n") + print(json.dumps({key: output[key] for key in ( + "status", "selected_bp", "selected_dfa", "selected_sdil")}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/experiments/conv_run.py b/experiments/conv_run.py index 9b41dd2..98c1f19 100644 --- a/experiments/conv_run.py +++ b/experiments/conv_run.py @@ -235,8 +235,14 @@ def run(args): "epochs": [], } + sync(args.device) + total_start = time.time() + predictor_warmup_wall = 0.0 + apical_warmup_wall = 0.0 loader_state = train.g.get_state().clone() if config is not None and config.learn_P and args.predictor_warmup_steps: + sync(args.device) + warmup_start = time.time() iterator = iter(train) for _ in range(args.predictor_warmup_steps): try: @@ -249,8 +255,12 @@ def run(args): forward["hiddens"], config.eta_P, config.nuisance_scale) counters["predictor_warmup_examples"] += x.shape[0] train.g.set_state(loader_state) + sync(args.device) + predictor_warmup_wall = time.time() - warmup_start if config is not None and args.a_warmup_steps: + sync(args.device) + warmup_start = time.time() iterator = iter(train) warmup_metrics = [] for _ in range(args.a_warmup_steps): @@ -277,14 +287,14 @@ def run(args): "steps": args.a_warmup_steps, "last": warmup_metrics[-1], } + sync(args.device) + apical_warmup_wall = time.time() - warmup_start step = 0 train_wall = 0.0 eval_wall = 0.0 validation_evaluations = 0 test_evaluations = 0 - sync(args.device) - total_start = time.time() for epoch in range(args.epochs): lr = scheduled_lr(args.lr, epoch, args) output_lr = (scheduled_lr(args.output_lr, epoch, args) @@ -393,6 +403,8 @@ def run(args): "work": work_report(net, args.mode, counters), "hardware": training_hardware, "timing": { + "predictor_warmup_wall_s": predictor_warmup_wall, + "apical_warmup_wall_s": apical_warmup_wall, "train_wall_s": train_wall, "evaluation_wall_s": eval_wall, "total_timed_wall_s": total_wall, diff --git a/experiments/oral_a_apical_screen.sh b/experiments/oral_a_apical_screen.sh new file mode 100755 index 0000000..7281def --- /dev/null +++ b/experiments/oral_a_apical_screen.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -euo pipefail + +device="${1:-cuda}" +mkdir -p results/oral_a_apical_screen + +for vectorizer in spatial_template channel_gated; do + for a_scale in 0 1; do + for eta_a in 0.001 0.01 0.1; do + tag="${vectorizer}_a${a_scale}_etaA${eta_a}" + python3 experiments/conv_run.py \ + --mode sdil --device "${device}" --depth 20 --width 16 \ + --seed 0 --loader_seed 0 --batch_size 128 --epochs 0 \ + --train_limit 10000 --val_examples 5000 --split_seed 2027 \ + --eval_split validation --eval_every 0 --augment_train 1 \ + --lr 0.03 --output_lr 0.1 --lr_schedule constant --warmup_epochs 0 \ + --momentum 0.9 --weight_decay 1e-4 --normalization batchnorm \ + --vectorizer_mode "${vectorizer}" --a_scale "${a_scale}" \ + --eta_A "${eta_a}" --a_warmup_steps 100 \ + --pert_sigma 0.01 --pert_directions 1 --pert_every 4 \ + --alignment_probe 32 \ + --out "results/oral_a_apical_screen/${tag}.json" + done + done +done diff --git a/experiments/oral_a_bp_reference.sh b/experiments/oral_a_bp_reference.sh new file mode 100755 index 0000000..cf5669e --- /dev/null +++ b/experiments/oral_a_bp_reference.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -euo pipefail + +variant="${1:-primary}" +device="${2:-cuda}" +mkdir -p results/oral_a_dev + +common=( + --mode bp --device "${device}" --depth 20 --width 16 --seed 0 --loader_seed 0 + --batch_size 128 --epochs 200 --val_examples 5000 --split_seed 2027 + --eval_split validation --eval_every 20 --augment_train 1 + --lr 0.1 --momentum 0.9 --normalization batchnorm +) + +case "${variant}" in + primary) + python3 experiments/conv_run.py "${common[@]}" \ + --lr_schedule step --lr_milestones 100,150 --lr_gamma 0.1 \ + --warmup_epochs 0 --weight_decay 1e-4 \ + --out results/oral_a_dev/bp_reference_primary.json + ;; + recovery) + python3 experiments/conv_run.py "${common[@]}" \ + --lr_schedule cosine --warmup_epochs 5 --weight_decay 5e-4 \ + --out results/oral_a_dev/bp_reference_recovery.json + ;; + *) + echo "variant must be primary or recovery" >&2 + exit 2 + ;; +esac diff --git a/experiments/oral_a_short_screen.py b/experiments/oral_a_short_screen.py new file mode 100755 index 0000000..51df778 --- /dev/null +++ b/experiments/oral_a_short_screen.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +"""Run a deterministic shard of the frozen A2b short accuracy grid.""" +import argparse +import json +import os +import subprocess +import sys + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--selection", default="results/oral_a_apical_selection.json") + parser.add_argument("--device", default="cuda") + parser.add_argument("--shard_index", type=int, default=0) + parser.add_argument("--num_shards", type=int, default=1) + parser.add_argument("--dry_run", action="store_true") + args = parser.parse_args() + if not 0 <= args.shard_index < args.num_shards: + raise ValueError("invalid shard index") + with open(args.selection) as handle: + selection = json.load(handle) + if selection["status"] != "selected": + raise ValueError("A2a did not select both vectorizer families") + + common = [ + sys.executable, "experiments/conv_run.py", "--device", args.device, + "--depth", "20", "--width", "16", "--seed", "0", "--loader_seed", "0", + "--batch_size", "128", "--epochs", "20", "--train_limit", "10000", + "--val_examples", "5000", "--split_seed", "2027", + "--eval_split", "validation", "--eval_every", "0", "--augment_train", "1", + "--lr_schedule", "cosine", "--warmup_epochs", "0", "--momentum", "0.9", + "--weight_decay", "1e-4", "--normalization", "batchnorm", + ] + jobs = [] + jobs.append(("bp_lr0.1", common + [ + "--mode", "bp", "--lr", "0.1", + "--out", "results/oral_a_short/bp_lr0.1.json"])) + for rate in (0.01, 0.03, 0.1): + jobs.append((f"dfa_lr{rate}", common + [ + "--mode", "dfa", "--lr", str(rate), "--output_lr", "0.1", + "--a_scale", "1", "--vectorizer_mode", "spatial_template", + "--out", f"results/oral_a_short/dfa_lr{rate}.json"])) + for mode in ("spatial_template", "channel_gated"): + chosen = selection["selected"][mode] + for rate in (0.01, 0.03, 0.1): + tag = f"sdil_{mode}_lr{rate}" + jobs.append((tag, common + [ + "--mode", "sdil", "--lr", str(rate), "--output_lr", "0.1", + "--vectorizer_mode", mode, + "--a_scale", str(chosen["a_scale"]), + "--eta_A", str(chosen["eta_A"]), "--a_warmup_steps", "100", + "--pert_sigma", "0.01", "--pert_directions", "1", + "--pert_every", "4", "--alignment_probe", "32", + "--out", f"results/oral_a_short/{tag}.json"])) + os.makedirs("results/oral_a_short", exist_ok=True) + selected_jobs = [job for index, job in enumerate(jobs) + if index % args.num_shards == args.shard_index] + for tag, command in selected_jobs: + print(tag, " ".join(command), flush=True) + if not args.dry_run: + subprocess.run(command, check=True) + + +if __name__ == "__main__": + main() |