diff options
| -rw-r--r-- | experiments/analyze_traffic_timescale.py | 52 | ||||
| -rwxr-xr-x | experiments/depth_task_validation_sweep.sh | 2 | ||||
| -rwxr-xr-x | experiments/traffic_timescale_validation_sweep.sh | 39 | ||||
| -rwxr-xr-x | experiments/traffic_validation_sweep.sh | 2 |
4 files changed, 93 insertions, 2 deletions
diff --git a/experiments/analyze_traffic_timescale.py b/experiments/analyze_traffic_timescale.py new file mode 100644 index 0000000..e266d8f --- /dev/null +++ b/experiments/analyze_traffic_timescale.py @@ -0,0 +1,52 @@ +"""Audit predictor-timescale validation development runs.""" +import glob +import json +import math +import os +import statistics + + +ROOT = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "results") + + +def finite_mean(values): + values = [value for value in values if value is not None and math.isfinite(value)] + return statistics.mean(values) if values else float("nan") + + +def main(): + rows = [] + for path in sorted(glob.glob(os.path.join(ROOT, "traffic_time_dev_v1_*.json"))): + with open(path) as handle: + row = json.load(handle) + if row.get("final", {}).get("eval_split") != "validation": + raise RuntimeError(f"non-validation timescale result: {path}") + if row.get("provenance", {}).get("git_dirty") is not False: + raise RuntimeError(f"dirty or unknown provenance: {path}") + if not row.get("split", {}).get("validation_index_sha256"): + raise RuntimeError(f"missing validation hash: {path}") + rows.append(row) + + print("| scale | eta_P | warmup | n | last val (%) | best val (%) | initial traffic R2 | final traffic R2 |") + print("|---:|---:|---:|---:|---:|---:|---:|---:|") + groups = {} + for row in rows: + args = row["args"] + key = (args["nuis_rho"], args["eta_P"], args["p_warmup_steps"]) + groups.setdefault(key, []).append(row) + + for key in sorted(groups): + group = groups[key] + last = [100 * row["final"]["val_acc"] for row in group] + best = [100 * max(step["val_acc"] for step in row["steps"] if "val_acc" in step) + for row in group] + initial_r2 = [finite_mean(next(step["traffic_r2"] for step in row["steps"] + if "traffic_r2" in step)) for row in group] + final_r2 = [finite_mean(row["final"]["traffic_r2"]) for row in group] + print(f"| {key[0]:g} | {key[1]:g} | {key[2]} | {len(group)} | " + f"{statistics.mean(last):.3f} | {statistics.mean(best):.3f} | " + f"{statistics.mean(initial_r2):.3f} | {statistics.mean(final_r2):.3f} |") + + +if __name__ == "__main__": + main() diff --git a/experiments/depth_task_validation_sweep.sh b/experiments/depth_task_validation_sweep.sh index 5105069..2ee0f67 100755 --- a/experiments/depth_task_validation_sweep.sh +++ b/experiments/depth_task_validation_sweep.sh @@ -67,7 +67,7 @@ for task_seed in $TASK_SEEDS; do --task_train_examples 50000 --task_test_examples 10000 \ --task_seed "$task_seed" $TASK_ARGS \ --val_examples 5000 --split_seed 2027 --eval_split validation \ - --seed "$model_seed" --log_every 200 --tag "$tag" + --diagnostics alignment --seed "$model_seed" --log_every 200 --tag "$tag" done done done diff --git a/experiments/traffic_timescale_validation_sweep.sh b/experiments/traffic_timescale_validation_sweep.sh new file mode 100755 index 0000000..593394a --- /dev/null +++ b/experiments/traffic_timescale_validation_sweep.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Validation-only development sweep for predictor tracking timescales. +# Usage: +# traffic_timescale_validation_sweep.sh <gpu> <scale> [eta_Ps] [warmup_steps] [seeds] +set -eu + +cd "$(dirname "$0")/.." +GPU="${1:?GPU index required}" +SCALE="${2:?topdown traffic scale required}" +ETA_PS="${3:-0.002 0.01 0.05}" +WARMUPS="${4:-0 200 1000}" +SEEDS="${5:-0}" +PYTHON="${PYTHON:-/home/yurenh2/miniconda3/envs/ep_pascal/bin/python3}" +scale_tag="${SCALE//./p}" + +for seed in $SEEDS; do + for eta_p in $ETA_PS; do + eta_tag="${eta_p//./p}" + for warmup in $WARMUPS; do + tag="traffic_time_dev_v1_mnist_topdown_rho${scale_tag}_ep${eta_tag}_wu${warmup}_d3_s${seed}" + out="results/${tag}.json" + if [ -s "$out" ]; then + echo "[$tag] exists; skipping" + continue + fi + CUDA_VISIBLE_DEVICES="$GPU" "$PYTHON" experiments/run.py \ + --mode sdil --dataset mnist --device cuda \ + --depth 3 --width 256 --residual 1 --act tanh \ + --epochs 15 --batch_size 128 --eta 0.05 --momentum 0.9 \ + --eta_A 0.02 --eta_P "$eta_p" \ + --pert_sigma 0.01 --pert_every 4 --pert_ndirs 16 --pert_mode simultaneous \ + --traffic_mode topdown --nuis_rho "$SCALE" \ + --use_residual 1 --learn_A 1 --learn_P 1 --p_neutral 1 \ + --p_warmup_steps "$warmup" --p_warmup_eta 0.05 \ + --val_examples 5000 --split_seed 2027 --eval_split validation \ + --diagnostics alignment --seed "$seed" --log_every 200 --tag "$tag" + done + done +done diff --git a/experiments/traffic_validation_sweep.sh b/experiments/traffic_validation_sweep.sh index 9d27e25..f7ea438 100755 --- a/experiments/traffic_validation_sweep.sh +++ b/experiments/traffic_validation_sweep.sh @@ -67,7 +67,7 @@ for seed in $SEEDS; do --learn_A 1 --learn_P 1 --p_neutral "$p_neutral" \ --p_warmup_steps 200 --p_warmup_eta 0.05 \ --val_examples 5000 --split_seed 2027 --eval_split validation \ - --seed "$seed" --log_every 100 --tag "$tag" + --diagnostics alignment --seed "$seed" --log_every 100 --tag "$tag" done done done |