diff options
| -rw-r--r-- | experiments/analyze_depth_tasks.py | 51 | ||||
| -rwxr-xr-x | experiments/depth_task_validation_sweep.sh | 74 |
2 files changed, 125 insertions, 0 deletions
diff --git a/experiments/analyze_depth_tasks.py b/experiments/analyze_depth_tasks.py new file mode 100644 index 0000000..f1a8e19 --- /dev/null +++ b/experiments/analyze_depth_tasks.py @@ -0,0 +1,51 @@ +"""Audit validation-only useful-depth screening runs.""" +import glob +import json +import os +import statistics + + +ROOT = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "results") + + +def main(): + groups = {} + for path in sorted(glob.glob(os.path.join(ROOT, "depthtask_dev_v1_*.json"))): + with open(path) as handle: + row = json.load(handle) + if row.get("final", {}).get("eval_split") != "validation": + raise RuntimeError(f"test result in development screen: {path}") + if row.get("provenance", {}).get("git_dirty") is not False: + raise RuntimeError(f"dirty or unknown provenance: {path}") + split = row.get("split", {}) + if not split.get("validation_index_sha256"): + raise RuntimeError(f"missing synthetic validation hash: {path}") + args = row["args"] + key = (args["dataset"], args["mode"], args["width"], args["depth"]) + groups.setdefault(key, []).append(row) + + print("| task | method | width | depth | task/model runs | validation acc (%) |") + print("|:---|:---|---:|---:|---:|---:|") + by_curve = {} + for key in sorted(groups): + rows = groups[key] + values = [100 * row["final"]["val_acc"] for row in rows] + mean = statistics.mean(values) + sd = statistics.stdev(values) if len(values) > 1 else None + value = f"{mean:.3f}" if sd is None else f"{mean:.3f} ± {sd:.3f}" + print(f"| {key[0]} | {key[1].upper()} | {key[2]} | {key[3]} | {len(rows)} | {value} |") + by_curve.setdefault(key[:3], {})[key[3]] = mean + + print() + for key, curve in sorted(by_curve.items()): + if len(curve) < 2: + continue + shallow, deep = min(curve), max(curve) + gain = curve[deep] - curve[shallow] + verdict = "PASS" if gain >= 5.0 else "FAIL" + print(f"{key[0]} {key[1].upper()} w{key[2]}: d{shallow}->d{deep} " + f"gain {gain:+.3f} points [{verdict} C2 BP-screen threshold]") + + +if __name__ == "__main__": + main() diff --git a/experiments/depth_task_validation_sweep.sh b/experiments/depth_task_validation_sweep.sh new file mode 100755 index 0000000..5105069 --- /dev/null +++ b/experiments/depth_task_validation_sweep.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# Validation-only screening for tasks on which additional depth is useful to BP. +# The test generator is not evaluated. Only tasks passing the BP depth-gain gate +# should advance to local-learning and lesion experiments. +# +# Usage: +# depth_task_validation_sweep.sh <gpu> <dataset> [depths] [modes] [width] [model_seeds] [task_seeds] +set -eu + +cd "$(dirname "$0")/.." +GPU="${1:?GPU index required}" +DATASET="${2:?dataset required: teacher, hierarchical, or tentmap}" +DEPTHS="${3:-1 2 4 6 8 12}" +MODES="${4:-bp}" +WIDTH_OVERRIDE="${5:-}" +MODEL_SEEDS="${6:-0}" +TASK_SEEDS="${7:-0}" +PYTHON="${PYTHON:-/home/yurenh2/miniconda3/envs/ep_pascal/bin/python3}" + +case "$DATASET" in + teacher) + WIDTH="${WIDTH_OVERRIDE:-64}" + ACT=tanh + RESIDUAL=1 + EPOCHS=25 + ETA=0.03 + TASK_ARGS="--task_n_in 128 --task_classes 10 --teacher_depth 8 --teacher_width 64 --teacher_residual 1" + ;; + hierarchical) + WIDTH="${WIDTH_OVERRIDE:-64}" + ACT=tanh + RESIDUAL=1 + EPOCHS=25 + ETA=0.03 + TASK_ARGS="--task_levels 6 --task_classes 10" + ;; + tentmap) + WIDTH="${WIDTH_OVERRIDE:-16}" + ACT=relu + RESIDUAL=0 + EPOCHS=40 + ETA=0.02 + TASK_ARGS="--task_levels 7 --task_n_in 4" + ;; + *) + echo "unknown depth task: $DATASET" >&2 + exit 2 + ;; +esac + +for task_seed in $TASK_SEEDS; do + for model_seed in $MODEL_SEEDS; do + for mode in $MODES; do + for depth in $DEPTHS; do + tag="depthtask_dev_v1_${DATASET}_${mode}_w${WIDTH}_d${depth}_t${task_seed}_s${model_seed}" + out="results/${tag}.json" + if [ -s "$out" ]; then + echo "[$tag] exists; skipping" + continue + fi + CUDA_VISIBLE_DEVICES="$GPU" "$PYTHON" experiments/run.py \ + --mode "$mode" --dataset "$DATASET" --device cuda \ + --depth "$depth" --width "$WIDTH" --act "$ACT" --residual "$RESIDUAL" \ + --epochs "$EPOCHS" --batch_size 128 --eta "$ETA" --momentum 0.9 \ + --eta_A 0.02 --eta_P 0.002 \ + --pert_sigma 0.01 --pert_every 4 --pert_ndirs 16 --pert_mode simultaneous \ + --task_train_examples 50000 --task_test_examples 10000 \ + --task_seed "$task_seed" $TASK_ARGS \ + --val_examples 5000 --split_seed 2027 --eval_split validation \ + --seed "$model_seed" --log_every 200 --tag "$tag" + done + done + done +done |