From a0696a3d72cc5bf69876e1716b74f7e1ac525dd0 Mon Sep 17 00:00:00 2001
From: YurenHao0426 <Blackhao0426@gmail.com>
Date: Wed, 22 Jul 2026 01:48:05 -0500
Subject: experiments: add useful-depth validation screen

---
 experiments/analyze_depth_tasks.py         | 51 ++++++++++++++++++++
 experiments/depth_task_validation_sweep.sh | 74 ++++++++++++++++++++++++++++++
 2 files changed, 125 insertions(+)
 create mode 100644 experiments/analyze_depth_tasks.py
 create mode 100755 experiments/depth_task_validation_sweep.sh

(limited to 'experiments')

diff --git a/experiments/analyze_depth_tasks.py b/experiments/analyze_depth_tasks.py
new file mode 100644
index 0000000..f1a8e19
--- /dev/null
+++ b/experiments/analyze_depth_tasks.py
@@ -0,0 +1,51 @@
+"""Audit validation-only useful-depth screening runs."""
+import glob
+import json
+import os
+import statistics
+
+
+ROOT = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "results")
+
+
+def main():
+    groups = {}
+    for path in sorted(glob.glob(os.path.join(ROOT, "depthtask_dev_v1_*.json"))):
+        with open(path) as handle:
+            row = json.load(handle)
+        if row.get("final", {}).get("eval_split") != "validation":
+            raise RuntimeError(f"test result in development screen: {path}")
+        if row.get("provenance", {}).get("git_dirty") is not False:
+            raise RuntimeError(f"dirty or unknown provenance: {path}")
+        split = row.get("split", {})
+        if not split.get("validation_index_sha256"):
+            raise RuntimeError(f"missing synthetic validation hash: {path}")
+        args = row["args"]
+        key = (args["dataset"], args["mode"], args["width"], args["depth"])
+        groups.setdefault(key, []).append(row)
+
+    print("| task | method | width | depth | task/model runs | validation acc (%) |")
+    print("|:---|:---|---:|---:|---:|---:|")
+    by_curve = {}
+    for key in sorted(groups):
+        rows = groups[key]
+        values = [100 * row["final"]["val_acc"] for row in rows]
+        mean = statistics.mean(values)
+        sd = statistics.stdev(values) if len(values) > 1 else None
+        value = f"{mean:.3f}" if sd is None else f"{mean:.3f} ± {sd:.3f}"
+        print(f"| {key[0]} | {key[1].upper()} | {key[2]} | {key[3]} | {len(rows)} | {value} |")
+        by_curve.setdefault(key[:3], {})[key[3]] = mean
+
+    print()
+    for key, curve in sorted(by_curve.items()):
+        if len(curve) < 2:
+            continue
+        shallow, deep = min(curve), max(curve)
+        gain = curve[deep] - curve[shallow]
+        verdict = "PASS" if gain >= 5.0 else "FAIL"
+        print(f"{key[0]} {key[1].upper()} w{key[2]}: d{shallow}->d{deep} "
+              f"gain {gain:+.3f} points [{verdict} C2 BP-screen threshold]")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/experiments/depth_task_validation_sweep.sh b/experiments/depth_task_validation_sweep.sh
new file mode 100755
index 0000000..5105069
--- /dev/null
+++ b/experiments/depth_task_validation_sweep.sh
@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+# Validation-only screening for tasks on which additional depth is useful to BP.
+# The test generator is not evaluated. Only tasks passing the BP depth-gain gate
+# should advance to local-learning and lesion experiments.
+#
+# Usage:
+#   depth_task_validation_sweep.sh <gpu> <dataset> [depths] [modes] [width] [model_seeds] [task_seeds]
+set -eu
+
+cd "$(dirname "$0")/.."
+GPU="${1:?GPU index required}"
+DATASET="${2:?dataset required: teacher, hierarchical, or tentmap}"
+DEPTHS="${3:-1 2 4 6 8 12}"
+MODES="${4:-bp}"
+WIDTH_OVERRIDE="${5:-}"
+MODEL_SEEDS="${6:-0}"
+TASK_SEEDS="${7:-0}"
+PYTHON="${PYTHON:-/home/yurenh2/miniconda3/envs/ep_pascal/bin/python3}"
+
+case "$DATASET" in
+  teacher)
+    WIDTH="${WIDTH_OVERRIDE:-64}"
+    ACT=tanh
+    RESIDUAL=1
+    EPOCHS=25
+    ETA=0.03
+    TASK_ARGS="--task_n_in 128 --task_classes 10 --teacher_depth 8 --teacher_width 64 --teacher_residual 1"
+    ;;
+  hierarchical)
+    WIDTH="${WIDTH_OVERRIDE:-64}"
+    ACT=tanh
+    RESIDUAL=1
+    EPOCHS=25
+    ETA=0.03
+    TASK_ARGS="--task_levels 6 --task_classes 10"
+    ;;
+  tentmap)
+    WIDTH="${WIDTH_OVERRIDE:-16}"
+    ACT=relu
+    RESIDUAL=0
+    EPOCHS=40
+    ETA=0.02
+    TASK_ARGS="--task_levels 7 --task_n_in 4"
+    ;;
+  *)
+    echo "unknown depth task: $DATASET" >&2
+    exit 2
+    ;;
+esac
+
+for task_seed in $TASK_SEEDS; do
+  for model_seed in $MODEL_SEEDS; do
+    for mode in $MODES; do
+      for depth in $DEPTHS; do
+        tag="depthtask_dev_v1_${DATASET}_${mode}_w${WIDTH}_d${depth}_t${task_seed}_s${model_seed}"
+        out="results/${tag}.json"
+        if [ -s "$out" ]; then
+          echo "[$tag] exists; skipping"
+          continue
+        fi
+        CUDA_VISIBLE_DEVICES="$GPU" "$PYTHON" experiments/run.py \
+          --mode "$mode" --dataset "$DATASET" --device cuda \
+          --depth "$depth" --width "$WIDTH" --act "$ACT" --residual "$RESIDUAL" \
+          --epochs "$EPOCHS" --batch_size 128 --eta "$ETA" --momentum 0.9 \
+          --eta_A 0.02 --eta_P 0.002 \
+          --pert_sigma 0.01 --pert_every 4 --pert_ndirs 16 --pert_mode simultaneous \
+          --task_train_examples 50000 --task_test_examples 10000 \
+          --task_seed "$task_seed" $TASK_ARGS \
+          --val_examples 5000 --split_seed 2027 --eval_split validation \
+          --seed "$model_seed" --log_every 200 --tag "$tag"
+      done
+    done
+  done
+done
-- 
cgit v1.2.3