experiments: add validation-only apical traffic sweep

2 files changed, 147 insertions, 0 deletions

diff --git a/experiments/analyze_traffic.py b/experiments/analyze_traffic.py
new file mode 100644
index 0000000..92faa13
--- /dev/null
+++ b/experiments/analyze_traffic.py
@@ -0,0 +1,73 @@
+"""Audit development validation runs for endogenous apical traffic.
+
+This script deliberately refuses test-evaluated files: its output is for
+protocol selection, not a paper-facing test table.
+"""
+import glob
+import json
+import math
+import os
+import statistics
+
+
+ROOT = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "results")
+PREFIX = "traffic_dev_v1_"
+
+
+def mean_sd(values):
+    mean = statistics.mean(values)
+    sd = statistics.stdev(values) if len(values) > 1 else float("nan")
+    return mean, sd
+
+
+def load_rows():
+    rows = []
+    for path in sorted(glob.glob(os.path.join(ROOT, PREFIX + "*.json"))):
+        with open(path) as handle:
+            row = json.load(handle)
+        if row.get("final", {}).get("eval_split") != "validation":
+            raise RuntimeError(f"non-validation result in development sweep: {path}")
+        split = row.get("split", {})
+        if not split.get("validation_index_sha256"):
+            raise RuntimeError(f"missing validation split hash: {path}")
+        if row.get("provenance", {}).get("git_dirty") is not False:
+            raise RuntimeError(f"dirty or unknown source revision: {path}")
+        rows.append(row)
+    return rows
+
+
+def main():
+    rows = load_rows()
+    print("| dataset | traffic | scale | signal | n | validation acc (%) | traffic R2 |")
+    print("|:---|:---|---:|:---|---:|---:|---:|")
+    groups = {}
+    for row in rows:
+        args = row["args"]
+        signal = "residual"
+        if not args["use_residual"]:
+            signal = "matched" if args["raw_scale_control"] == "match_innovation_norm" else "raw"
+        elif not args["p_neutral"]:
+            signal = "taskfit"
+        key = (args["dataset"], args["traffic_mode"], args["nuis_rho"], signal)
+        groups.setdefault(key, []).append(row)
+
+    for key in sorted(groups):
+        group = groups[key]
+        acc = [100 * row["final"]["val_acc"] for row in group]
+        r2 = [statistics.mean(v for v in row["final"]["traffic_r2"]
+                              if v is not None and math.isfinite(v))
+              for row in group
+              if any(v is not None and math.isfinite(v)
+                     for v in row["final"]["traffic_r2"])]
+        am, asd = mean_sd(acc)
+        r2_text = "—"
+        if r2:
+            rm, rsd = mean_sd(r2)
+            r2_text = f"{rm:.3f}" if len(r2) == 1 else f"{rm:.3f} ± {rsd:.3f}"
+        acc_text = f"{am:.3f}" if len(acc) == 1 else f"{am:.3f} ± {asd:.3f}"
+        print(f"| {key[0]} | {key[1]} | {key[2]:g} | {key[3]} | {len(group)} | "
+              f"{acc_text} | {r2_text} |")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/experiments/traffic_validation_sweep.sh b/experiments/traffic_validation_sweep.sh
new file mode 100755
index 0000000..9d27e25
--- /dev/null
+++ b/experiments/traffic_validation_sweep.sh
@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+# Development-only validation sweep for endogenous non-teaching apical traffic.
+# No test-set metric is emitted. Freeze a protocol before creating test results.
+#
+# Usage:
+#   traffic_validation_sweep.sh <gpu> [dataset] [families] [scales] [signals] [seeds]
+# Example:
+#   traffic_validation_sweep.sh 5 mnist "soma topdown" "0 0.2 0.5" \
+#       "raw matched residual taskfit" "0 1"
+set -eu
+
+cd "$(dirname "$0")/.."
+GPU="${1:?GPU index required}"
+DATASET="${2:-mnist}"
+FAMILIES="${3:-soma topdown}"
+SCALES="${4:-0 0.2 0.5}"
+SIGNALS="${5:-raw matched residual taskfit}"
+SEEDS="${6:-0}"
+PYTHON="${PYTHON:-/home/yurenh2/miniconda3/envs/ep_pascal/bin/python3}"
+
+for seed in $SEEDS; do
+  for family in $FAMILIES; do
+    for scale in $SCALES; do
+      # The zero-traffic condition is family-independent.
+      if [ "$scale" = "0" ] && [ "$family" != "soma" ]; then
+        continue
+      fi
+      scale_tag="${scale//./p}"
+      for signal in $SIGNALS; do
+        use_residual=1
+        raw_control=none
+        p_neutral=1
+        case "$signal" in
+          raw)
+            use_residual=0
+            ;;
+          matched)
+            use_residual=0
+            raw_control=match_innovation_norm
+            ;;
+          residual)
+            ;;
+          taskfit)
+            p_neutral=0
+            ;;
+          *)
+            echo "unknown signal control: $signal" >&2
+            exit 2
+            ;;
+        esac
+
+        tag="traffic_dev_v1_${DATASET}_${family}_rho${scale_tag}_${signal}_d3_s${seed}"
+        out="results/${tag}.json"
+        if [ -s "$out" ]; then
+          echo "[$tag] exists; skipping"
+          continue
+        fi
+        CUDA_VISIBLE_DEVICES="$GPU" "$PYTHON" experiments/run.py \
+          --mode sdil --dataset "$DATASET" --device cuda \
+          --depth 3 --width 256 --residual 1 --act tanh \
+          --epochs 15 --batch_size 128 --eta 0.05 --momentum 0.9 \
+          --eta_A 0.02 --eta_P 0.002 \
+          --pert_sigma 0.01 --pert_every 4 --pert_ndirs 16 \
+          --pert_mode simultaneous \
+          --traffic_mode "$family" --nuis_rho "$scale" \
+          --use_residual "$use_residual" --raw_scale_control "$raw_control" \
+          --learn_A 1 --learn_P 1 --p_neutral "$p_neutral" \
+          --p_warmup_steps 200 --p_warmup_eta 0.05 \
+          --val_examples 5000 --split_seed 2027 --eval_split validation \
+          --seed "$seed" --log_every 100 --tag "$tag"
+      done
+    done
+  done
+done