diff options
| -rw-r--r-- | experiments/analyze_traffic.py | 73 | ||||
| -rwxr-xr-x | experiments/traffic_validation_sweep.sh | 74 |
2 files changed, 147 insertions, 0 deletions
diff --git a/experiments/analyze_traffic.py b/experiments/analyze_traffic.py new file mode 100644 index 0000000..92faa13 --- /dev/null +++ b/experiments/analyze_traffic.py @@ -0,0 +1,73 @@ +"""Audit development validation runs for endogenous apical traffic. + +This script deliberately refuses test-evaluated files: its output is for +protocol selection, not a paper-facing test table. +""" +import glob +import json +import math +import os +import statistics + + +ROOT = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "results") +PREFIX = "traffic_dev_v1_" + + +def mean_sd(values): + mean = statistics.mean(values) + sd = statistics.stdev(values) if len(values) > 1 else float("nan") + return mean, sd + + +def load_rows(): + rows = [] + for path in sorted(glob.glob(os.path.join(ROOT, PREFIX + "*.json"))): + with open(path) as handle: + row = json.load(handle) + if row.get("final", {}).get("eval_split") != "validation": + raise RuntimeError(f"non-validation result in development sweep: {path}") + split = row.get("split", {}) + if not split.get("validation_index_sha256"): + raise RuntimeError(f"missing validation split hash: {path}") + if row.get("provenance", {}).get("git_dirty") is not False: + raise RuntimeError(f"dirty or unknown source revision: {path}") + rows.append(row) + return rows + + +def main(): + rows = load_rows() + print("| dataset | traffic | scale | signal | n | validation acc (%) | traffic R2 |") + print("|:---|:---|---:|:---|---:|---:|---:|") + groups = {} + for row in rows: + args = row["args"] + signal = "residual" + if not args["use_residual"]: + signal = "matched" if args["raw_scale_control"] == "match_innovation_norm" else "raw" + elif not args["p_neutral"]: + signal = "taskfit" + key = (args["dataset"], args["traffic_mode"], args["nuis_rho"], signal) + groups.setdefault(key, []).append(row) + + for key in sorted(groups): + group = groups[key] + acc = [100 * row["final"]["val_acc"] for row in group] + r2 = [statistics.mean(v for v in row["final"]["traffic_r2"] + if v is not None and math.isfinite(v)) + for row in group + if any(v is not None and math.isfinite(v) + for v in row["final"]["traffic_r2"])] + am, asd = mean_sd(acc) + r2_text = "—" + if r2: + rm, rsd = mean_sd(r2) + r2_text = f"{rm:.3f}" if len(r2) == 1 else f"{rm:.3f} ± {rsd:.3f}" + acc_text = f"{am:.3f}" if len(acc) == 1 else f"{am:.3f} ± {asd:.3f}" + print(f"| {key[0]} | {key[1]} | {key[2]:g} | {key[3]} | {len(group)} | " + f"{acc_text} | {r2_text} |") + + +if __name__ == "__main__": + main() diff --git a/experiments/traffic_validation_sweep.sh b/experiments/traffic_validation_sweep.sh new file mode 100755 index 0000000..9d27e25 --- /dev/null +++ b/experiments/traffic_validation_sweep.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# Development-only validation sweep for endogenous non-teaching apical traffic. +# No test-set metric is emitted. Freeze a protocol before creating test results. +# +# Usage: +# traffic_validation_sweep.sh <gpu> [dataset] [families] [scales] [signals] [seeds] +# Example: +# traffic_validation_sweep.sh 5 mnist "soma topdown" "0 0.2 0.5" \ +# "raw matched residual taskfit" "0 1" +set -eu + +cd "$(dirname "$0")/.." +GPU="${1:?GPU index required}" +DATASET="${2:-mnist}" +FAMILIES="${3:-soma topdown}" +SCALES="${4:-0 0.2 0.5}" +SIGNALS="${5:-raw matched residual taskfit}" +SEEDS="${6:-0}" +PYTHON="${PYTHON:-/home/yurenh2/miniconda3/envs/ep_pascal/bin/python3}" + +for seed in $SEEDS; do + for family in $FAMILIES; do + for scale in $SCALES; do + # The zero-traffic condition is family-independent. + if [ "$scale" = "0" ] && [ "$family" != "soma" ]; then + continue + fi + scale_tag="${scale//./p}" + for signal in $SIGNALS; do + use_residual=1 + raw_control=none + p_neutral=1 + case "$signal" in + raw) + use_residual=0 + ;; + matched) + use_residual=0 + raw_control=match_innovation_norm + ;; + residual) + ;; + taskfit) + p_neutral=0 + ;; + *) + echo "unknown signal control: $signal" >&2 + exit 2 + ;; + esac + + tag="traffic_dev_v1_${DATASET}_${family}_rho${scale_tag}_${signal}_d3_s${seed}" + out="results/${tag}.json" + if [ -s "$out" ]; then + echo "[$tag] exists; skipping" + continue + fi + CUDA_VISIBLE_DEVICES="$GPU" "$PYTHON" experiments/run.py \ + --mode sdil --dataset "$DATASET" --device cuda \ + --depth 3 --width 256 --residual 1 --act tanh \ + --epochs 15 --batch_size 128 --eta 0.05 --momentum 0.9 \ + --eta_A 0.02 --eta_P 0.002 \ + --pert_sigma 0.01 --pert_every 4 --pert_ndirs 16 \ + --pert_mode simultaneous \ + --traffic_mode "$family" --nuis_rho "$scale" \ + --use_residual "$use_residual" --raw_scale_control "$raw_control" \ + --learn_A 1 --learn_P 1 --p_neutral "$p_neutral" \ + --p_warmup_steps 200 --p_warmup_eta 0.05 \ + --val_examples 5000 --split_seed 2027 --eval_split validation \ + --seed "$seed" --log_every 100 --tag "$tag" + done + done + done +done |