diff options
Diffstat (limited to 'experiments')
| -rw-r--r-- | experiments/analyze_c1_confirm.py | 153 | ||||
| -rwxr-xr-x | experiments/c1_innovation_confirm.sh | 83 | ||||
| -rw-r--r-- | experiments/protocol_smoke.py | 12 |
3 files changed, 247 insertions, 1 deletions
diff --git a/experiments/analyze_c1_confirm.py b/experiments/analyze_c1_confirm.py new file mode 100644 index 0000000..2ce3424 --- /dev/null +++ b/experiments/analyze_c1_confirm.py @@ -0,0 +1,153 @@ +"""Audit the frozen C1 mixed-apical-traffic confirmation panel.""" +import glob +import json +import math +import os +import statistics + + +ROOT = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "results") +PREFIX = "c1_confirm_v1_" +MODEL_SEEDS = tuple(range(5)) +TRAFFIC_FAMILIES = (("soma", 0.5, 1234), ("soma", 0.5, 5678), + ("topdown", 0.2, 1234), ("topdown", 0.2, 5678)) +SIGNALS = ("raw", "matched", "residual", "taskfit") + + +def mean_sd(values): + return (statistics.mean(values), + statistics.stdev(values) if len(values) > 1 else 0.0) + + +def signal_name(args): + if not args["use_residual"]: + return ("matched" if args["raw_scale_control"] == "match_innovation_norm" + else "raw") + return "residual" if args["p_neutral"] else "taskfit" + + +def finite_mean(values): + finite = [value for value in values if value is not None and math.isfinite(value)] + if not finite: + raise RuntimeError("expected at least one finite diagnostic") + return statistics.mean(finite) + + +def audit_row(path, row): + args = row["args"] + required = { + "mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, + "residual": 1, "epochs": 15, "batch_size": 128, "eta": 0.05, + "momentum": 0.9, "eta_A": 0.02, "eta_P": 0.05, + "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 1, + "pert_mode": "simultaneous", "learn_A": 1, "learn_P": 1, + "p_warmup_steps": 200, "p_warmup_eta": 0.05, + "val_examples": 0, "eval_split": "test", "eval_every": 0, + "diagnostics": "alignment", "diagnostics_schedule": "final", + } + mismatches = {key: (args.get(key), value) for key, value in required.items() + if args.get(key) != value} + if mismatches: + raise RuntimeError(f"protocol mismatch {path}: {mismatches}") + if row.get("final", {}).get("eval_split") != "test": + raise RuntimeError(f"non-test confirmation row: {path}") + if any("eval_acc" in step or "cos_r_negg" in step for step in row.get("steps", [])): + raise RuntimeError(f"intermediate held-out metric/diagnostic: {path}") + protocol = row.get("diagnostic_protocol", {}) + if protocol != {"probe_source": "training_prefix", "probe_examples": 512, + "schedule": "final"}: + raise RuntimeError(f"diagnostic protocol mismatch {path}: {protocol}") + if row.get("provenance", {}).get("git_dirty") is not False: + raise RuntimeError(f"dirty or unknown source provenance: {path}") + if "hardware" not in row or "training_forward_equivalent_examples" not in row.get("cost", {}): + raise RuntimeError(f"missing cost/memory audit: {path}") + + +def main(): + paths = sorted(glob.glob(os.path.join(ROOT, PREFIX + "*.json"))) + rows = [] + commits = set() + for path in paths: + with open(path) as handle: + row = json.load(handle) + audit_row(path, row) + commits.add(row["provenance"]["git_commit"]) + rows.append(row) + if len(commits) != 1: + raise RuntimeError(f"expected one clean source commit, got {commits}") + + groups = {} + for row in rows: + args = row["args"] + family = (args["traffic_mode"], args["nuis_rho"], args["traffic_seed"]) + groups.setdefault((family, signal_name(args)), []).append(row) + + expected = {(("none", 0.0, 1234), signal) for signal in SIGNALS[:3]} + expected |= {(family, signal) for family in TRAFFIC_FAMILIES for signal in SIGNALS} + actual = set(groups) + bad_counts = {key: len(value) for key, value in groups.items() if len(value) != 5} + if actual != expected or bad_counts or len(rows) != 95: + raise RuntimeError(f"incomplete/unexpected C1 panel: rows={len(rows)}, " + f"missing={expected - actual}, extra={actual - expected}, " + f"bad_counts={bad_counts}") + for key, group in groups.items(): + group.sort(key=lambda row: row["args"]["seed"]) + seeds = tuple(row["args"]["seed"] for row in group) + if seeds != MODEL_SEEDS: + raise RuntimeError(f"unpaired model seeds for {key}: {seeds}") + + print(f"commit={next(iter(commits))} model_seeds={list(MODEL_SEEDS)} rows={len(rows)}") + print("| traffic | rho | traffic seed | signal | test (%) | mean cos(r,-g) | traffic R2 |") + print("|:---|---:|---:|:---|---:|---:|---:|") + order = (("none", 0.0, 1234),) + TRAFFIC_FAMILIES + for family in order: + signals = SIGNALS[:3] if family[0] == "none" else SIGNALS + for signal in signals: + group = groups[(family, signal)] + acc = mean_sd([100 * row["final"]["test_acc"] for row in group]) + cosine = mean_sd([finite_mean(row["final"]["cos_r_negg"]) for row in group]) + r2_values = [] + if family[0] != "none": + r2_values = [finite_mean(row["final"]["traffic_r2"]) for row in group] + r2_text = "--" if not r2_values else f"{mean_sd(r2_values)[0]:.3f}" + print(f"| {family[0]} | {family[1]:g} | {family[2]} | {signal} | " + f"{acc[0]:.3f} +/- {acc[1]:.3f} | {cosine[0]:+.3f} | {r2_text} |") + + gate_checks = [] + for family in TRAFFIC_FAMILIES: + matched = groups[(family, "matched")] + residual = groups[(family, "residual")] + paired_gain = statistics.mean( + 100 * (res["final"]["test_acc"] - raw["final"]["test_acc"]) + for res, raw in zip(residual, matched)) + mean_r2 = statistics.mean(finite_mean(row["final"]["traffic_r2"]) + for row in residual) + mean_alignment = statistics.mean(finite_mean(row["final"]["cos_r_negg"]) + for row in residual) + passed = paired_gain >= 2.0 and mean_r2 >= 0.5 and mean_alignment >= 0.1 + gate_checks.append(passed) + print(f"{family}: residual-matched={paired_gain:+.3f} points, " + f"R2={mean_r2:.3f}, alignment={mean_alignment:+.3f}: " + f"{'PASS' if passed else 'FAIL'}") + + no_family = ("none", 0.0, 1234) + no_residual = groups[(no_family, "residual")] + no_harms = {} + for signal in ("raw", "matched"): + comparator = groups[(no_family, signal)] + no_harms[signal] = statistics.mean( + 100 * (res["final"]["test_acc"] - other["final"]["test_acc"]) + for res, other in zip(no_residual, comparator)) + no_traffic_pass = min(no_harms.values()) >= -0.5 + gate_checks.append(no_traffic_pass) + print(f"no-traffic residual-minus controls={no_harms}: " + f"{'PASS' if no_traffic_pass else 'FAIL'}") + + passed = all(gate_checks) + print(f"C1 confirmation gate: {'PASS' if passed else 'FAIL'}") + if not passed: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/experiments/c1_innovation_confirm.sh b/experiments/c1_innovation_confirm.sh new file mode 100755 index 0000000..82442e6 --- /dev/null +++ b/experiments/c1_innovation_confirm.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# Frozen five-seed test confirmation of innovation under endogenous apical traffic. +# Usage: c1_innovation_confirm.sh <gpu> "<model seeds>" +set -eu + +cd "$(dirname "$0")/.." +GPU="${1:?GPU index required}" +SEEDS="${2:-0 1 2 3 4}" +PYTHON="${PYTHON:-/home/yurenh2/miniconda3/envs/ep_pascal/bin/python3}" + +run_one() { + model_seed="$1" + family="$2" + scale="$3" + traffic_seed="$4" + signal="$5" + + use_residual=1 + raw_control=none + p_neutral=1 + case "$signal" in + raw) + use_residual=0 + ;; + matched) + use_residual=0 + raw_control=match_innovation_norm + ;; + residual) + ;; + taskfit) + p_neutral=0 + ;; + *) + echo "unknown signal control: $signal" >&2 + exit 2 + ;; + esac + + scale_tag="${scale//./p}" + tag="c1_confirm_v1_mnist_${family}_rho${scale_tag}_t${traffic_seed}_${signal}_d3_s${model_seed}" + out="results/${tag}.json" + if [ -s "$out" ]; then + echo "[$tag] exists; skipping" + return + fi + + CUDA_VISIBLE_DEVICES="$GPU" "$PYTHON" experiments/run.py \ + --mode sdil --dataset mnist --device cuda \ + --depth 3 --width 256 --residual 1 --act tanh \ + --epochs 15 --batch_size 128 --eta 0.05 --momentum 0.9 \ + --eta_A 0.02 --eta_P 0.05 \ + --pert_sigma 0.01 --pert_every 4 --pert_ndirs 1 \ + --pert_mode simultaneous \ + --traffic_mode "$family" --traffic_seed "$traffic_seed" --nuis_rho "$scale" \ + --use_residual "$use_residual" --raw_scale_control "$raw_control" \ + --learn_A 1 --learn_P 1 --p_neutral "$p_neutral" \ + --p_warmup_steps 200 --p_warmup_eta 0.05 \ + --val_examples 0 --eval_split test --eval_every 0 \ + --diagnostics alignment --diagnostics_schedule final --probe_bs 512 \ + --seed "$model_seed" --log_every 100000 --tag "$tag" +} + +for seed in $SEEDS; do + # At rho=0, raw, norm-matched raw, and neutral-period innovation should agree. + for signal in raw matched residual; do + run_one "$seed" none 0 1234 "$signal" + done + + # Levels and predictor timescale were selected on the training-only + # validation screens. Two independently seeded projections of each traffic + # construction are frozen here; no test result changes this matrix. + for spec in soma:0.5 topdown:0.2; do + IFS=: read -r family scale <<EOF +$spec +EOF + for traffic_seed in 1234 5678; do + for signal in raw matched residual taskfit; do + run_one "$seed" "$family" "$scale" "$traffic_seed" "$signal" + done + done + done +done diff --git a/experiments/protocol_smoke.py b/experiments/protocol_smoke.py index 551987c..01c914d 100644 --- a/experiments/protocol_smoke.py +++ b/experiments/protocol_smoke.py @@ -7,7 +7,7 @@ import torch sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from sdil.data import get_dataset_splits from sdil.core import SDILConfig, SDILNet -from experiments.run import calibration_work_per_event +from experiments.run import calibration_work_per_event, fixed_training_probe def loader_labels(loader): @@ -33,6 +33,15 @@ def main(): assert sum(y.numel() for _, y in validation) == 1000 assert sum(y.numel() for _, y in test) == 10000 + probe_loader = get_dataset_splits( + "mnist", batch_size=256, device="cpu", val_examples=1000, + split_seed=2027)[0] + generator_state = probe_loader.g.get_state().clone() + probe_x, probe_y = fixed_training_probe(probe_loader, 137, "cpu") + assert torch.equal(generator_state, probe_loader.g.get_state()) + assert torch.equal(probe_x, probe_loader.x[:137]) + assert torch.equal(probe_y, probe_loader.y[:137]) + net = SDILNet([10, 8, 8, 3], device="cpu") simultaneous = calibration_work_per_event( net, SDILConfig(pert_ndirs=2, pert_mode="simultaneous")) @@ -47,6 +56,7 @@ def main(): assert abs(layerwise["forward_equivalent_batches"] - 11.0 / 3.0) < 1e-12 print("validation split hash:", metadata["validation_index_sha256"]) print("train/validation/test: 59000/1000/10000; stratification exact") + print("training-prefix diagnostic probe: deterministic; shuffle state unchanged") print("simultaneous/layerwise calibration cost accounting: exact") print("ALL PROTOCOL SMOKE CHECKS PASSED") |