experiments: freeze direct perturbation C2 diagnosis

2 files changed, 246 insertions, 0 deletions

diff --git a/experiments/analyze_c2_nodepert_validation.py b/experiments/analyze_c2_nodepert_validation.py
new file mode 100644
index 0000000..634ea0d
--- /dev/null
+++ b/experiments/analyze_c2_nodepert_validation.py
@@ -0,0 +1,194 @@
+"""Audit the frozen C2 direct node-perturbation causal diagnosis."""
+import glob
+import json
+import math
+import os
+import statistics
+
+import analyze_c2_context_validation as context_audit
+
+
+ROOT = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "results")
+PREFIX = "c2_nodepert_val_v1_"
+DEPTHS = (1, 4)
+TASK_SEEDS = (3, 4, 5)
+MODEL_SEEDS = (0, 1, 2, 3, 4)
+
+
+def audit_nodepert_row(path, row):
+    args = row["args"]
+    required = {
+        "mode": "nodepert", "dataset": "tentmap", "width": 8,
+        "act": "relu", "residual": 1, "epochs": 80, "batch_size": 256,
+        "eta": 0.03, "momentum": 0.9, "pert_sigma": 0.01,
+        "pert_every": 1, "pert_ndirs": 16, "pert_mode": "layerwise",
+        "traffic_mode": "none", "nuis_rho": 0.0, "use_residual": 0,
+        "learn_A": 0, "learn_P": 0, "p_neutral": 1,
+        "task_train_examples": 10000, "task_test_examples": 5000,
+        "task_levels": 2, "task_n_in": 1, "val_examples": 2000,
+        "split_seed": 2027, "eval_split": "validation", "eval_every": 0,
+        "diagnostics": "alignment", "diagnostics_schedule": "final",
+        "probe_bs": 512,
+    }
+    mismatches = {key: (args.get(key), expected) for key, expected in required.items()
+                  if args.get(key) != expected}
+    expected_lesion = 1.0 / 3.0 if args["depth"] == 4 else 0.0
+    if abs(args.get("residual_lesion_fraction", 0.0) - expected_lesion) > 1e-12:
+        mismatches["residual_lesion_fraction"] = (
+            args.get("residual_lesion_fraction"), expected_lesion)
+    if mismatches:
+        raise RuntimeError(f"protocol mismatch {path}: {mismatches}")
+    if row["final"].get("eval_split") != "validation":
+        raise RuntimeError(f"test-contaminated validation row: {path}")
+    if any("eval_acc" in step or "cos_q_negg" in step for step in row.get("steps", [])):
+        raise RuntimeError(f"intermediate held-out metric/diagnostic: {path}")
+    split = row.get("split", {})
+    if (not split.get("split_from_training_only")
+            or split.get("validation_examples") != 2000
+            or split.get("evaluation_split") != "validation"):
+        raise RuntimeError(f"invalid validation split {path}: {split}")
+    protocol = row.get("diagnostic_protocol", {})
+    if protocol != {"probe_source": "training_prefix", "probe_examples": 512,
+                    "schedule": "final"}:
+        raise RuntimeError(f"diagnostic protocol mismatch {path}: {protocol}")
+    if row.get("provenance", {}).get("git_dirty") is not False:
+        raise RuntimeError(f"dirty or unknown source provenance: {path}")
+    if args["depth"] == 4:
+        lesion = row["final"].get("residual_lesion")
+        if not lesion or lesion.get("lesioned_layers") != [3]:
+            raise RuntimeError(f"incorrect d4 lesion: {path}")
+
+
+def mean_sd(values):
+    return statistics.mean(values), statistics.stdev(values)
+
+
+def load_context_reference():
+    rows = {}
+    split_hashes = {}
+    for path in sorted(glob.glob(os.path.join(ROOT, context_audit.PREFIX + "*.json"))):
+        with open(path) as handle:
+            row = json.load(handle)
+        context_audit.audit_row(path, row)
+        args = row["args"]
+        key = (args["mode"], args["depth"], args["task_seed"], args["seed"])
+        rows[key] = row
+        split_hashes.setdefault(args["task_seed"], set()).add(
+            row["split"]["validation_index_sha256"])
+    expected = {(method, depth, task_seed, model_seed)
+                for method in context_audit.METHODS for depth in DEPTHS
+                for task_seed in TASK_SEEDS for model_seed in MODEL_SEEDS}
+    if set(rows) != expected:
+        raise RuntimeError("context reference panel is incomplete or mixed")
+    return rows, split_hashes
+
+
+def main():
+    reference, split_hashes = load_context_reference()
+    paths = sorted(glob.glob(os.path.join(ROOT, PREFIX + "*.json")))
+    rows = {}
+    commits = set()
+    for path in paths:
+        with open(path) as handle:
+            row = json.load(handle)
+        audit_nodepert_row(path, row)
+        args = row["args"]
+        key = (args["depth"], args["task_seed"], args["seed"])
+        if key in rows:
+            raise RuntimeError(f"duplicate nodepert row: {key}")
+        rows[key] = row
+        commits.add(row["provenance"]["git_commit"])
+        split_hashes.setdefault(args["task_seed"], set()).add(
+            row["split"]["validation_index_sha256"])
+    expected = {(depth, task_seed, model_seed) for depth in DEPTHS
+                for task_seed in TASK_SEEDS for model_seed in MODEL_SEEDS}
+    if set(rows) != expected or len(commits) != 1:
+        raise RuntimeError(f"incomplete/mixed nodepert panel: rows={len(rows)}, "
+                           f"missing={expected - set(rows)}, extra={set(rows) - expected}, "
+                           f"commits={commits}")
+    if any(len(hashes) != 1 for hashes in split_hashes.values()):
+        raise RuntimeError(f"nodepert/reference split mismatch: {split_hashes}")
+
+    accs = {}
+    gains = {}
+    for method in context_audit.METHODS:
+        for depth in DEPTHS:
+            accs[(method, depth)] = [
+                100 * reference[(method, depth, task_seed, model_seed)]["final"]["val_acc"]
+                for task_seed in TASK_SEEDS for model_seed in MODEL_SEEDS]
+    for depth in DEPTHS:
+        accs[("nodepert", depth)] = [
+            100 * rows[(depth, task_seed, model_seed)]["final"]["val_acc"]
+            for task_seed in TASK_SEEDS for model_seed in MODEL_SEEDS]
+    for method in (*context_audit.METHODS, "nodepert"):
+        gains[method] = [deep - shallow for shallow, deep in
+                         zip(accs[(method, 1)], accs[(method, 4)])]
+
+    print(f"nodepert_commit={next(iter(commits))} rows={len(rows)} "
+          f"task_seeds={list(TASK_SEEDS)}")
+    print("| method | depth | validation (%) | depth gain | lesion drop |")
+    print("|:---|---:|---:|---:|---:|")
+    for method in (*context_audit.METHODS, "nodepert"):
+        for depth in DEPTHS:
+            mean, sd = mean_sd(accs[(method, depth)])
+            gain_text = "--" if depth == 1 else f"{statistics.mean(gains[method]):+.3f}"
+            if depth == 1:
+                lesion_text = "--"
+            else:
+                source = rows if method == "nodepert" else reference
+                drops = []
+                for task_seed in TASK_SEEDS:
+                    for model_seed in MODEL_SEEDS:
+                        key = ((4, task_seed, model_seed) if method == "nodepert"
+                               else (method, 4, task_seed, model_seed))
+                        drops.append(100 * source[key]["final"]["residual_lesion"]
+                                     ["lesion_acc_drop"])
+                lesion_text = f"{statistics.mean(drops):+.3f}"
+            print(f"| {method} | {depth} | {mean:.3f} +/- {sd:.3f} | "
+                  f"{gain_text} | {lesion_text} |")
+
+    bp_gain = statistics.mean(gains["bp"])
+    nodepert_gain = statistics.mean(gains["nodepert"])
+    recovery = nodepert_gain / bp_gain if bp_gain > 0 else -math.inf
+    positive_gains = sum(value > 0 for value in gains["nodepert"])
+    nodepert_d4 = statistics.mean(accs[("nodepert", 4)])
+    context_d4 = statistics.mean(accs[("sdil", 4)])
+    strongest_fixed = max(statistics.mean(accs[(method, 4)]) for method in ("fa", "dfa"))
+    lesion_drops = [100 * rows[(4, task_seed, model_seed)]["final"]
+                    ["residual_lesion"]["lesion_acc_drop"]
+                    for task_seed in TASK_SEEDS for model_seed in MODEL_SEEDS]
+    lesion_mean = statistics.mean(lesion_drops)
+    lesion_positive = sum(value > 0 for value in lesion_drops)
+    nonfinite = [key for key, row in rows.items()
+                 if not math.isfinite(row["final"].get("val_loss", math.nan))]
+    d4_rows = [rows[(4, task_seed, model_seed)] for task_seed in TASK_SEEDS
+               for model_seed in MODEL_SEEDS]
+    forward_work = statistics.mean(
+        row["cost"]["training_forward_equivalent_examples"] for row in d4_rows)
+    ordinary_work = statistics.mean(
+        row["cost"]["ordinary_training_forward_examples"] for row in d4_rows)
+    scalar_queries = statistics.mean(
+        row["cost"]["calibration_example_loss_evaluations"] for row in d4_rows)
+    wall = statistics.mean(row["final"]["wall_s"] for row in d4_rows)
+
+    passed = (not nonfinite and bp_gain >= 5.0 and recovery >= 0.7
+              and positive_gains >= 10 and nodepert_d4 - context_d4 >= 2.0
+              and lesion_mean >= 2.0 and lesion_positive >= 10)
+    print(f"BP gain={bp_gain:+.3f}; direct-NP gain={nodepert_gain:+.3f}; "
+          f"recovery={100 * recovery:.1f}%; positive gains={positive_gains}/15")
+    print(f"direct-NP d4 minus context-SDIL={nodepert_d4 - context_d4:+.3f}; "
+          f"minus strongest FA/DFA={nodepert_d4 - strongest_fixed:+.3f}")
+    print(f"direct-NP lesion mean={lesion_mean:+.3f}; positive={lesion_positive}/15; "
+          f"nonfinite={nonfinite}")
+    print(f"d4 cost: scalar loss queries={scalar_queries:.0f}; "
+          f"forward-equivalent examples={forward_work:.0f} "
+          f"({forward_work / ordinary_work:.1f}x ordinary); wall={wall:.1f}s")
+    print("C2 causal diagnosis: " +
+          ("AMORTIZATION BOTTLENECK CONFIRMED" if passed else
+           "DIRECT CAUSAL ESTIMATOR/LOCAL OPTIMIZATION ALSO INSUFFICIENT"))
+    if not passed:
+        raise SystemExit(1)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/experiments/c2_nodepert_validation.sh b/experiments/c2_nodepert_validation.sh
new file mode 100755
index 0000000..4a9e523
--- /dev/null
+++ b/experiments/c2_nodepert_validation.sh
@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+# Frozen C2 causal diagnosis after validation-only task-0 development.
+#
+# Development selection (task seed 0, model seeds 0--2, d4 only):
+#   * K1 estimator/LR screen selected layerwise eta=0.03.
+#   * At eta=0.03, K4 reached 95.183% and K16 reached 97.017% mean
+#     validation accuracy.  The predeclared rule selected the smaller K only
+#     when it was within one point of the best, so K16 is frozen here.
+#
+# This independent panel uses the same untouched task/data seeds 3--5 and
+# student seeds 0--4 as the earlier context-vectorizer panel.  It evaluates
+# validation once after training; the independent synthetic test sets remain
+# untouched.  The direct targets are deliberately query-expensive: this is a
+# causal diagnostic and unamortized baseline, not a proposed efficient method.
+# Usage: c2_nodepert_validation.sh <gpu> "<model seeds>"
+set -eu
+
+cd "$(dirname "$0")/.."
+GPU="${1:?GPU index required}"
+MODEL_SEEDS="${2:-0 1 2 3 4}"
+PYTHON="${PYTHON:-/home/yurenh2/miniconda3/envs/ep_pascal/bin/python3}"
+
+for task_seed in 3 4 5; do
+  for model_seed in $MODEL_SEEDS; do
+    for depth in 1 4; do
+      tag="c2_nodepert_val_v1_tent_l2_w8_nodepert_d${depth}_t${task_seed}_s${model_seed}"
+      out="results/${tag}.json"
+      if [ -s "$out" ]; then
+        echo "[$tag] exists; skipping"
+        continue
+      fi
+      lesion=0
+      if [ "$depth" -eq 4 ]; then
+        lesion=0.3333333333333333
+      fi
+      CUDA_VISIBLE_DEVICES="$GPU" "$PYTHON" experiments/run.py \
+        --mode nodepert --dataset tentmap --device cuda \
+        --depth "$depth" --width 8 --act relu --residual 1 \
+        --residual_lesion_fraction "$lesion" \
+        --epochs 80 --batch_size 256 --eta 0.03 --momentum 0.9 \
+        --pert_sigma 0.01 --pert_every 1 --pert_ndirs 16 \
+        --pert_mode layerwise \
+        --traffic_mode none --nuis_rho 0 \
+        --use_residual 0 --learn_A 0 --learn_P 0 --p_neutral 1 \
+        --task_train_examples 10000 --task_test_examples 5000 \
+        --task_levels 2 --task_n_in 1 --task_seed "$task_seed" \
+        --val_examples 2000 --split_seed 2027 --eval_split validation --eval_every 0 \
+        --diagnostics alignment --diagnostics_schedule final --probe_bs 512 \
+        --seed "$model_seed" --log_every 100000 --tag "$tag"
+    done
+  done
+done