From e8d698e9da1e2441d1d137eb8f8ab6cd46b4282c Mon Sep 17 00:00:00 2001 From: YurenHao0426 Date: Wed, 22 Jul 2026 04:16:20 -0500 Subject: experiments: freeze direct perturbation C2 diagnosis --- experiments/analyze_c2_nodepert_validation.py | 194 ++++++++++++++++++++++++++ experiments/c2_nodepert_validation.sh | 52 +++++++ 2 files changed, 246 insertions(+) create mode 100644 experiments/analyze_c2_nodepert_validation.py create mode 100755 experiments/c2_nodepert_validation.sh diff --git a/experiments/analyze_c2_nodepert_validation.py b/experiments/analyze_c2_nodepert_validation.py new file mode 100644 index 0000000..634ea0d --- /dev/null +++ b/experiments/analyze_c2_nodepert_validation.py @@ -0,0 +1,194 @@ +"""Audit the frozen C2 direct node-perturbation causal diagnosis.""" +import glob +import json +import math +import os +import statistics + +import analyze_c2_context_validation as context_audit + + +ROOT = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "results") +PREFIX = "c2_nodepert_val_v1_" +DEPTHS = (1, 4) +TASK_SEEDS = (3, 4, 5) +MODEL_SEEDS = (0, 1, 2, 3, 4) + + +def audit_nodepert_row(path, row): + args = row["args"] + required = { + "mode": "nodepert", "dataset": "tentmap", "width": 8, + "act": "relu", "residual": 1, "epochs": 80, "batch_size": 256, + "eta": 0.03, "momentum": 0.9, "pert_sigma": 0.01, + "pert_every": 1, "pert_ndirs": 16, "pert_mode": "layerwise", + "traffic_mode": "none", "nuis_rho": 0.0, "use_residual": 0, + "learn_A": 0, "learn_P": 0, "p_neutral": 1, + "task_train_examples": 10000, "task_test_examples": 5000, + "task_levels": 2, "task_n_in": 1, "val_examples": 2000, + "split_seed": 2027, "eval_split": "validation", "eval_every": 0, + "diagnostics": "alignment", "diagnostics_schedule": "final", + "probe_bs": 512, + } + mismatches = {key: (args.get(key), expected) for key, expected in required.items() + if args.get(key) != expected} + expected_lesion = 1.0 / 3.0 if args["depth"] == 4 else 0.0 + if abs(args.get("residual_lesion_fraction", 0.0) - expected_lesion) > 1e-12: + mismatches["residual_lesion_fraction"] = ( + args.get("residual_lesion_fraction"), expected_lesion) + if mismatches: + raise RuntimeError(f"protocol mismatch {path}: {mismatches}") + if row["final"].get("eval_split") != "validation": + raise RuntimeError(f"test-contaminated validation row: {path}") + if any("eval_acc" in step or "cos_q_negg" in step for step in row.get("steps", [])): + raise RuntimeError(f"intermediate held-out metric/diagnostic: {path}") + split = row.get("split", {}) + if (not split.get("split_from_training_only") + or split.get("validation_examples") != 2000 + or split.get("evaluation_split") != "validation"): + raise RuntimeError(f"invalid validation split {path}: {split}") + protocol = row.get("diagnostic_protocol", {}) + if protocol != {"probe_source": "training_prefix", "probe_examples": 512, + "schedule": "final"}: + raise RuntimeError(f"diagnostic protocol mismatch {path}: {protocol}") + if row.get("provenance", {}).get("git_dirty") is not False: + raise RuntimeError(f"dirty or unknown source provenance: {path}") + if args["depth"] == 4: + lesion = row["final"].get("residual_lesion") + if not lesion or lesion.get("lesioned_layers") != [3]: + raise RuntimeError(f"incorrect d4 lesion: {path}") + + +def mean_sd(values): + return statistics.mean(values), statistics.stdev(values) + + +def load_context_reference(): + rows = {} + split_hashes = {} + for path in sorted(glob.glob(os.path.join(ROOT, context_audit.PREFIX + "*.json"))): + with open(path) as handle: + row = json.load(handle) + context_audit.audit_row(path, row) + args = row["args"] + key = (args["mode"], args["depth"], args["task_seed"], args["seed"]) + rows[key] = row + split_hashes.setdefault(args["task_seed"], set()).add( + row["split"]["validation_index_sha256"]) + expected = {(method, depth, task_seed, model_seed) + for method in context_audit.METHODS for depth in DEPTHS + for task_seed in TASK_SEEDS for model_seed in MODEL_SEEDS} + if set(rows) != expected: + raise RuntimeError("context reference panel is incomplete or mixed") + return rows, split_hashes + + +def main(): + reference, split_hashes = load_context_reference() + paths = sorted(glob.glob(os.path.join(ROOT, PREFIX + "*.json"))) + rows = {} + commits = set() + for path in paths: + with open(path) as handle: + row = json.load(handle) + audit_nodepert_row(path, row) + args = row["args"] + key = (args["depth"], args["task_seed"], args["seed"]) + if key in rows: + raise RuntimeError(f"duplicate nodepert row: {key}") + rows[key] = row + commits.add(row["provenance"]["git_commit"]) + split_hashes.setdefault(args["task_seed"], set()).add( + row["split"]["validation_index_sha256"]) + expected = {(depth, task_seed, model_seed) for depth in DEPTHS + for task_seed in TASK_SEEDS for model_seed in MODEL_SEEDS} + if set(rows) != expected or len(commits) != 1: + raise RuntimeError(f"incomplete/mixed nodepert panel: rows={len(rows)}, " + f"missing={expected - set(rows)}, extra={set(rows) - expected}, " + f"commits={commits}") + if any(len(hashes) != 1 for hashes in split_hashes.values()): + raise RuntimeError(f"nodepert/reference split mismatch: {split_hashes}") + + accs = {} + gains = {} + for method in context_audit.METHODS: + for depth in DEPTHS: + accs[(method, depth)] = [ + 100 * reference[(method, depth, task_seed, model_seed)]["final"]["val_acc"] + for task_seed in TASK_SEEDS for model_seed in MODEL_SEEDS] + for depth in DEPTHS: + accs[("nodepert", depth)] = [ + 100 * rows[(depth, task_seed, model_seed)]["final"]["val_acc"] + for task_seed in TASK_SEEDS for model_seed in MODEL_SEEDS] + for method in (*context_audit.METHODS, "nodepert"): + gains[method] = [deep - shallow for shallow, deep in + zip(accs[(method, 1)], accs[(method, 4)])] + + print(f"nodepert_commit={next(iter(commits))} rows={len(rows)} " + f"task_seeds={list(TASK_SEEDS)}") + print("| method | depth | validation (%) | depth gain | lesion drop |") + print("|:---|---:|---:|---:|---:|") + for method in (*context_audit.METHODS, "nodepert"): + for depth in DEPTHS: + mean, sd = mean_sd(accs[(method, depth)]) + gain_text = "--" if depth == 1 else f"{statistics.mean(gains[method]):+.3f}" + if depth == 1: + lesion_text = "--" + else: + source = rows if method == "nodepert" else reference + drops = [] + for task_seed in TASK_SEEDS: + for model_seed in MODEL_SEEDS: + key = ((4, task_seed, model_seed) if method == "nodepert" + else (method, 4, task_seed, model_seed)) + drops.append(100 * source[key]["final"]["residual_lesion"] + ["lesion_acc_drop"]) + lesion_text = f"{statistics.mean(drops):+.3f}" + print(f"| {method} | {depth} | {mean:.3f} +/- {sd:.3f} | " + f"{gain_text} | {lesion_text} |") + + bp_gain = statistics.mean(gains["bp"]) + nodepert_gain = statistics.mean(gains["nodepert"]) + recovery = nodepert_gain / bp_gain if bp_gain > 0 else -math.inf + positive_gains = sum(value > 0 for value in gains["nodepert"]) + nodepert_d4 = statistics.mean(accs[("nodepert", 4)]) + context_d4 = statistics.mean(accs[("sdil", 4)]) + strongest_fixed = max(statistics.mean(accs[(method, 4)]) for method in ("fa", "dfa")) + lesion_drops = [100 * rows[(4, task_seed, model_seed)]["final"] + ["residual_lesion"]["lesion_acc_drop"] + for task_seed in TASK_SEEDS for model_seed in MODEL_SEEDS] + lesion_mean = statistics.mean(lesion_drops) + lesion_positive = sum(value > 0 for value in lesion_drops) + nonfinite = [key for key, row in rows.items() + if not math.isfinite(row["final"].get("val_loss", math.nan))] + d4_rows = [rows[(4, task_seed, model_seed)] for task_seed in TASK_SEEDS + for model_seed in MODEL_SEEDS] + forward_work = statistics.mean( + row["cost"]["training_forward_equivalent_examples"] for row in d4_rows) + ordinary_work = statistics.mean( + row["cost"]["ordinary_training_forward_examples"] for row in d4_rows) + scalar_queries = statistics.mean( + row["cost"]["calibration_example_loss_evaluations"] for row in d4_rows) + wall = statistics.mean(row["final"]["wall_s"] for row in d4_rows) + + passed = (not nonfinite and bp_gain >= 5.0 and recovery >= 0.7 + and positive_gains >= 10 and nodepert_d4 - context_d4 >= 2.0 + and lesion_mean >= 2.0 and lesion_positive >= 10) + print(f"BP gain={bp_gain:+.3f}; direct-NP gain={nodepert_gain:+.3f}; " + f"recovery={100 * recovery:.1f}%; positive gains={positive_gains}/15") + print(f"direct-NP d4 minus context-SDIL={nodepert_d4 - context_d4:+.3f}; " + f"minus strongest FA/DFA={nodepert_d4 - strongest_fixed:+.3f}") + print(f"direct-NP lesion mean={lesion_mean:+.3f}; positive={lesion_positive}/15; " + f"nonfinite={nonfinite}") + print(f"d4 cost: scalar loss queries={scalar_queries:.0f}; " + f"forward-equivalent examples={forward_work:.0f} " + f"({forward_work / ordinary_work:.1f}x ordinary); wall={wall:.1f}s") + print("C2 causal diagnosis: " + + ("AMORTIZATION BOTTLENECK CONFIRMED" if passed else + "DIRECT CAUSAL ESTIMATOR/LOCAL OPTIMIZATION ALSO INSUFFICIENT")) + if not passed: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/experiments/c2_nodepert_validation.sh b/experiments/c2_nodepert_validation.sh new file mode 100755 index 0000000..4a9e523 --- /dev/null +++ b/experiments/c2_nodepert_validation.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Frozen C2 causal diagnosis after validation-only task-0 development. +# +# Development selection (task seed 0, model seeds 0--2, d4 only): +# * K1 estimator/LR screen selected layerwise eta=0.03. +# * At eta=0.03, K4 reached 95.183% and K16 reached 97.017% mean +# validation accuracy. The predeclared rule selected the smaller K only +# when it was within one point of the best, so K16 is frozen here. +# +# This independent panel uses the same untouched task/data seeds 3--5 and +# student seeds 0--4 as the earlier context-vectorizer panel. It evaluates +# validation once after training; the independent synthetic test sets remain +# untouched. The direct targets are deliberately query-expensive: this is a +# causal diagnostic and unamortized baseline, not a proposed efficient method. +# Usage: c2_nodepert_validation.sh "" +set -eu + +cd "$(dirname "$0")/.." +GPU="${1:?GPU index required}" +MODEL_SEEDS="${2:-0 1 2 3 4}" +PYTHON="${PYTHON:-/home/yurenh2/miniconda3/envs/ep_pascal/bin/python3}" + +for task_seed in 3 4 5; do + for model_seed in $MODEL_SEEDS; do + for depth in 1 4; do + tag="c2_nodepert_val_v1_tent_l2_w8_nodepert_d${depth}_t${task_seed}_s${model_seed}" + out="results/${tag}.json" + if [ -s "$out" ]; then + echo "[$tag] exists; skipping" + continue + fi + lesion=0 + if [ "$depth" -eq 4 ]; then + lesion=0.3333333333333333 + fi + CUDA_VISIBLE_DEVICES="$GPU" "$PYTHON" experiments/run.py \ + --mode nodepert --dataset tentmap --device cuda \ + --depth "$depth" --width 8 --act relu --residual 1 \ + --residual_lesion_fraction "$lesion" \ + --epochs 80 --batch_size 256 --eta 0.03 --momentum 0.9 \ + --pert_sigma 0.01 --pert_every 1 --pert_ndirs 16 \ + --pert_mode layerwise \ + --traffic_mode none --nuis_rho 0 \ + --use_residual 0 --learn_A 0 --learn_P 0 --p_neutral 1 \ + --task_train_examples 10000 --task_test_examples 5000 \ + --task_levels 2 --task_n_in 1 --task_seed "$task_seed" \ + --val_examples 2000 --split_seed 2027 --eval_split validation --eval_every 0 \ + --diagnostics alignment --diagnostics_schedule final --probe_bs 512 \ + --seed "$model_seed" --log_every 100000 --tag "$tag" + done + done +done -- cgit v1.2.3