diff options
| -rw-r--r-- | ROADMAP.md | 16 | ||||
| -rw-r--r-- | experiments/analyze_c2_bp_depth_validation.py | 109 | ||||
| -rwxr-xr-x | experiments/c2_bp_depth_validation.sh | 34 |
3 files changed, 159 insertions, 0 deletions
@@ -95,6 +95,22 @@ recover at least 70% of that paired gain; the strongest competing local rule mus than 50%, or SDIL must beat it by at least 2 points at the deep endpoint. Removing the final third of trained blocks must reduce SDIL performance, ruling out an effective shallow-network solution. +The C2 candidate is frozen from scratch pilots before formal validation: a two-level Telgarsky +tent-map binary task, one informative input, width 8, ReLU residual students, 80 epochs, +batch size 256, `eta=0.03`, and momentum 0.9. Each task has 10,000 generated training examples, +of which a stratified 2,000-example validation split (`split_seed=2027`) is held out; its untouched +test set has 5,000 independently generated examples. Shallow/deep are fixed to d1/d4. A BP shape +screen uses task seeds 0, 1, 2, student seed 0, and depths 1, 2, 3, 4, 6. Contextual depths cannot +replace d4 after results are observed. The BP screen passes only if mean d1-to-d4 gain is at least +5 points, every task seed gains, mean d4 final-third-lesion drop is at least 2 points, and every +task seed is hurt by the lesion. Only final validation metrics are observed. + +If BP passes, the local-rule validation panel freezes d1/d4 for BP, FA, DFA, and no-traffic SDIL +with C3's `K=1/every=4`, crossing the same three task seeds with student seeds 0--4. Its test panel +may run only if the original C2 gate holds on validation and the d4 SDIL lesion loses at least +2 points on average with at least 10 of 15 paired runs harmed. Test confirmation retrains on the +same 8,000-example training subset and evaluates each task seed's untouched 5,000 examples once. + ### C3. Causal calibration is genuinely amortized **Status: passed on 2026-07-22.** The frozen five-seed confirmation retained 112.9% of the K16/e4 diff --git a/experiments/analyze_c2_bp_depth_validation.py b/experiments/analyze_c2_bp_depth_validation.py new file mode 100644 index 0000000..f024049 --- /dev/null +++ b/experiments/analyze_c2_bp_depth_validation.py @@ -0,0 +1,109 @@ +"""Audit the frozen BP useful-depth validation curve for C2.""" +import glob +import json +import os +import statistics + + +ROOT = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "results") +PREFIX = "c2_bp_curve_val_v1_" +DEPTHS = (1, 2, 3, 4, 6) +TASK_SEEDS = (0, 1, 2) + + +def audit_row(path, row): + args = row["args"] + required = { + "mode": "bp", "dataset": "tentmap", "width": 8, "act": "relu", + "residual": 1, "epochs": 80, "batch_size": 256, + "eta": 0.03, "momentum": 0.9, + "task_train_examples": 10000, "task_test_examples": 5000, + "task_levels": 2, "task_n_in": 1, + "val_examples": 2000, "split_seed": 2027, + "eval_split": "validation", "eval_every": 0, + "diagnostics": "none", "diagnostics_schedule": "final", + "probe_bs": 512, "seed": 0, + } + mismatches = {key: (args.get(key), expected) for key, expected in required.items() + if args.get(key) != expected} + depth = args["depth"] + expected_lesion = 0.0 if depth == 1 else 1.0 / 3.0 + if abs(args.get("residual_lesion_fraction", 0.0) - expected_lesion) > 1e-12: + mismatches["residual_lesion_fraction"] = ( + args.get("residual_lesion_fraction"), expected_lesion) + if mismatches: + raise RuntimeError(f"protocol mismatch {path}: {mismatches}") + if row["final"].get("eval_split") != "validation": + raise RuntimeError(f"test-contaminated development row: {path}") + if any("eval_acc" in step for step in row.get("steps", [])): + raise RuntimeError(f"intermediate validation metric: {path}") + split = row.get("split", {}) + if (not split.get("split_from_training_only") + or split.get("validation_examples") != 2000 + or split.get("evaluation_split") != "validation"): + raise RuntimeError(f"invalid validation split {path}: {split}") + if row.get("provenance", {}).get("git_dirty") is not False: + raise RuntimeError(f"dirty or unknown source provenance: {path}") + if depth > 1: + lesion = row["final"].get("residual_lesion") + if not lesion or abs(row["lesion_protocol"]["fraction_of_interior_blocks"] - 1 / 3) > 1e-12: + raise RuntimeError(f"missing final-third lesion: {path}") + + +def main(): + paths = sorted(glob.glob(os.path.join(ROOT, PREFIX + "*.json"))) + rows = {} + commits = set() + split_hashes = {} + for path in paths: + with open(path) as handle: + row = json.load(handle) + audit_row(path, row) + args = row["args"] + key = (args["task_seed"], args["depth"]) + if key in rows: + raise RuntimeError(f"duplicate row: {key}") + rows[key] = row + commits.add(row["provenance"]["git_commit"]) + split_hashes.setdefault(args["task_seed"], set()).add( + row["split"]["validation_index_sha256"]) + expected = {(task_seed, depth) for task_seed in TASK_SEEDS for depth in DEPTHS} + if set(rows) != expected or len(commits) != 1: + raise RuntimeError(f"incomplete/mixed screen: rows={len(rows)}, " + f"missing={expected - set(rows)}, extra={set(rows) - expected}, " + f"commits={commits}") + if any(len(hashes) != 1 for hashes in split_hashes.values()): + raise RuntimeError(f"depths did not share splits within tasks: {split_hashes}") + + print(f"commit={next(iter(commits))} rows={len(rows)} task_seeds={list(TASK_SEEDS)}") + print("| depth | mean validation (%) | SD | per-task validation (%) | lesion drop (%) |") + print("|---:|---:|---:|:---|:---|") + for depth in DEPTHS: + accs = [100 * rows[(task_seed, depth)]["final"]["val_acc"] + for task_seed in TASK_SEEDS] + if depth == 1: + lesion_text = "--" + else: + drops = [100 * rows[(task_seed, depth)]["final"]["residual_lesion"]["lesion_acc_drop"] + for task_seed in TASK_SEEDS] + lesion_text = ", ".join(f"{value:+.2f}" for value in drops) + print(f"| {depth} | {statistics.mean(accs):.3f} | {statistics.stdev(accs):.3f} | " + f"{', '.join(f'{value:.2f}' for value in accs)} | {lesion_text} |") + + gains = [100 * (rows[(task_seed, 4)]["final"]["val_acc"] + - rows[(task_seed, 1)]["final"]["val_acc"]) + for task_seed in TASK_SEEDS] + lesion_drops = [100 * rows[(task_seed, 4)]["final"]["residual_lesion"]["lesion_acc_drop"] + for task_seed in TASK_SEEDS] + passed = (statistics.mean(gains) >= 5.0 and all(gain > 0 for gain in gains) + and statistics.mean(lesion_drops) >= 2.0 + and all(drop > 0 for drop in lesion_drops)) + print(f"frozen d1->d4 gains={gains}, mean={statistics.mean(gains):+.3f}") + print(f"d4 final-third lesion drops={lesion_drops}, mean={statistics.mean(lesion_drops):+.3f}") + print(f"C2 BP useful-depth screen: {'PASS' if passed else 'FAIL'}") + if not passed: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/experiments/c2_bp_depth_validation.sh b/experiments/c2_bp_depth_validation.sh new file mode 100755 index 0000000..f294d27 --- /dev/null +++ b/experiments/c2_bp_depth_validation.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Frozen BP useful-depth screen for the C2 Telgarsky candidate. +# Usage: c2_bp_depth_validation.sh <gpu> "<task seeds>" +set -eu + +cd "$(dirname "$0")/.." +GPU="${1:?GPU index required}" +TASK_SEEDS="${2:-0 1 2}" +PYTHON="${PYTHON:-/home/yurenh2/miniconda3/envs/ep_pascal/bin/python3}" + +for task_seed in $TASK_SEEDS; do + for depth in 1 2 3 4 6; do + tag="c2_bp_curve_val_v1_tent_l2_w8_d${depth}_t${task_seed}_s0" + out="results/${tag}.json" + if [ -s "$out" ]; then + echo "[$tag] exists; skipping" + continue + fi + lesion=0 + if [ "$depth" -gt 1 ]; then + lesion=0.3333333333333333 + fi + CUDA_VISIBLE_DEVICES="$GPU" "$PYTHON" experiments/run.py \ + --mode bp --dataset tentmap --device cuda \ + --depth "$depth" --width 8 --act relu --residual 1 \ + --residual_lesion_fraction "$lesion" \ + --epochs 80 --batch_size 256 --eta 0.03 --momentum 0.9 \ + --task_train_examples 10000 --task_test_examples 5000 \ + --task_levels 2 --task_n_in 1 --task_seed "$task_seed" \ + --val_examples 2000 --split_seed 2027 --eval_split validation --eval_every 0 \ + --diagnostics none --diagnostics_schedule final --probe_bs 512 \ + --seed 0 --log_every 100000 --tag "$tag" + done +done |