experiment: freeze same-path bias confirmation

4 files changed, 516 insertions, 0 deletions

diff --git a/CONTRASTIVE_BIAS_CONFIRMATION.md b/CONTRASTIVE_BIAS_CONFIRMATION.md
new file mode 100644
index 0000000..d20b3f4
--- /dev/null
+++ b/CONTRASTIVE_BIAS_CONFIRMATION.md
@@ -0,0 +1,60 @@
+# Same-path contrastive-bias confirmation
+
+## Status and purpose
+
+This is a post-screen confirmation frozen after the B1 results were known.  It
+is not the untouched B2 in `CONTRASTIVE_BIAS.md`.  B1 showed the intended raw
+failure and innovation recovery at every activity-bias ratio, but its complete
+gate failed because ordinary clean DP ended at 69.66% instead of 70% and a
+common-bias run differed from it by 2.28 accuracy points despite exactly zero
+teaching-difference error.
+
+This confirmation removes that control mismatch.  All four conditions use the
+same explicit-difference training path.  All four conditions for one seed run
+sequentially on the same physical GPU.
+
+## Frozen panel
+
+The implementation, upstream revision, model, data split, batch order, and
+local bias equations are unchanged from B1.  The five new model/minibatch seeds
+are `1989, 1990, 1991, 1992, 1993`.  Every run uses author Dual Prop miniCNN on
+CIFAR-10, `alpha=0`, `beta=0.1`, `fwK`, 16 inference passes, batch size 100,
+and the complete 130-epoch author schedule: peak learning rate 0.025, warm-up
+learning rate 0.001, 10 warm-up epochs, decay epoch 120, and final learning
+rate `2e-6`.  Test is not evaluated.
+
+For every seed, the four ratio-4 conditions are:
+
+1. `same_path_clean`: common activity bias with the raw rule.  Its compartment
+   difference is exactly zero, so this is clean DP through the same code path.
+2. `raw`: differential activity bias with the raw rule.
+3. `innovation`: differential activity bias with the local neutral affine
+   subtraction.
+4. `oracle`: differential activity bias with exact generated-bias subtraction.
+
+The ratio is calibrated once at initialization and frozen.  A raw run that
+becomes nonfinite is retained.  Clean, innovation, and oracle must complete all
+130 epochs.  Test remains untouched until the full validation panel and audit
+exist.
+
+## Confirmation gate
+
+The panel passes only if all source, registry, hardware, split, and test
+isolation checks pass and:
+
+- same-path clean, innovation, and oracle are finite for 130 epochs in all five
+  seeds, and mean same-path-clean final validation accuracy is at least 80%;
+- raw is nonfinite or at least 20 accuracy points below same-path clean in
+  every seed;
+- innovation is above raw in every seed and the one-sided 95% lower bound on
+  the paired gain is above 20 points;
+- the one-sided 95% upper bound on the paired clean-minus-innovation deficit is
+  below 2 points;
+- the two-sided 95% bound on the absolute mean innovation-minus-oracle gap is
+  below 2 points;
+- innovation's post-subtraction bias RMS is at most `1e-3` of raw bias in every
+  run, and every predictor reports zero task-instruction observations;
+- every seed's four conditions record the same physical GPU UUID.
+
+Failure is retained and narrows the claim.  Passing permits a separately
+frozen, once-only test evaluation; it does not retroactively make B1 pass.
diff --git a/experiments/analyze_contrastive_bias_c1.py b/experiments/analyze_contrastive_bias_c1.py
new file mode 100644
index 0000000..3fc3bb3
--- /dev/null
+++ b/experiments/analyze_contrastive_bias_c1.py
@@ -0,0 +1,154 @@
+#!/usr/bin/env python3
+"""Audit the same-path five-seed contrastive-bias confirmation."""
+import argparse
+import json
+import math
+from pathlib import Path
+import statistics
+
+from contrastive_bias_c1 import CONDITIONS, RESULT_ROOT, SEEDS
+
+
+T95 = 2.131846786326649
+
+
+def read_json(path):
+    with open(path, encoding="utf-8") as handle:
+        return json.load(handle)
+
+
+def record_path(seed, condition):
+    name = f"dp-bias-c1-s{seed}-{condition.replace('_', '-')}"
+    return RESULT_ROOT / (name + ".json")
+
+
+def bound(values, absolute_mean=False):
+    mean = statistics.fmean(values)
+    sem = statistics.stdev(values) / math.sqrt(len(values))
+    center = abs(mean) if absolute_mean else mean
+    return {"mean": mean, "sem": sem, "upper_95": center + T95 * sem}
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--out", type=Path,
+                        default=RESULT_ROOT.parent / "c1_gate.json")
+    args = parser.parse_args()
+    conditions = tuple(row[0] for row in CONDITIONS)
+    missing = [
+        f"s{seed}:{condition}" for seed in SEEDS for condition in conditions
+        if not record_path(seed, condition).is_file()
+    ]
+    if missing:
+        raise RuntimeError("missing C1 cells: " + ", ".join(missing))
+    records = {}
+    for seed in SEEDS:
+        for condition in conditions:
+            record = read_json(record_path(seed, condition))
+            if (
+                record.get("status") != "completed"
+                or record.get("seed") != seed
+                or record.get("condition") != condition
+                or record.get("ratio") != 4.0
+                or not math.isnan(float(
+                    (record.get("history") or {}).get("test_accuracy", float("nan"))))
+            ):
+                raise RuntimeError(f"invalid C1 record s{seed}:{condition}")
+            records[(seed, condition)] = record
+
+    rows = []
+    gains = []
+    deficits = []
+    innovation_oracle = []
+    for seed in SEEDS:
+        histories = {
+            condition: records[(seed, condition)]["history"]
+            for condition in conditions
+        }
+        clean = histories["same_path_clean"]["final_validation_accuracy"]
+        raw = histories["raw"]["final_validation_accuracy"]
+        innovation = histories["innovation"]["final_validation_accuracy"]
+        oracle = histories["oracle"]["final_validation_accuracy"]
+        raw_failed = (
+            histories["raw"]["finite"] is not True or clean - raw >= 20.0)
+        gains.append(innovation - raw)
+        deficits.append(clean - innovation)
+        innovation_oracle.append(innovation - oracle)
+        uuids = {
+            records[(seed, condition)]["hardware"]["uuid"]
+            for condition in conditions
+        }
+        rows.append({
+            "seed": seed, "same_path_clean": clean, "raw": raw,
+            "innovation": innovation, "oracle": oracle,
+            "raw_finite": histories["raw"]["finite"],
+            "raw_failed": raw_failed, "innovation_minus_raw": innovation - raw,
+            "clean_minus_innovation": clean - innovation,
+            "innovation_minus_oracle": innovation - oracle,
+            "single_physical_gpu": len(uuids) == 1,
+            "physical_gpu_uuid": next(iter(uuids)) if len(uuids) == 1 else None,
+        })
+
+    gain_stats = bound(gains)
+    gain_stats["lower_95"] = gain_stats["mean"] - T95 * gain_stats["sem"]
+    deficit_stats = bound(deficits)
+    oracle_stats = bound(innovation_oracle, absolute_mean=True)
+    complete_finite = all(
+        records[(seed, condition)]["history"]["finite"] is True
+        and records[(seed, condition)]["history"]["epochs_completed"] == 130
+        for seed in SEEDS
+        for condition in ("same_path_clean", "innovation", "oracle")
+    )
+    clean_mean = statistics.fmean(row["same_path_clean"] for row in rows)
+    post_bias_max = max(
+        max(abs(value) for value in records[(seed, "innovation")]["history"]
+            ["curves"]["post_bias_raw_bias_rms_ratio"])
+        for seed in SEEDS
+    )
+    instruction_max = max(
+        max(abs(value) for value in records[(seed, condition)]["history"]
+            ["curves"]["instruction_observations_for_predictor"])
+        for seed in SEEDS for condition in conditions
+    )
+    source_locks = {
+        json.dumps(record["source"], sort_keys=True) for record in records.values()
+    }
+    registry_locks = {record["registry_sha256"] for record in records.values()}
+    checks = {
+        "complete_finite_clean_innovation_oracle": complete_finite,
+        "mean_same_path_clean_at_least_80": clean_mean >= 80.0,
+        "raw_failed_every_seed": all(row["raw_failed"] for row in rows),
+        "innovation_above_raw_every_seed": all(value > 0 for value in gains),
+        "gain_lower_95_above_20": gain_stats["lower_95"] > 20.0,
+        "clean_deficit_upper_95_below_2": deficit_stats["upper_95"] < 2.0,
+        "innovation_oracle_abs_mean_bound_below_2": oracle_stats["upper_95"] < 2.0,
+        "post_bias_ratio_at_most_1e_3": post_bias_max <= 1e-3,
+        "zero_instruction_observations": instruction_max == 0.0,
+        "same_gpu_within_every_seed": all(row["single_physical_gpu"] for row in rows),
+        "single_source_lock": len(source_locks) == 1,
+        "single_registry_lock": len(registry_locks) == 1,
+    }
+    first = records[(SEEDS[0], conditions[0])]
+    report = {
+        "stage": "contrastive_bias_c1", "gate": (
+            "pass" if all(checks.values()) else "fail"),
+        "checks": checks, "rows": rows,
+        "mean_same_path_clean_validation_accuracy": clean_mean,
+        "paired_innovation_minus_raw": gain_stats,
+        "paired_clean_minus_innovation": deficit_stats,
+        "paired_innovation_minus_oracle": oracle_stats,
+        "maximum_post_bias_ratio": post_bias_max,
+        "maximum_instruction_observations": instruction_max,
+        "num_expected_records": 20, "num_audited_records": len(records),
+        "source": first["source"], "registry_sha256": first["registry_sha256"],
+        "test_policy": "none",
+    }
+    args.out.parent.mkdir(parents=True, exist_ok=True)
+    with open(args.out, "w", encoding="utf-8") as handle:
+        json.dump(report, handle, indent=2, sort_keys=True)
+        handle.write("\n")
+    print(json.dumps(report, indent=2, sort_keys=True))
+
+
+if __name__ == "__main__":
+    main()
diff --git a/experiments/contrastive_bias_c1.py b/experiments/contrastive_bias_c1.py
new file mode 100644
index 0000000..0ef1276
--- /dev/null
+++ b/experiments/contrastive_bias_c1.py
@@ -0,0 +1,265 @@
+#!/usr/bin/env python3
+"""Five-seed same-path full-schedule contrastive-bias confirmation."""
+import argparse
+import hashlib
+import json
+import math
+import os
+from pathlib import Path
+import subprocess
+import time
+
+import numpy as np
+
+
+ROOT = Path(__file__).resolve().parents[1]
+PROTOCOL = ROOT / "CONTRASTIVE_BIAS_CONFIRMATION.md"
+RESULT_ROOT = ROOT / "results" / "contrastive_bias" / "c1"
+BIAS_PATCH = (
+    ROOT / "external" / "dualprop_patches" /
+    "0020-experiment-add-neuron-specific-bias-to-Dual-Prop.patch"
+)
+UPSTREAM = "7b2595b34421e1483a721dbfdeff8cdabda3a1ff"
+SEEDS = (1989, 1990, 1991, 1992, 1993)
+CONDITIONS = (
+    ("same_path_clean", "common", "raw"),
+    ("raw", "activity", "raw"),
+    ("innovation", "activity", "innovation"),
+    ("oracle", "activity", "oracle"),
+)
+
+
+def sha256(path):
+    digest = hashlib.sha256()
+    with open(path, "rb") as handle:
+        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def git_output(repo, *args):
+    return subprocess.run(
+        ["git", *args], cwd=repo, check=True, capture_output=True, text=True
+    ).stdout.strip()
+
+
+def command_for(seed, condition, kind, rule, author_python):
+    name = f"dp-bias-c1-s{seed}-{condition.replace('_', '-')}"
+    command = [
+        author_python, "train.py", "--model", "miniCNN", "--dataset", "cifar10",
+        "--num-epochs", "130", "--batch-size", "100",
+        "--learning-rate", "0.025", "--learning-rate-final", "2e-6",
+        "--warmup-learning-rate", "0.001", "--warmup-epochs", "10",
+        "--decay-epochs", "120", "--momentum", "0.9",
+        "--weight-decay", "5e-4", "--dtype", "float32",
+        "--param-dtype", "float32", "--percent-train", "90",
+        "--percent-val", "10", "--seeds", str(seed),
+        "--feedback-seed", "1729", "--gradient-diagnostics", "none",
+        "--spectral-diagnostics", "none", "--test-policy", "none",
+        "--early-stop-policy", "none", "--learning-algorithm",
+        "dualprop-lagr-ff", "--experiment-name", name,
+        "--optimizer-schedule", "author", "--loss", "sce",
+        "--alpha", "0.0", "--beta", "0.1", "--inference-sequence", "fwK",
+        "--inference-passes-nudged", "16", "--dp-bias-kind", kind,
+        "--dp-bias-rule", rule, "--dp-bias-ratio", "4.0",
+        "--dp-bias-seed", "6100", "--dp-bias-calibration-examples", "64",
+    ]
+    return name, command
+
+
+def jobs(author_python):
+    rows = []
+    for seed in SEEDS:
+        for condition, kind, rule in CONDITIONS:
+            name, command = command_for(
+                seed, condition, kind, rule, author_python)
+            rows.append({
+                "seed": seed, "condition": condition, "kind": kind,
+                "rule": rule, "ratio": 4.0, "experiment_name": name,
+                "command": command,
+                "output": str(RESULT_ROOT / (name + ".json")),
+                "timeout_seconds": 60 * 60,
+            })
+    if len(rows) != 20 or len({row["experiment_name"] for row in rows}) != 20:
+        raise AssertionError("C1 registry must contain 20 unique cells")
+    return rows
+
+
+def registry_sha256(rows):
+    payload = [
+        {key: value for key, value in row.items() if key != "output"}
+        for row in rows
+    ]
+    return hashlib.sha256(json.dumps(
+        payload, sort_keys=True, separators=(",", ":")
+    ).encode()).hexdigest()
+
+
+def source_report(author_root):
+    if git_output(ROOT, "status", "--porcelain", "--untracked-files=no"):
+        raise RuntimeError("C1 requires clean tracked SDIL source")
+    if git_output(author_root, "status", "--porcelain", "--untracked-files=no"):
+        raise RuntimeError("C1 requires clean tracked author source")
+    tracked = [
+        PROTOCOL, Path(__file__).resolve(),
+        ROOT / "experiments" / "analyze_contrastive_bias_c1.py", BIAS_PATCH,
+    ]
+    for path in tracked:
+        subprocess.run([
+            "git", "ls-files", "--error-unmatch", str(path.relative_to(ROOT))
+        ], cwd=ROOT, check=True, capture_output=True)
+    return {
+        "sdil_commit": git_output(ROOT, "rev-parse", "HEAD"),
+        "author_commit": git_output(author_root, "rev-parse", "HEAD"),
+        "author_upstream": UPSTREAM,
+        "tracked_files": {
+            str(path.relative_to(ROOT)): sha256(path) for path in tracked
+        },
+    }
+
+
+def gpu_report(physical_index):
+    output = subprocess.run([
+        "nvidia-smi", f"--id={physical_index}",
+        "--query-gpu=index,uuid,name,memory.total", "--format=csv,noheader,nounits",
+    ], check=True, capture_output=True, text=True).stdout.strip()
+    fields = [part.strip() for part in output.split(",")]
+    visible = os.environ.get("CUDA_VISIBLE_DEVICES")
+    if len(fields) != 4 or fields[0] != str(physical_index):
+        raise RuntimeError(f"could not resolve physical GPU {physical_index}: {output}")
+    if visible != str(physical_index):
+        raise RuntimeError(
+            f"CUDA_VISIBLE_DEVICES must equal physical GPU {physical_index}, got {visible}")
+    return {
+        "physical_index": int(fields[0]), "uuid": fields[1], "name": fields[2],
+        "memory_total_mib": int(fields[3]), "cuda_visible_devices": visible,
+    }
+
+
+def ensure_launch(source, rows):
+    path = RESULT_ROOT / "launch.json"
+    expected = {
+        "stage": "contrastive_bias_c1", "source": source,
+        "registry_sha256": registry_sha256(rows), "num_jobs": len(rows),
+        "seeds": list(SEEDS), "conditions": [row[0] for row in CONDITIONS],
+        "allowed_physical_gpus": [5, 7], "test_policy": "none",
+    }
+    if path.is_file():
+        with open(path, encoding="utf-8") as handle:
+            if json.load(handle) != expected:
+                raise RuntimeError("C1 launch lock drift")
+    else:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        with open(path, "w", encoding="utf-8") as handle:
+            json.dump(expected, handle, indent=2, sort_keys=True)
+            handle.write("\n")
+    return path
+
+
+def summarize_hist(path):
+    hist = np.load(path, allow_pickle=True).item()
+    completed = int(hist["epochs_completed"])
+    keys = (
+        "val_loss", "val_accuracy", "train_loss", "train_accuracy",
+        "train_time", "val_time", "raw_bias_clean_difference_rms_ratio",
+        "post_bias_raw_bias_rms_ratio", "used_clean_difference_rms_ratio",
+        "maximum_used_clean_difference_relative_error", "neutral_observations",
+        "instruction_observations_for_predictor",
+    )
+    curves = {
+        key: [float(value) for value in np.asarray(hist[key])[:completed]]
+        for key in keys
+    }
+    finite = completed == 130 and all(
+        math.isfinite(value)
+        for key in ("val_loss", "val_accuracy", "train_loss")
+        for value in curves[key]
+    )
+    return {
+        "epochs_completed": completed, "finite": finite,
+        "final_validation_accuracy": curves["val_accuracy"][-1],
+        "best_validation_accuracy": float(hist["best_validation_accuracy"]),
+        "best_epoch": int(hist["best_epoch"]),
+        "test_accuracy": float(hist["test_accuracy"]),
+        "dp_bias_initialization": hist.get("dp_bias_initialization"),
+        "curves": curves,
+    }
+
+
+def find_hist(author_root, experiment_name):
+    paths = list((author_root / "runs" / experiment_name).glob("*/hist.npy"))
+    if len(paths) != 1:
+        raise RuntimeError(
+            f"expected one history for {experiment_name}, found {len(paths)}")
+    return paths[0]
+
+
+def run_job(job, author_root, source, registry_hash, gpu):
+    output = Path(job["output"])
+    if output.exists():
+        print(f"preserving s{job['seed']} {job['condition']}", flush=True)
+        return
+    if git_output(ROOT, "rev-parse", "HEAD") != source["sdil_commit"]:
+        raise RuntimeError("SDIL commit changed after C1 launch")
+    if git_output(author_root, "rev-parse", "HEAD") != source["author_commit"]:
+        raise RuntimeError("author commit changed after C1 launch")
+    print("RUN", " ".join(job["command"]), flush=True)
+    started = time.time()
+    try:
+        result = subprocess.run(
+            job["command"], cwd=author_root, timeout=job["timeout_seconds"])
+        return_code = result.returncode
+        status = "completed" if return_code == 0 else "nonzero_exit"
+    except subprocess.TimeoutExpired:
+        return_code, status = None, "timeout"
+    history = None
+    history_path = None
+    if status == "completed":
+        try:
+            resolved = find_hist(author_root, job["experiment_name"])
+            history_path = str(resolved)
+            history = summarize_hist(resolved)
+        except Exception as error:
+            status = "missing_or_invalid_history"
+            history = {"error": repr(error)}
+    record = {
+        **job, "stage": "contrastive_bias_c1", "source": source,
+        "registry_sha256": registry_hash, "hardware": gpu, "status": status,
+        "return_code": return_code, "driver_wall_seconds": time.time() - started,
+        "completed_unix_time": time.time(), "author_history": history_path,
+        "history": history,
+    }
+    output.parent.mkdir(parents=True, exist_ok=True)
+    with open(output, "w", encoding="utf-8") as handle:
+        json.dump(record, handle, indent=2, sort_keys=True)
+        handle.write("\n")
+    print(f"DONE status={status} s{job['seed']} {job['condition']}", flush=True)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--author-root", type=Path, required=True)
+    parser.add_argument("--author-python", required=True)
+    parser.add_argument("--physical-gpu", type=int, choices=(5, 7), required=True)
+    parser.add_argument("--seed-shard-index", type=int, choices=(0, 1), required=True)
+    parser.add_argument("--dry-run", action="store_true")
+    args = parser.parse_args()
+    args.author_root = args.author_root.resolve()
+    rows = jobs(args.author_python)
+    selected_seeds = SEEDS[args.seed_shard_index::2]
+    selected = [row for row in rows if row["seed"] in selected_seeds]
+    if args.dry_run:
+        for row in selected:
+            print(row["seed"], row["condition"], " ".join(row["command"]))
+        return
+    source = source_report(args.author_root)
+    gpu = gpu_report(args.physical_gpu)
+    launch = ensure_launch(source, rows)
+    print(f"C1 launch lock: {launch}", flush=True)
+    registry_hash = registry_sha256(rows)
+    for row in selected:
+        run_job(row, args.author_root, source, registry_hash, gpu)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/experiments/contrastive_bias_c1_smoke.py b/experiments/contrastive_bias_c1_smoke.py
new file mode 100644
index 0000000..eb66934
--- /dev/null
+++ b/experiments/contrastive_bias_c1_smoke.py
@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+"""Static checks for the frozen same-path C1 registry."""
+import json
+from pathlib import Path
+import sys
+
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT / "experiments"))
+from contrastive_bias_c1 import CONDITIONS, SEEDS, jobs, registry_sha256
+
+
+def main():
+    rows = jobs("/frozen/author/python")
+    assert len(rows) == 20
+    assert {row["seed"] for row in rows} == set(SEEDS)
+    assert {row["condition"] for row in rows} == {row[0] for row in CONDITIONS}
+    for seed in SEEDS:
+        seed_rows = [row for row in rows if row["seed"] == seed]
+        assert [row["condition"] for row in seed_rows] == [
+            row[0] for row in CONDITIONS]
+    for row in rows:
+        command = row["command"]
+        assert command[command.index("--seeds") + 1] == str(row["seed"])
+        assert command[command.index("--num-epochs") + 1] == "130"
+        assert command[command.index("--test-policy") + 1] == "none"
+        assert command[command.index("--dp-bias-ratio") + 1] == "4.0"
+        assert command[command.index("--dp-bias-kind") + 1] == row["kind"]
+        assert command[command.index("--dp-bias-rule") + 1] == row["rule"]
+    print(json.dumps({
+        "status": "passed", "num_cells": len(rows), "seeds": list(SEEDS),
+        "registry_sha256": registry_sha256(rows),
+    }, indent=2, sort_keys=True))
+
+
+if __name__ == "__main__":
+    main()