experiment: make Transformer 27-cell T2 executable

6 files changed, 801 insertions, 0 deletions

diff --git a/TRANSFORMER_CROSSOVER.md b/TRANSFORMER_CROSSOVER.md
index 8be1e72..019c0e6 100644
--- a/TRANSFORMER_CROSSOVER.md
+++ b/TRANSFORMER_CROSSOVER.md
@@ -181,6 +181,22 @@ T2 is a single-seed scaling panel, not the final uncertainty estimate.  An
 untouched multi-seed T3 registry must contain every one of the same 27 cells;
 winner-only or SDIL-only replication cannot support the scaling claim.
 
+The executable T2 contract is
+`experiments/transformer_crossover_t2.py`. It refuses an incomplete T1
+selector, requires the selector to be committed, binds its hash, selected
+rates, clean source, environment freeze, and GPU5/7 allowlist into a new launch
+lock, and enumerates all 27 method--depth cells before sharding. Expensive EP,
+Dual Propagation, and Forward--Forward cells are interleaved across shards.
+`experiments/analyze_transformer_crossover_t2.py` requires all manifests,
+retains timeout and nonfinite outcomes, rejects data/source/architecture/test
+drift, and checks token presentations, candidate evaluations, relaxation
+passes, local VJPs, logical queries, and enumerated MACs. The immutable
+registry can be checked before T1 finishes with:
+
+```bash
+python experiments/transformer_crossover_t2_smoke.py
+```
+
 ## Metrics and cost accounting
 
 The primary task metric is full-validation next-token NLL; perplexity and
diff --git a/experiments/analyze_transformer_crossover_t2.py b/experiments/analyze_transformer_crossover_t2.py
new file mode 100644
index 0000000..5eeb226
--- /dev/null
+++ b/experiments/analyze_transformer_crossover_t2.py
@@ -0,0 +1,323 @@
+#!/usr/bin/env python3
+"""Audit the complete failure-retaining 27-cell Transformer T2 panel."""
+import argparse
+import hashlib
+import json
+import math
+import os
+import sys
+
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, ROOT)
+from experiments.transformer_crossover_t2 import (
+    DEFAULT_SELECTOR,
+    DEPTHS,
+    METHODS,
+    registry_sha256,
+    selector_report,
+    t2_jobs,
+)
+
+
+TRAIN_HASH = (
+    "6ec305602a99ac2802745a134e1f5e33e2231b4855525b00b9aebb730ac2626f"
+)
+VALIDATION_HASH = (
+    "d37d30cc0c8327c270d493299c3dca54135f6d5f1c9ef60cda78076e311204b1"
+)
+EXPECTED_TRAIN_TOKENS = 5000 * 32 * 64
+EXPECTED_VALIDATION_TOKENS = 1742 * 64
+PARAMETERS = {4: 813_568, 8: 1_602_048, 12: 2_390_528}
+
+
+def sha256(path):
+    digest = hashlib.sha256()
+    with open(path, "rb") as handle:
+        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def read_json(path):
+    with open(path, encoding="utf-8") as handle:
+        return json.load(handle)
+
+
+def expected_work(record, job):
+    work = record["work"]
+    method = job["method"]
+    depth = job["depth"]
+    train = EXPECTED_TRAIN_TOKENS
+    validation = EXPECTED_VALIDATION_TOKENS
+    presentations = 2 * train if method in ("pepita", "ff") else train
+    candidate = 65 * validation if method == "ff" else 0
+    relaxation = 0
+    local_vjp = 0
+    if method == "dualprop":
+        relaxation = 16 * train
+        local_vjp = train * (depth + 1) * 16
+    elif method == "ep":
+        relaxation = 24 * train + 20 * validation
+        local_vjp = train * (depth + 1) * 24
+    assert work["ordinary_training_tokens"] == train
+    assert work["ordinary_validation_tokens"] == validation
+    assert work["training_token_presentations"] == presentations
+    assert work["candidate_token_presentations"] == candidate
+    assert work["relaxation_token_passes"] == relaxation
+    assert work["local_vjp_token_evaluations"] == local_vjp
+    assert work["logical_task_loss_queries"] == 0
+    assert work["completed_optimizer_steps"] == 5000
+    assert work["forward_parameter_count"] == PARAMETERS[depth]
+    assert work["full_forward_token_passes"] > 0
+    assert work["enumerated_full_forward_macs"] > 0
+    if method in ("dfa", "pepita", "ff", "dualprop", "ep"):
+        assert (
+            work["local_block_token_evaluations"] > 0
+            or method == "ff"
+        )
+    return work
+
+
+def audit_completed(job, manifest, source):
+    assert manifest["output_exists"] is True
+    assert manifest["output_sha256"] == sha256(job["output"])
+    record = read_json(job["output"])
+    provenance = record["provenance"]
+    assert provenance["git_commit"] == source["git_commit"]
+    assert provenance["git_tracked_dirty"] is False
+    args = record["args"]
+    expected_args = {
+        "method": job["method"],
+        "depth": job["depth"],
+        "width": 128,
+        "heads": 4,
+        "mlp_ratio": 4,
+        "context_length": 64,
+        "batch_size": 32,
+        "eval_batch_size": 32,
+        "train_steps": 5000,
+        "lr": job["rate"],
+        "schedule": "cosine",
+        "min_lr": job["rate"] * 0.1,
+        "warmup_steps": 100,
+        "weight_decay": 0.1,
+        "run_seed": 0,
+        "model_seed": 2027,
+        "loader_seed": 0,
+        "negative_seed": 5001,
+        "ep_sign_seed": 5002,
+        "traffic_ratio": 4.0,
+        "ff_threshold": 2.0,
+        "ff_score_from_layer": 1,
+        "ep_beta": 0.5,
+        "ep_dt": 0.5,
+        "ep_free_steps": 20,
+        "ep_nudge_steps": 4,
+        "dp_alpha": 0.0,
+        "dp_beta": 0.1,
+        "dp_inference_passes": 16,
+        "eval_every": 0,
+        "max_val_batches": 0,
+    }
+    for key, expected in expected_args.items():
+        assert args[key] == expected, (
+            f"{job['experiment_name']}: argument drift in {key}"
+        )
+    assert record["protocol_family"] == (
+        "transformer_local_learning_crossover"
+    )
+    dataset = record["dataset"]
+    assert dataset["train_sha256"] == TRAIN_HASH
+    assert dataset["validation_sha256"] == VALIDATION_HASH
+    architecture = record["architecture"]
+    assert architecture["depth"] == job["depth"]
+    assert architecture["width"] == 128
+    assert architecture["heads"] == 4
+    assert architecture["context_length"] == 64
+    assert architecture["forward_parameter_count"] == PARAMETERS[job["depth"]]
+    evaluation = record["evaluation_protocol"]
+    assert evaluation["split"] == "validation"
+    assert evaluation["test_evaluations"] == 0
+    assert evaluation["test_used_for_selection"] is False
+    assert len(record["validation"]) == 1
+    final = record["final"]
+    assert final["tokens"] == EXPECTED_VALIDATION_TOKENS
+    assert final["step"] <= 5000
+    first_nonfinite = record["first_nonfinite_step"]
+    nll = float(final["nll"])
+    perplexity = float(final["perplexity"])
+    accuracy = float(final["accuracy"])
+    is_finite = (
+        first_nonfinite is None
+        and final["step"] == 5000
+        and math.isfinite(nll)
+        and math.isfinite(perplexity)
+        and math.isfinite(accuracy)
+    )
+    work = None
+    if is_finite:
+        work = expected_work(record, job)
+    else:
+        assert record["work"]["ordinary_training_tokens"] <= (
+            EXPECTED_TRAIN_TOKENS
+        )
+        assert record["work"]["logical_task_loss_queries"] == 0
+    hardware = record["hardware"]
+    assert hardware["cuda_visible_devices"] in ("5", "7")
+    assert hardware["device_name"] == "NVIDIA GeForce GTX 1080"
+    assert hardware["peak_memory_allocated_bytes"] is not None
+    assert hardware["peak_memory_reserved_bytes"] is not None
+    return {
+        "finite": is_finite,
+        "first_nonfinite_step": first_nonfinite,
+        "completed_optimizer_steps":
+            int(record["work"]["completed_optimizer_steps"]),
+        "final_validation_nll": nll if math.isfinite(nll) else None,
+        "final_validation_perplexity":
+            perplexity if math.isfinite(perplexity) else None,
+        "final_validation_accuracy":
+            accuracy if math.isfinite(accuracy) else None,
+        "forward_parameter_count":
+            int(architecture["forward_parameter_count"]),
+        "feedback_parameter_count":
+            int(architecture["feedback_parameter_count"]),
+        "peak_memory_allocated_bytes":
+            int(hardware["peak_memory_allocated_bytes"]),
+        "total_wall_seconds": float(record["total_wall_seconds"]),
+        "work": work if work is not None else record["work"],
+    }
+
+
+def audit_job(job, source, selector):
+    manifest_path = job["output"] + ".manifest.json"
+    if not os.path.isfile(manifest_path):
+        raise AssertionError(f"missing T2 manifest: {job['experiment_name']}")
+    manifest = read_json(manifest_path)
+    for key in (
+        "stage",
+        "method",
+        "architecture",
+        "depth",
+        "rate",
+        "experiment_name",
+        "output",
+        "timeout_seconds",
+        "command",
+    ):
+        assert manifest[key] == job[key], (
+            f"{job['experiment_name']}: manifest drift in {key}"
+        )
+    assert manifest["source"] == source
+    assert manifest["selector"] == selector
+    hardware = manifest["hardware_lock"]
+    assert hardware["physical_gpu_index"] in (5, 7)
+    assert hardware["physical_gpu_uuid"]
+    common = {
+        "cell_id": f"transformer{job['depth']}::{job['method']}",
+        "method": job["method"],
+        "depth": job["depth"],
+        "rate": job["rate"],
+        "status": manifest["status"],
+        "manifest": os.path.relpath(manifest_path, ROOT),
+        "driver_wall_seconds": float(manifest["driver_wall_seconds"]),
+        "physical_gpu_index": hardware["physical_gpu_index"],
+        "physical_gpu_uuid": hardware["physical_gpu_uuid"],
+        "output_sha256": manifest["output_sha256"],
+    }
+    if manifest["status"] == "completed":
+        return {
+            **common,
+            **audit_completed(job, manifest, source),
+        }
+    assert manifest["status"] in {
+        "timeout", "nonzero_exit", "missing_output"
+    }
+    if manifest["output_exists"]:
+        assert manifest["output_sha256"] == sha256(job["output"])
+    else:
+        assert manifest["output_sha256"] is None
+    return {
+        **common,
+        "finite": False,
+        "first_nonfinite_step": None,
+        "completed_optimizer_steps": None,
+        "final_validation_nll": None,
+        "final_validation_perplexity": None,
+        "final_validation_accuracy": None,
+        "forward_parameter_count": None,
+        "feedback_parameter_count": None,
+        "peak_memory_allocated_bytes": None,
+        "total_wall_seconds": None,
+        "work": None,
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--selector", default=DEFAULT_SELECTOR)
+    parser.add_argument(
+        "--out", default="results/transformer_crossover/t2_audit.json"
+    )
+    args = parser.parse_args()
+    selector = selector_report(args.selector)
+    jobs = t2_jobs(selector["selected_rates"])
+    launch_path = os.path.join(
+        ROOT, "results", "transformer_crossover", "t2_launch.json"
+    )
+    assert os.path.isfile(launch_path), "missing Transformer T2 launch lock"
+    launch = read_json(launch_path)
+    assert launch["stage"] == "t2"
+    assert launch["selector"] == selector
+    assert launch["registry_sha256"] == registry_sha256(jobs)
+    assert launch["num_jobs"] == 27
+    assert launch["allowed_physical_gpus"] == [5, 7]
+    source = launch["source"]
+    records = [audit_job(job, source, selector) for job in jobs]
+    assert len(records) == 27
+    assert len({record["cell_id"] for record in records}) == 27
+    assert {
+        (record["method"], record["depth"]) for record in records
+    } == {(method, depth) for method in METHODS for depth in DEPTHS}
+    failures = [
+        record["cell_id"] for record in records if not record["finite"]
+    ]
+    report = {
+        "audit_status": "passed",
+        "stage": "transformer_crossover_t2",
+        "complete_grid": True,
+        "failure_retaining": True,
+        "num_expected_cells": 27,
+        "num_audited_cells": len(records),
+        "num_finite_cells": len(records) - len(failures),
+        "failed_or_incomplete_cells": failures,
+        "test_policy": "none",
+        "source": source,
+        "selector": selector,
+        "launch_lock": {
+            "path": os.path.relpath(launch_path, ROOT),
+            "sha256": sha256(launch_path),
+            "registry_sha256": launch["registry_sha256"],
+        },
+        "records": records,
+    }
+    os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True)
+    with open(args.out, "w", encoding="utf-8") as handle:
+        json.dump(report, handle, indent=2, sort_keys=True)
+        handle.write("\n")
+    print(
+        json.dumps(
+            {
+                "audit_status": report["audit_status"],
+                "num_audited_cells": len(records),
+                "num_finite_cells": report["num_finite_cells"],
+                "failed_or_incomplete_cells": failures,
+            },
+            indent=2,
+            sort_keys=True,
+        )
+    )
+
+
+if __name__ == "__main__":
+    main()
diff --git a/experiments/finalize_accept.sh b/experiments/finalize_accept.sh
index 9a511b9..e445caa 100755
--- a/experiments/finalize_accept.sh
+++ b/experiments/finalize_accept.sh
@@ -25,6 +25,8 @@ experiments/finalize_claims.sh
   experiments/oral_a_dynamic_scaling_smoke.py
 /home/yurenh2/miniconda3/envs/ep_pascal/bin/python3 \
   experiments/resnet_crossover_r2_smoke.py
+/home/yurenh2/miniconda3/envs/ep_pascal/bin/python3 \
+  experiments/transformer_crossover_t2_smoke.py
 /home/yurenh2/miniconda3/envs/ep_pascal/bin/python3 -m py_compile \
   experiments/bci_td_run.py experiments/analyze_bci_td_development.py \
   experiments/bci_td_confirmation.py \
@@ -42,6 +44,8 @@ experiments/finalize_claims.sh
   experiments/analyze_oral_a_dynamic_scaling_v2.py \
   experiments/resnet_crossover_r2.py \
   experiments/analyze_resnet_crossover_r2.py \
+  experiments/transformer_crossover_t2.py \
+  experiments/analyze_transformer_crossover_t2.py \
   experiments/plot_resnet_confirmation.py \
   experiments/plot_oral_a_scaling.py \
   experiments/plot_bci_v2_confirmation.py \
diff --git a/experiments/transformer_crossover_native.py b/experiments/transformer_crossover_native.py
index 7ccb8fb..ca47751 100644
--- a/experiments/transformer_crossover_native.py
+++ b/experiments/transformer_crossover_native.py
@@ -365,6 +365,7 @@ def work_report(
         "candidate_token_presentations": sum(
             row["candidate_token_presentations"]
             for row in validation_evaluations),
+        "logical_task_loss_queries": 0,
         "relaxation_token_passes": relaxation,
         "local_vjp_token_evaluations": local_vjps,
         "full_forward_token_passes": full_forward_token_passes,
diff --git a/experiments/transformer_crossover_t2.py b/experiments/transformer_crossover_t2.py
new file mode 100644
index 0000000..17f61a2
--- /dev/null
+++ b/experiments/transformer_crossover_t2.py
@@ -0,0 +1,392 @@
+#!/usr/bin/env python3
+"""Immutable driver for the complete 27-cell Transformer T2 crossover."""
+import argparse
+import hashlib
+import json
+import os
+import subprocess
+import sys
+import time
+
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+PROTOCOL = os.path.join(ROOT, "TRANSFORMER_CROSSOVER.md")
+RESULT_ROOT = os.path.join(ROOT, "results", "transformer_crossover")
+DEFAULT_SELECTOR = os.path.join(RESULT_ROOT, "p1_selector.json")
+METHODS = (
+    "bp", "fa", "dfa", "pepita", "ff", "ep", "dualprop",
+    "clean_kp", "sdil",
+)
+DEPTHS = (4, 8, 12)
+ORDER = (
+    ("ep", 12),
+    ("dualprop", 12),
+    ("ff", 12),
+    ("ep", 8),
+    ("dualprop", 8),
+    ("ff", 8),
+    ("ep", 4),
+    ("dualprop", 4),
+    ("ff", 4),
+    ("pepita", 12),
+    ("sdil", 12),
+    ("clean_kp", 12),
+    ("fa", 12),
+    ("dfa", 12),
+    ("bp", 12),
+    ("pepita", 8),
+    ("sdil", 8),
+    ("clean_kp", 8),
+    ("fa", 8),
+    ("dfa", 8),
+    ("bp", 8),
+    ("pepita", 4),
+    ("sdil", 4),
+    ("clean_kp", 4),
+    ("fa", 4),
+    ("dfa", 4),
+    ("bp", 4),
+)
+
+
+def sha256(path):
+    digest = hashlib.sha256()
+    with open(path, "rb") as handle:
+        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def text_sha256(text):
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
+
+def read_json(path):
+    with open(path, encoding="utf-8") as handle:
+        return json.load(handle)
+
+
+def git_output(*args):
+    return subprocess.run(
+        ["git", *args],
+        cwd=ROOT,
+        check=True,
+        capture_output=True,
+        text=True,
+    ).stdout.strip()
+
+
+def rate_tag(rate):
+    return f"{rate:g}".replace(".", "p")
+
+
+def selector_report(path):
+    selector = read_json(path)
+    if not (
+        selector.get("gate") == "pass"
+        and selector.get("stage") == "transformer_crossover_p1"
+        and selector.get("num_expected_records") == 27
+        and selector.get("num_audited_records") == 27
+        and selector.get("missing_experiments") == []
+        and selector.get("test_policy") == "none"
+        and set(selector.get("selected", {})) == set(METHODS)
+    ):
+        raise RuntimeError(
+            "Transformer T2 requires the complete passed T1 selector"
+        )
+    rates = {}
+    for method in METHODS:
+        rate = float(selector["selected"][method]["rate"])
+        if not rate > 0:
+            raise RuntimeError(f"invalid selected Transformer rate: {method}")
+        rates[method] = rate
+    return {
+        "path": os.path.relpath(os.path.abspath(path), ROOT),
+        "sha256": sha256(path),
+        "p1_source": selector["source"],
+        "selected_rates": rates,
+    }
+
+
+def source_report(selector_path):
+    if git_output("status", "--porcelain", "--untracked-files=no"):
+        raise RuntimeError("Transformer T2 requires clean tracked source")
+    paths = [
+        PROTOCOL,
+        os.path.abspath(__file__),
+        os.path.join(
+            ROOT, "experiments", "analyze_transformer_crossover_t2.py"
+        ),
+        os.path.join(
+            ROOT, "experiments", "transformer_crossover_native.py"
+        ),
+        os.path.join(
+            ROOT, "experiments", "transformer_feedback_smoke.py"
+        ),
+        os.path.join(ROOT, "sdil", "transformer.py"),
+        os.path.abspath(selector_path),
+    ]
+    for path in paths:
+        relative = os.path.relpath(path, ROOT)
+        tracked = subprocess.run(
+            ["git", "ls-files", "--error-unmatch", relative],
+            cwd=ROOT,
+            capture_output=True,
+        ).returncode == 0
+        if not tracked:
+            raise RuntimeError(f"untracked Transformer T2 source: {relative}")
+    freeze = subprocess.run(
+        [sys.executable, "-m", "pip", "freeze"],
+        check=True,
+        capture_output=True,
+        text=True,
+    ).stdout
+    return {
+        "git_commit": git_output("rev-parse", "HEAD"),
+        "protocol_path": PROTOCOL,
+        "protocol_sha256": sha256(PROTOCOL),
+        "tracked_files": {
+            os.path.relpath(path, ROOT): sha256(path) for path in paths
+        },
+        "python": sys.executable,
+        "environment_freeze_sha256": text_sha256(freeze),
+        "environment_freeze": freeze.splitlines(),
+    }
+
+
+def t2_command(method, depth, rate):
+    name = f"transformer-t2-{method}-d{depth}-lr{rate_tag(rate)}"
+    output = os.path.join(RESULT_ROOT, "t2", name + ".json")
+    command = [
+        sys.executable,
+        "experiments/transformer_crossover_native.py",
+        "--method", method,
+        "--out", output,
+        "--device", "cuda",
+        "--depth", str(depth),
+        "--width", "128",
+        "--heads", "4",
+        "--mlp_ratio", "4",
+        "--context_length", "64",
+        "--batch_size", "32",
+        "--eval_batch_size", "32",
+        "--train_steps", "5000",
+        "--lr", str(rate),
+        "--schedule", "cosine",
+        "--min_lr", str(rate * 0.1),
+        "--warmup_steps", "100",
+        "--weight_decay", "0.1",
+        "--run_seed", "0",
+        "--model_seed", "2027",
+        "--loader_seed", "0",
+        "--negative_seed", "5001",
+        "--ep_sign_seed", "5002",
+        "--traffic_ratio", "4.0",
+        "--ff_threshold", "2.0",
+        "--ff_score_from_layer", "1",
+        "--ep_beta", "0.5",
+        "--ep_dt", "0.5",
+        "--ep_free_steps", "20",
+        "--ep_nudge_steps", "4",
+        "--dp_alpha", "0.0",
+        "--dp_beta", "0.1",
+        "--dp_inference_passes", "16",
+        "--eval_every", "0",
+        "--log_every", "50",
+        "--max_val_batches", "0",
+    ]
+    return {
+        "stage": "t2",
+        "method": method,
+        "architecture": f"transformer{depth}",
+        "depth": depth,
+        "rate": rate,
+        "experiment_name": name,
+        "output": output,
+        "timeout_seconds": 48 * 60 * 60,
+        "command": command,
+    }
+
+
+def t2_jobs(selected_rates):
+    if set(selected_rates) != set(METHODS):
+        raise ValueError("T2 requires one selected rate for every method")
+    jobs = [
+        t2_command(method, depth, float(selected_rates[method]))
+        for method, depth in ORDER
+    ]
+    identifiers = [job["experiment_name"] for job in jobs]
+    cells = {(job["method"], job["depth"]) for job in jobs}
+    if (
+        len(jobs) != 27
+        or len(set(identifiers)) != 27
+        or cells != {(method, depth) for method in METHODS for depth in DEPTHS}
+    ):
+        raise AssertionError(
+            "Transformer T2 registry must contain 27 unique cells"
+        )
+    return jobs
+
+
+def registry_sha256(jobs):
+    encoded = json.dumps(
+        jobs, sort_keys=True, separators=(",", ":")
+    ).encode("utf-8")
+    return hashlib.sha256(encoded).hexdigest()
+
+
+def ensure_launch(source, selector, jobs):
+    path = os.path.join(RESULT_ROOT, "t2_launch.json")
+    expected = {
+        "stage": "t2",
+        "source": source,
+        "selector": selector,
+        "registry_sha256": registry_sha256(jobs),
+        "num_jobs": len(jobs),
+        "allowed_physical_gpus": [5, 7],
+    }
+    if os.path.isfile(path):
+        if read_json(path) != expected:
+            raise RuntimeError("Transformer T2 launch lock drift")
+        return path
+    os.makedirs(os.path.dirname(path), exist_ok=True)
+    with open(path, "w", encoding="utf-8") as handle:
+        json.dump(expected, handle, indent=2, sort_keys=True)
+        handle.write("\n")
+    return path
+
+
+def assert_source_unchanged(source):
+    if git_output("status", "--porcelain", "--untracked-files=no"):
+        raise RuntimeError("tracked source changed after Transformer T2 launch")
+    if git_output("rev-parse", "HEAD") != source["git_commit"]:
+        raise RuntimeError("source commit changed after Transformer T2 launch")
+    for relative, expected_hash in source["tracked_files"].items():
+        if sha256(os.path.join(ROOT, relative)) != expected_hash:
+            raise RuntimeError(f"Transformer T2 source drift: {relative}")
+    freeze = subprocess.run(
+        [sys.executable, "-m", "pip", "freeze"],
+        check=True,
+        capture_output=True,
+        text=True,
+    ).stdout
+    if text_sha256(freeze) != source["environment_freeze_sha256"]:
+        raise RuntimeError("Transformer T2 environment drift")
+
+
+def physical_gpu_report(dry_run=False):
+    visible = os.environ.get("CUDA_VISIBLE_DEVICES")
+    if dry_run:
+        return {
+            "cuda_visible_devices": visible,
+            "physical_gpu_index": None,
+            "physical_gpu_uuid": None,
+        }
+    if visible not in {"5", "7"}:
+        raise RuntimeError("Transformer T2 requires physical GPU 5 or 7")
+    query = subprocess.run(
+        [
+            "nvidia-smi",
+            "--query-gpu=index,uuid,name",
+            "--format=csv,noheader,nounits",
+        ],
+        check=True,
+        capture_output=True,
+        text=True,
+    ).stdout.splitlines()
+    rows = {}
+    for line in query:
+        index, uuid, name = [value.strip() for value in line.split(",", 2)]
+        rows[index] = {"uuid": uuid, "name": name}
+    if visible not in rows:
+        raise RuntimeError(f"physical GPU {visible} not found")
+    return {
+        "cuda_visible_devices": visible,
+        "physical_gpu_index": int(visible),
+        "physical_gpu_uuid": rows[visible]["uuid"],
+        "physical_gpu_name": rows[visible]["name"],
+    }
+
+
+def run_job(job, source, selector, gpu, dry_run):
+    if not dry_run:
+        assert_source_unchanged(source)
+        if sha256(os.path.join(ROOT, selector["path"])) != selector["sha256"]:
+            raise RuntimeError("Transformer T2 selector changed after launch")
+    manifest_path = job["output"] + ".manifest.json"
+    if os.path.exists(manifest_path):
+        print(f"preserving {job['experiment_name']}", flush=True)
+        return
+    if os.path.exists(job["output"]):
+        raise RuntimeError(f"orphaned T2 output requires audit: {job['output']}")
+    print("RUN", " ".join(job["command"]), flush=True)
+    if dry_run:
+        return
+    os.makedirs(os.path.dirname(job["output"]), exist_ok=True)
+    started = time.time()
+    try:
+        result = subprocess.run(
+            job["command"], cwd=ROOT, timeout=job["timeout_seconds"]
+        )
+        return_code = result.returncode
+        status = "completed" if return_code == 0 else "nonzero_exit"
+    except subprocess.TimeoutExpired:
+        return_code = None
+        status = "timeout"
+    output_exists = os.path.isfile(job["output"])
+    if status == "completed" and not output_exists:
+        status = "missing_output"
+    manifest = {
+        **job,
+        "source": source,
+        "selector": selector,
+        "hardware_lock": gpu,
+        "status": status,
+        "return_code": return_code,
+        "output_exists": output_exists,
+        "output_sha256": sha256(job["output"]) if output_exists else None,
+        "driver_wall_seconds": time.time() - started,
+        "completed_unix_time": time.time(),
+    }
+    with open(manifest_path, "w", encoding="utf-8") as handle:
+        json.dump(manifest, handle, indent=2, sort_keys=True)
+        handle.write("\n")
+    print(f"DONE status={status} {job['experiment_name']}", flush=True)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--selector", default=DEFAULT_SELECTOR)
+    parser.add_argument("--shard-index", type=int, default=0)
+    parser.add_argument("--num-shards", type=int, default=1)
+    parser.add_argument("--method", choices=METHODS)
+    parser.add_argument("--depth", type=int, choices=DEPTHS)
+    parser.add_argument("--dry-run", action="store_true")
+    args = parser.parse_args()
+    if not 0 <= args.shard_index < args.num_shards:
+        raise ValueError("invalid Transformer T2 shard")
+    selector = selector_report(args.selector)
+    source = source_report(args.selector)
+    gpu = physical_gpu_report(args.dry_run)
+    complete_jobs = t2_jobs(selector["selected_rates"])
+    if not args.dry_run:
+        launch = ensure_launch(source, selector, complete_jobs)
+        print(f"T2 launch lock: {launch}", flush=True)
+    jobs = complete_jobs
+    if args.method:
+        jobs = [job for job in jobs if job["method"] == args.method]
+    if args.depth is not None:
+        jobs = [job for job in jobs if job["depth"] == args.depth]
+    jobs = [
+        job
+        for index, job in enumerate(jobs)
+        if index % args.num_shards == args.shard_index
+    ]
+    if not jobs:
+        raise ValueError("no Transformer T2 jobs selected")
+    for job in jobs:
+        run_job(job, source, selector, gpu, args.dry_run)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/experiments/transformer_crossover_t2_smoke.py b/experiments/transformer_crossover_t2_smoke.py
new file mode 100644
index 0000000..febca40
--- /dev/null
+++ b/experiments/transformer_crossover_t2_smoke.py
@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+"""Deterministic audit of the complete Transformer T2 registry."""
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from experiments.transformer_crossover_t2 import DEPTHS, METHODS, t2_jobs
+
+
+RATES = {
+    "bp": 0.001,
+    "fa": 0.001,
+    "dfa": 0.001,
+    "pepita": 0.0003,
+    "ff": 0.0003,
+    "ep": 0.0003,
+    "dualprop": 0.0003,
+    "clean_kp": 0.001,
+    "sdil": 0.001,
+}
+
+
+def value(command, flag):
+    index = command.index(flag)
+    return command[index + 1]
+
+
+def main():
+    jobs = t2_jobs(RATES)
+    assert len(jobs) == 27
+    assert {(job["method"], job["depth"]) for job in jobs} == {
+        (method, depth) for method in METHODS for depth in DEPTHS
+    }
+    assert len({job["experiment_name"] for job in jobs}) == 27
+    assert len({job["output"] for job in jobs}) == 27
+    for job in jobs:
+        command = job["command"]
+        rate = RATES[job["method"]]
+        assert value(command, "--depth") == str(job["depth"])
+        assert value(command, "--train_steps") == "5000"
+        assert value(command, "--batch_size") == "32"
+        assert value(command, "--context_length") == "64"
+        assert value(command, "--lr") == str(rate)
+        assert value(command, "--schedule") == "cosine"
+        assert value(command, "--min_lr") == str(rate * 0.1)
+        assert value(command, "--warmup_steps") == "100"
+        assert value(command, "--eval_every") == "0"
+        assert value(command, "--max_val_batches") == "0"
+        assert job["timeout_seconds"] == 48 * 60 * 60
+        assert "--test" not in command
+        assert value(command, "--traffic_ratio") == "4.0"
+        assert value(command, "--ep_free_steps") == "20"
+        assert value(command, "--ep_nudge_steps") == "4"
+        assert value(command, "--dp_inference_passes") == "16"
+    shard0 = jobs[0::2]
+    shard1 = jobs[1::2]
+    assert len(shard0) == 14 and len(shard1) == 13
+    for method in ("ep", "dualprop", "ff"):
+        assert any(job["method"] == method for job in shard0)
+        assert any(job["method"] == method for job in shard1)
+    print("Transformer T2 registry: 27/27 cells; schedules and shards exact")
+
+
+if __name__ == "__main__":
+    main()