experiment: package portable A6000 crossover

10 files changed, 542 insertions, 93 deletions

diff --git a/experiments/analyze_resnet_crossover_r2.py b/experiments/analyze_resnet_crossover_r2.py
index 3c2bf46..ce89165 100644
--- a/experiments/analyze_resnet_crossover_r2.py
+++ b/experiments/analyze_resnet_crossover_r2.py
@@ -18,6 +18,7 @@ from experiments.resnet_crossover_r2 import (
     registry_sha256,
     selector_report,
 )
+from experiments.crossover_hardware import assert_hardware_report
 
 
 def sha256(path):
@@ -216,7 +217,9 @@ def work_metrics(record, job, history):
     }
 
 
-def audit_job(job, expected_source, expected_selector):
+def audit_job(
+    job, expected_source, expected_selector, hardware_policy
+):
     manifest_path = job["output"] + ".manifest.json"
     if not os.path.isfile(manifest_path):
         raise AssertionError(f"missing R2 manifest: {job['experiment_name']}")
@@ -240,8 +243,7 @@ def audit_job(job, expected_source, expected_selector):
     assert manifest["source"] == expected_source
     assert manifest["selector"] == expected_selector
     hardware = manifest["hardware_lock"]
-    assert hardware["physical_gpu_index"] in (5, 7)
-    assert hardware["physical_gpu_uuid"]
+    assert_hardware_report(hardware, hardware_policy)
     common = {
         "cell_id": f"resnet{job['depth']}::{job['method']}",
         "method": job["method"],
@@ -331,8 +333,14 @@ def audit_job(job, expected_source, expected_selector):
             expected_ordinary *= job["depth"]
         assert work["ordinary_training_examples"] == expected_ordinary
     hardware_record = record["hardware"]
-    assert hardware_record["cuda_visible_devices"] in ("5", "7")
-    assert hardware_record["cuda_device_name"] == "NVIDIA GeForce GTX 1080"
+    assert (
+        hardware_record["cuda_visible_devices"]
+        == hardware["cuda_visible_devices"]
+    )
+    assert (
+        hardware_record["cuda_device_name"]
+        == hardware["physical_gpu_name"]
+    )
     parameter_count = architecture.get("forward_parameter_count")
     if parameter_count is None:
         parameter_count = architecture["forward_parameters"]
@@ -370,9 +378,15 @@ def main():
     assert launch["selector"] == selector
     assert launch["registry_sha256"] == registry_sha256(jobs)
     assert launch["num_jobs"] == 27
-    assert launch["allowed_physical_gpus"] == [5, 7]
+    hardware_policy = launch["hardware_policy"]
+    assert (
+        launch["allowed_physical_gpus"]
+        == hardware_policy["allowed_physical_gpu_indices"]
+    )
     source = launch["source"]
-    records = [audit_job(job, source, selector) for job in jobs]
+    records = [
+        audit_job(job, source, selector, hardware_policy) for job in jobs
+    ]
     assert len(records) == 27
     assert len({record["cell_id"] for record in records}) == 27
     assert {
@@ -401,6 +415,7 @@ def main():
         "test_policy": "none",
         "source": source,
         "selector": selector,
+        "hardware_policy": hardware_policy,
         "launch_lock": {
             "path": os.path.relpath(launch_path, ROOT),
             "sha256": sha256(launch_path),
diff --git a/experiments/analyze_transformer_crossover_t2.py b/experiments/analyze_transformer_crossover_t2.py
index 5eeb226..74c3ef6 100644
--- a/experiments/analyze_transformer_crossover_t2.py
+++ b/experiments/analyze_transformer_crossover_t2.py
@@ -18,6 +18,7 @@ from experiments.transformer_crossover_t2 import (
     selector_report,
     t2_jobs,
 )
+from experiments.crossover_hardware import assert_hardware_report
 
 
 TRAIN_HASH = (
@@ -79,7 +80,7 @@ def expected_work(record, job):
     return work
 
 
-def audit_completed(job, manifest, source):
+def audit_completed(job, manifest, source, hardware_policy):
     assert manifest["output_exists"] is True
     assert manifest["output_sha256"] == sha256(job["output"])
     record = read_json(job["output"])
@@ -164,8 +165,13 @@ def audit_completed(job, manifest, source):
         )
         assert record["work"]["logical_task_loss_queries"] == 0
     hardware = record["hardware"]
-    assert hardware["cuda_visible_devices"] in ("5", "7")
-    assert hardware["device_name"] == "NVIDIA GeForce GTX 1080"
+    locked_hardware = manifest["hardware_lock"]
+    assert_hardware_report(locked_hardware, hardware_policy)
+    assert (
+        hardware["cuda_visible_devices"]
+        == locked_hardware["cuda_visible_devices"]
+    )
+    assert hardware["device_name"] == locked_hardware["physical_gpu_name"]
     assert hardware["peak_memory_allocated_bytes"] is not None
     assert hardware["peak_memory_reserved_bytes"] is not None
     return {
@@ -189,7 +195,7 @@ def audit_completed(job, manifest, source):
     }
 
 
-def audit_job(job, source, selector):
+def audit_job(job, source, selector, hardware_policy):
     manifest_path = job["output"] + ".manifest.json"
     if not os.path.isfile(manifest_path):
         raise AssertionError(f"missing T2 manifest: {job['experiment_name']}")
@@ -211,8 +217,7 @@ def audit_job(job, source, selector):
     assert manifest["source"] == source
     assert manifest["selector"] == selector
     hardware = manifest["hardware_lock"]
-    assert hardware["physical_gpu_index"] in (5, 7)
-    assert hardware["physical_gpu_uuid"]
+    assert_hardware_report(hardware, hardware_policy)
     common = {
         "cell_id": f"transformer{job['depth']}::{job['method']}",
         "method": job["method"],
@@ -228,7 +233,7 @@ def audit_job(job, source, selector):
     if manifest["status"] == "completed":
         return {
             **common,
-            **audit_completed(job, manifest, source),
+            **audit_completed(job, manifest, source, hardware_policy),
         }
     assert manifest["status"] in {
         "timeout", "nonzero_exit", "missing_output"
@@ -271,9 +276,15 @@ def main():
     assert launch["selector"] == selector
     assert launch["registry_sha256"] == registry_sha256(jobs)
     assert launch["num_jobs"] == 27
-    assert launch["allowed_physical_gpus"] == [5, 7]
+    hardware_policy = launch["hardware_policy"]
+    assert (
+        launch["allowed_physical_gpus"]
+        == hardware_policy["allowed_physical_gpu_indices"]
+    )
     source = launch["source"]
-    records = [audit_job(job, source, selector) for job in jobs]
+    records = [
+        audit_job(job, source, selector, hardware_policy) for job in jobs
+    ]
     assert len(records) == 27
     assert len({record["cell_id"] for record in records}) == 27
     assert {
@@ -294,6 +305,7 @@ def main():
         "test_policy": "none",
         "source": source,
         "selector": selector,
+        "hardware_policy": hardware_policy,
         "launch_lock": {
             "path": os.path.relpath(launch_path, ROOT),
             "sha256": sha256(launch_path),
diff --git a/experiments/bootstrap_plain_cnn.sh b/experiments/bootstrap_plain_cnn.sh
new file mode 100755
index 0000000..0d6145f
--- /dev/null
+++ b/experiments/bootstrap_plain_cnn.sh
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ "$#" -ne 1 ]]; then
+  echo "usage: $0 /absolute/path/to/dualprop-sdil" >&2
+  exit 2
+fi
+
+target="$1"
+if [[ "$target" != /* ]]; then
+  echo "target must be an absolute path" >&2
+  exit 2
+fi
+if [[ -e "$target" ]]; then
+  echo "refusing to overwrite existing target: $target" >&2
+  exit 2
+fi
+
+main_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+patch_root="$main_root/external/dualprop_patches"
+base_revision="7b2595b34421e1483a721dbfdeff8cdabda3a1ff"
+
+if [[ "$(find "$patch_root" -maxdepth 1 -name '*.patch' | wc -l)" -ne 19 ]]; then
+  echo "expected 19 frozen Dual Propagation patches" >&2
+  exit 1
+fi
+
+git clone https://github.com/Rasmuskh/dualprop_icml_2024.git "$target"
+git -C "$target" checkout -b sdil-a6000 "$base_revision"
+git -C "$target" config user.name "SDIL replication runner"
+git -C "$target" config user.email "sdil-replication@invalid.example"
+git -C "$target" am --committer-date-is-author-date "$patch_root"/*.patch
+
+echo "plain-CNN runner ready at $target"
+git -C "$target" log -1 --oneline
diff --git a/experiments/crossover_hardware.py b/experiments/crossover_hardware.py
new file mode 100644
index 0000000..6ace655
--- /dev/null
+++ b/experiments/crossover_hardware.py
@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+"""Hardware profiles shared by portable crossover launchers and audits."""
+import os
+import subprocess
+
+
+PROFILES = {
+    "timan107_gtx1080": {
+        "allowed_physical_gpu_indices": [5, 7],
+        "allowed_device_names": ["NVIDIA GeForce GTX 1080"],
+    },
+    "a6000": {
+        "allowed_physical_gpu_indices": None,
+        "allowed_device_names": ["NVIDIA RTX A6000"],
+    },
+}
+DEFAULT_PROFILE = "timan107_gtx1080"
+
+
+def profile_choices():
+    return tuple(PROFILES)
+
+
+def policy(profile, physical_gpu_index=None):
+    if profile not in PROFILES:
+        raise ValueError(f"unknown crossover hardware profile: {profile}")
+    configured = PROFILES[profile]
+    allowed = configured["allowed_physical_gpu_indices"]
+    if allowed is None:
+        if physical_gpu_index is None:
+            raise ValueError(
+                f"{profile} policy requires a resolved physical GPU index"
+            )
+        allowed = [int(physical_gpu_index)]
+    return {
+        "profile": profile,
+        "allowed_physical_gpu_indices": list(allowed),
+        "allowed_device_names": list(configured["allowed_device_names"]),
+        "single_gpu_per_process": True,
+    }
+
+
+def _query_physical_gpus():
+    query = subprocess.run(
+        [
+            "nvidia-smi",
+            "--query-gpu=index,uuid,name,memory.total",
+            "--format=csv,noheader,nounits",
+        ],
+        check=True,
+        capture_output=True,
+        text=True,
+    ).stdout.splitlines()
+    rows = {}
+    for line in query:
+        index, uuid, name, memory = [
+            value.strip() for value in line.split(",", 3)
+        ]
+        rows[index] = {
+            "uuid": uuid,
+            "name": name,
+            "total_memory_mib": int(memory),
+        }
+    return rows
+
+
+def physical_gpu_report(profile, dry_run=False):
+    visible = os.environ.get("CUDA_VISIBLE_DEVICES")
+    if dry_run:
+        return {
+            "hardware_profile": profile,
+            "cuda_visible_devices": visible,
+            "physical_gpu_index": None,
+            "physical_gpu_uuid": None,
+            "physical_gpu_name": None,
+            "total_memory_mib": None,
+        }
+    if visible is None or not visible.isdigit():
+        raise RuntimeError(
+            "formal crossover jobs require CUDA_VISIBLE_DEVICES to contain "
+            "one physical numeric GPU index"
+        )
+    physical_index = int(visible)
+    hardware_policy = policy(profile, physical_index)
+    if physical_index not in hardware_policy["allowed_physical_gpu_indices"]:
+        raise RuntimeError(
+            f"physical GPU {physical_index} is not allowed by {profile}"
+        )
+    rows = _query_physical_gpus()
+    if visible not in rows:
+        raise RuntimeError(f"physical GPU {visible} not found")
+    row = rows[visible]
+    if row["name"] not in hardware_policy["allowed_device_names"]:
+        raise RuntimeError(
+            f"{profile} requires one of "
+            f"{hardware_policy['allowed_device_names']}, found {row['name']}"
+        )
+    return {
+        "hardware_profile": profile,
+        "cuda_visible_devices": visible,
+        "physical_gpu_index": physical_index,
+        "physical_gpu_uuid": row["uuid"],
+        "physical_gpu_name": row["name"],
+        "total_memory_mib": row["total_memory_mib"],
+    }
+
+
+def policy_for_report(profile, report):
+    return policy(profile, report["physical_gpu_index"])
+
+
+def assert_hardware_report(report, hardware_policy):
+    assert report["hardware_profile"] == hardware_policy["profile"]
+    assert (
+        report["physical_gpu_index"]
+        in hardware_policy["allowed_physical_gpu_indices"]
+    )
+    assert report["physical_gpu_uuid"]
+    assert report["physical_gpu_name"] in hardware_policy[
+        "allowed_device_names"
+    ]
+    assert report["cuda_visible_devices"] == str(
+        report["physical_gpu_index"]
+    )
diff --git a/experiments/crossover_hardware_smoke.py b/experiments/crossover_hardware_smoke.py
new file mode 100755
index 0000000..89c11c2
--- /dev/null
+++ b/experiments/crossover_hardware_smoke.py
@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+"""Dependency-free checks for frozen crossover hardware profiles."""
+from crossover_hardware import (
+    assert_hardware_report,
+    policy,
+    policy_for_report,
+)
+
+
+def main():
+    report = {
+        "hardware_profile": "a6000",
+        "cuda_visible_devices": "3",
+        "physical_gpu_index": 3,
+        "physical_gpu_uuid": "GPU-smoke",
+        "physical_gpu_name": "NVIDIA RTX A6000",
+        "total_memory_mib": 49140,
+    }
+    a6000 = policy_for_report("a6000", report)
+    assert a6000["allowed_physical_gpu_indices"] == [3]
+    assert_hardware_report(report, a6000)
+    gtx = policy("timan107_gtx1080")
+    assert gtx["allowed_physical_gpu_indices"] == [5, 7]
+    assert gtx["allowed_device_names"] == ["NVIDIA GeForce GTX 1080"]
+    print("crossover hardware smoke: all checks passed")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/experiments/prepare_shakespeare_char.py b/experiments/prepare_shakespeare_char.py
new file mode 100755
index 0000000..f09fc80
--- /dev/null
+++ b/experiments/prepare_shakespeare_char.py
@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+"""Download and deterministically tokenize the frozen Tiny Shakespeare data."""
+import argparse
+import hashlib
+import os
+import pickle
+import urllib.request
+
+import numpy as np
+
+
+URL = (
+    "https://raw.githubusercontent.com/karpathy/char-rnn/master/"
+    "data/tinyshakespeare/input.txt"
+)
+EXPECTED = {
+    "input.txt":
+        "86c4e6aa9db7c042ec79f339dcb96d42b0075e16b8fc2e86bf0ca57e2dc565ed",
+    "train.bin":
+        "6ec305602a99ac2802745a134e1f5e33e2231b4855525b00b9aebb730ac2626f",
+    "val.bin":
+        "d37d30cc0c8327c270d493299c3dca54135f6d5f1c9ef60cda78076e311204b1",
+}
+
+
+def sha256(path):
+    digest = hashlib.sha256()
+    with open(path, "rb") as handle:
+        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def verify(path, expected):
+    actual = sha256(path)
+    if actual != expected:
+        raise RuntimeError(
+            f"hash mismatch for {path}: expected {expected}, found {actual}"
+        )
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--out-dir", required=True)
+    args = parser.parse_args()
+    root = os.path.abspath(args.out_dir)
+    os.makedirs(root, exist_ok=True)
+    input_path = os.path.join(root, "input.txt")
+    if not os.path.isfile(input_path):
+        urllib.request.urlretrieve(URL, input_path)
+    verify(input_path, EXPECTED["input.txt"])
+
+    with open(input_path, encoding="utf-8") as handle:
+        data = handle.read()
+    chars = sorted(set(data))
+    stoi = {char: index for index, char in enumerate(chars)}
+    itos = {index: char for char, index in stoi.items()}
+    encoded = np.asarray([stoi[char] for char in data], dtype=np.uint16)
+    boundary = int(len(encoded) * 0.9)
+    encoded[:boundary].tofile(os.path.join(root, "train.bin"))
+    encoded[boundary:].tofile(os.path.join(root, "val.bin"))
+    with open(os.path.join(root, "meta.pkl"), "wb") as handle:
+        pickle.dump(
+            {"vocab_size": len(chars), "itos": itos, "stoi": stoi},
+            handle,
+        )
+
+    for name, expected in EXPECTED.items():
+        verify(os.path.join(root, name), expected)
+    print(
+        f"prepared Tiny Shakespeare at {root}: "
+        f"{boundary} train, {len(encoded) - boundary} validation tokens"
+    )
+
+
+if __name__ == "__main__":
+    main()
diff --git a/experiments/resnet_crossover_r2.py b/experiments/resnet_crossover_r2.py
index a9d5dfe..a7e78d7 100644
--- a/experiments/resnet_crossover_r2.py
+++ b/experiments/resnet_crossover_r2.py
@@ -10,6 +10,15 @@ import time
 
 
 ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, ROOT)
+from experiments.crossover_hardware import (
+    DEFAULT_PROFILE,
+    physical_gpu_report,
+    policy_for_report,
+    profile_choices,
+)
+
+
 PROTOCOL = os.path.join(ROOT, "RESNET_CROSSOVER.md")
 RESULT_ROOT = os.path.join(ROOT, "results", "resnet_crossover")
 DEFAULT_SELECTOR = os.path.join(RESULT_ROOT, "p1_selector.json")
@@ -113,6 +122,7 @@ def source_report(selector_path):
         os.path.join(ROOT, "experiments", "analyze_resnet_crossover_r2.py"),
         os.path.join(ROOT, "experiments", "resnet_crossover_native.py"),
         os.path.join(ROOT, "experiments", "conv_run.py"),
+        os.path.join(ROOT, "experiments", "crossover_hardware.py"),
         os.path.join(ROOT, "sdil", "conv_crossover.py"),
         os.path.join(ROOT, "sdil", "conv.py"),
         os.path.join(ROOT, "sdil", "data.py"),
@@ -295,7 +305,7 @@ def registry_sha256(jobs):
     return hashlib.sha256(encoded).hexdigest()
 
 
-def ensure_launch(source, selector, jobs):
+def ensure_launch(source, selector, jobs, hardware_policy):
     path = os.path.join(RESULT_ROOT, "r2_launch.json")
     expected = {
         "stage": "r2",
@@ -303,7 +313,9 @@ def ensure_launch(source, selector, jobs):
         "selector": selector,
         "registry_sha256": registry_sha256(jobs),
         "num_jobs": len(jobs),
-        "allowed_physical_gpus": [5, 7],
+        "hardware_policy": hardware_policy,
+        "allowed_physical_gpus":
+            hardware_policy["allowed_physical_gpu_indices"],
     }
     if os.path.isfile(path):
         if read_json(path) != expected:
@@ -326,40 +338,6 @@ def assert_source_unchanged(source):
             raise RuntimeError(f"ResNet R2 source drift: {relative}")
 
 
-def physical_gpu_report(dry_run=False):
-    visible = os.environ.get("CUDA_VISIBLE_DEVICES")
-    if dry_run:
-        return {
-            "cuda_visible_devices": visible,
-            "physical_gpu_index": None,
-            "physical_gpu_uuid": None,
-        }
-    if visible not in {"5", "7"}:
-        raise RuntimeError("ResNet R2 requires physical GPU 5 or 7")
-    query = subprocess.run(
-        [
-            "nvidia-smi",
-            "--query-gpu=index,uuid,name",
-            "--format=csv,noheader,nounits",
-        ],
-        check=True,
-        capture_output=True,
-        text=True,
-    ).stdout.splitlines()
-    rows = {}
-    for line in query:
-        index, uuid, name = [value.strip() for value in line.split(",", 2)]
-        rows[index] = {"uuid": uuid, "name": name}
-    if visible not in rows:
-        raise RuntimeError(f"physical GPU {visible} not found")
-    return {
-        "cuda_visible_devices": visible,
-        "physical_gpu_index": int(visible),
-        "physical_gpu_uuid": rows[visible]["uuid"],
-        "physical_gpu_name": rows[visible]["name"],
-    }
-
-
 def run_job(job, source, selector, gpu, dry_run):
     if not dry_run:
         assert_source_unchanged(source)
@@ -414,16 +392,24 @@ def main():
     parser.add_argument("--num-shards", type=int, default=1)
     parser.add_argument("--method", choices=METHODS)
     parser.add_argument("--depth", type=int, choices=DEPTHS)
+    parser.add_argument(
+        "--hardware-profile",
+        choices=profile_choices(),
+        default=os.environ.get("SDIL_HARDWARE_PROFILE", DEFAULT_PROFILE),
+    )
     parser.add_argument("--dry-run", action="store_true")
     args = parser.parse_args()
     if not 0 <= args.shard_index < args.num_shards:
         raise ValueError("invalid ResNet R2 shard")
     selector = selector_report(args.selector)
     source = source_report(args.selector)
-    gpu = physical_gpu_report(args.dry_run)
+    gpu = physical_gpu_report(args.hardware_profile, args.dry_run)
     complete_jobs = r2_jobs(selector["selected_rates"])
     if not args.dry_run:
-        launch = ensure_launch(source, selector, complete_jobs)
+        hardware_policy = policy_for_report(args.hardware_profile, gpu)
+        launch = ensure_launch(
+            source, selector, complete_jobs, hardware_policy
+        )
         print(f"R2 launch lock: {launch}", flush=True)
     jobs = complete_jobs
     if args.method:
diff --git a/experiments/run_a6000_matrix.py b/experiments/run_a6000_matrix.py
new file mode 100755
index 0000000..e5330ef
--- /dev/null
+++ b/experiments/run_a6000_matrix.py
@@ -0,0 +1,182 @@
+#!/usr/bin/env python3
+"""Sequential, restart-safe launcher for the complete A6000 81-cell matrix."""
+import argparse
+import os
+import subprocess
+import sys
+
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, ROOT)
+from experiments.crossover_hardware import physical_gpu_report
+from experiments.resnet_crossover_r2 import selector_report as resnet_selector
+from experiments.transformer_crossover_t2 import (
+    selector_report as transformer_selector,
+)
+
+
+def run(command, cwd, environment, dry_run):
+    rendered = " ".join(command)
+    print(f"[{cwd}] {rendered}", flush=True)
+    if not dry_run:
+        subprocess.run(command, cwd=cwd, env=environment, check=True)
+
+
+def require_file(path, label):
+    if not os.path.isfile(path):
+        raise RuntimeError(f"{label} is not frozen in this run packet: {path}")
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--plain-root", required=True)
+    parser.add_argument("--plain-python", required=True)
+    parser.add_argument(
+        "--family",
+        choices=("all", "plain", "resnet", "transformer"),
+        default="all",
+    )
+    parser.add_argument("--dry-run", action="store_true")
+    args = parser.parse_args()
+
+    plain_root = os.path.abspath(args.plain_root)
+    plain_python = os.path.abspath(args.plain_python)
+    require_file(plain_python, "plain-CNN Python executable")
+    require_file(
+        os.path.join(plain_root, "crossover_grid.py"),
+        "patched plain-CNN runner",
+    )
+    gpu = physical_gpu_report("a6000", args.dry_run)
+    print(f"hardware lock: {gpu}", flush=True)
+
+    environment = os.environ.copy()
+    environment["SDIL_HARDWARE_PROFILE"] = "a6000"
+    environment["SDIL_MAIN_ROOT"] = ROOT
+    families = (
+        ("plain", "resnet", "transformer")
+        if args.family == "all" else (args.family,)
+    )
+
+    plain_audit = os.path.join(plain_root, "runs", "plain-p2-audit.json")
+    if "plain" in families:
+        run(
+            [
+                plain_python,
+                "tests/local_rules_smoke.py",
+            ],
+            plain_root,
+            environment,
+            args.dry_run,
+        )
+        run(
+            [
+                plain_python,
+                "crossover_grid.py",
+                "--stage", "p2",
+                "--hardware-profile", "a6000",
+            ],
+            plain_root,
+            environment,
+            args.dry_run,
+        )
+        run(
+            [
+                plain_python,
+                "analyze_p2.py",
+                "--output", plain_audit,
+            ],
+            plain_root,
+            environment,
+            args.dry_run,
+        )
+
+    resnet_selector_path = os.path.join(
+        ROOT, "results", "resnet_crossover", "p1_selector.json"
+    )
+    if "resnet" in families:
+        require_file(resnet_selector_path, "complete ResNet P1 selector")
+        if not args.dry_run:
+            resnet_selector(resnet_selector_path)
+        run(
+            [
+                sys.executable,
+                "experiments/resnet_crossover_smoke.py",
+            ],
+            ROOT,
+            environment,
+            args.dry_run,
+        )
+        run(
+            [
+                sys.executable,
+                "experiments/resnet_crossover_r2.py",
+                "--hardware-profile", "a6000",
+            ],
+            ROOT,
+            environment,
+            args.dry_run,
+        )
+        run(
+            [
+                sys.executable,
+                "experiments/analyze_resnet_crossover_r2.py",
+            ],
+            ROOT,
+            environment,
+            args.dry_run,
+        )
+
+    transformer_selector_path = os.path.join(
+        ROOT, "results", "transformer_crossover", "p1_selector.json"
+    )
+    if "transformer" in families:
+        require_file(
+            transformer_selector_path, "complete Transformer T1 selector"
+        )
+        if not args.dry_run:
+            transformer_selector(transformer_selector_path)
+        run(
+            [
+                sys.executable,
+                "experiments/transformer_feedback_smoke.py",
+            ],
+            ROOT,
+            environment,
+            args.dry_run,
+        )
+        run(
+            [
+                sys.executable,
+                "experiments/transformer_crossover_t2.py",
+                "--hardware-profile", "a6000",
+            ],
+            ROOT,
+            environment,
+            args.dry_run,
+        )
+        run(
+            [
+                sys.executable,
+                "experiments/analyze_transformer_crossover_t2.py",
+            ],
+            ROOT,
+            environment,
+            args.dry_run,
+        )
+
+    if args.family == "all":
+        require_file(plain_audit, "complete plain-CNN P2 audit")
+        run(
+            [
+                sys.executable,
+                "experiments/analyze_crossover_81.py",
+                "--plain", plain_audit,
+            ],
+            ROOT,
+            environment,
+            args.dry_run,
+        )
+
+
+if __name__ == "__main__":
+    main()
diff --git a/experiments/transformer_crossover_native.py b/experiments/transformer_crossover_native.py
index ca47751..6a0269e 100644
--- a/experiments/transformer_crossover_native.py
+++ b/experiments/transformer_crossover_native.py
@@ -25,7 +25,10 @@ METHODS = (
     "bp", "fa", "dfa", "pepita", "ff", "ep", "dualprop",
     "clean_kp", "sdil",
 )
-DEFAULT_DATA_DIR = "/home/yurenh2/local-transformer/data/shakespeare_char"
+DEFAULT_DATA_DIR = os.environ.get(
+    "SDIL_TRANSFORMER_DATA_DIR",
+    "/home/yurenh2/local-transformer/data/shakespeare_char",
+)
 
 
 def file_sha256(path):
diff --git a/experiments/transformer_crossover_t2.py b/experiments/transformer_crossover_t2.py
index 17f61a2..4f12e56 100644
--- a/experiments/transformer_crossover_t2.py
+++ b/experiments/transformer_crossover_t2.py
@@ -10,6 +10,15 @@ import time
 
 
 ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, ROOT)
+from experiments.crossover_hardware import (
+    DEFAULT_PROFILE,
+    physical_gpu_report,
+    policy_for_report,
+    profile_choices,
+)
+
+
 PROTOCOL = os.path.join(ROOT, "TRANSFORMER_CROSSOVER.md")
 RESULT_ROOT = os.path.join(ROOT, "results", "transformer_crossover")
 DEFAULT_SELECTOR = os.path.join(RESULT_ROOT, "p1_selector.json")
@@ -123,6 +132,7 @@ def source_report(selector_path):
         os.path.join(
             ROOT, "experiments", "transformer_feedback_smoke.py"
         ),
+        os.path.join(ROOT, "experiments", "crossover_hardware.py"),
         os.path.join(ROOT, "sdil", "transformer.py"),
         os.path.abspath(selector_path),
     ]
@@ -235,7 +245,7 @@ def registry_sha256(jobs):
     return hashlib.sha256(encoded).hexdigest()
 
 
-def ensure_launch(source, selector, jobs):
+def ensure_launch(source, selector, jobs, hardware_policy):
     path = os.path.join(RESULT_ROOT, "t2_launch.json")
     expected = {
         "stage": "t2",
@@ -243,7 +253,9 @@ def ensure_launch(source, selector, jobs):
         "selector": selector,
         "registry_sha256": registry_sha256(jobs),
         "num_jobs": len(jobs),
-        "allowed_physical_gpus": [5, 7],
+        "hardware_policy": hardware_policy,
+        "allowed_physical_gpus":
+            hardware_policy["allowed_physical_gpu_indices"],
     }
     if os.path.isfile(path):
         if read_json(path) != expected:
@@ -274,40 +286,6 @@ def assert_source_unchanged(source):
         raise RuntimeError("Transformer T2 environment drift")
 
 
-def physical_gpu_report(dry_run=False):
-    visible = os.environ.get("CUDA_VISIBLE_DEVICES")
-    if dry_run:
-        return {
-            "cuda_visible_devices": visible,
-            "physical_gpu_index": None,
-            "physical_gpu_uuid": None,
-        }
-    if visible not in {"5", "7"}:
-        raise RuntimeError("Transformer T2 requires physical GPU 5 or 7")
-    query = subprocess.run(
-        [
-            "nvidia-smi",
-            "--query-gpu=index,uuid,name",
-            "--format=csv,noheader,nounits",
-        ],
-        check=True,
-        capture_output=True,
-        text=True,
-    ).stdout.splitlines()
-    rows = {}
-    for line in query:
-        index, uuid, name = [value.strip() for value in line.split(",", 2)]
-        rows[index] = {"uuid": uuid, "name": name}
-    if visible not in rows:
-        raise RuntimeError(f"physical GPU {visible} not found")
-    return {
-        "cuda_visible_devices": visible,
-        "physical_gpu_index": int(visible),
-        "physical_gpu_uuid": rows[visible]["uuid"],
-        "physical_gpu_name": rows[visible]["name"],
-    }
-
-
 def run_job(job, source, selector, gpu, dry_run):
     if not dry_run:
         assert_source_unchanged(source)
@@ -361,16 +339,24 @@ def main():
     parser.add_argument("--num-shards", type=int, default=1)
     parser.add_argument("--method", choices=METHODS)
     parser.add_argument("--depth", type=int, choices=DEPTHS)
+    parser.add_argument(
+        "--hardware-profile",
+        choices=profile_choices(),
+        default=os.environ.get("SDIL_HARDWARE_PROFILE", DEFAULT_PROFILE),
+    )
     parser.add_argument("--dry-run", action="store_true")
     args = parser.parse_args()
     if not 0 <= args.shard_index < args.num_shards:
         raise ValueError("invalid Transformer T2 shard")
     selector = selector_report(args.selector)
     source = source_report(args.selector)
-    gpu = physical_gpu_report(args.dry_run)
+    gpu = physical_gpu_report(args.hardware_profile, args.dry_run)
     complete_jobs = t2_jobs(selector["selected_rates"])
     if not args.dry_run:
-        launch = ensure_launch(source, selector, complete_jobs)
+        hardware_policy = policy_for_report(args.hardware_profile, gpu)
+        launch = ensure_launch(
+            source, selector, complete_jobs, hardware_policy
+        )
         print(f"T2 launch lock: {launch}", flush=True)
     jobs = complete_jobs
     if args.method: