experiment: lock crossover selector sources

6 files changed, 162 insertions, 5 deletions

diff --git a/RESNET_CROSSOVER.md b/RESNET_CROSSOVER.md
index 28c8395..bcb9fad 100644
--- a/RESNET_CROSSOVER.md
+++ b/RESNET_CROSSOVER.md
@@ -102,6 +102,13 @@ before launch.  A capped or failed cell is recorded and the shard continues.
 The complete analyzer rejects missing/duplicate cells, source or architecture
 drift, test evaluation, ordinary-example drift, and silent failure omission.
 
+The R1 selector itself uses the same source discipline: one pre-launch lock
+binds its complete 19-job registry to a clean commit, tracked-file hashes, and
+the physical-GPU5/7 allowlist.  Every job rechecks the lock before starting,
+and its manifest records the resolved physical UUID.  Formal launch occurs
+from a detached worktree so unrelated main-branch commits cannot mix
+revisions inside a shard.
+
 ## R3 untouched confirmation
 
 No R3 test endpoint opens until all 27 R2 records and the complete analyzer
diff --git a/TRANSFORMER_CROSSOVER.md b/TRANSFORMER_CROSSOVER.md
index df0cf0f..8be1e72 100644
--- a/TRANSFORMER_CROSSOVER.md
+++ b/TRANSFORMER_CROSSOVER.md
@@ -159,7 +159,10 @@ job but remain reported.  No intermediate validation, test token, development
 pilot endpoint, or cross-method setting is used by the selector.
 
 The per-job cap is 48 hours.  Only timan107 physical GPUs 5 and 7 are allowed.
-A launch manifest must record the physical GPU UUID, source commit, tracked
+Before either shard starts, one immutable launch lock binds the complete
+27-job registry hash to the source commit, tracked-file hashes, environment
+freeze, and allowed physical GPUs.  Every subsequent cell rechecks that source
+lock.  Its manifest records the physical GPU UUID, source commit, tracked
 dirty state, environment freeze, dataset hashes, arguments, peak memory,
 wall time, and exit state.  A failure advances the shard rather than deleting
 the cell.
diff --git a/experiments/analyze_resnet_crossover_p1.py b/experiments/analyze_resnet_crossover_p1.py
index cbce299..b030755 100644
--- a/experiments/analyze_resnet_crossover_p1.py
+++ b/experiments/analyze_resnet_crossover_p1.py
@@ -7,7 +7,7 @@ import json
 import math
 import os
 
-from experiments.resnet_crossover_grid import p1_jobs
+from experiments.resnet_crossover_grid import p1_jobs, registry_sha256
 
 
 ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
@@ -86,6 +86,9 @@ def audit_job(job, expected_source):
         assert manifest[key] == job[key], (
             f"{job['experiment_name']}: drift in {key}")
     assert manifest["source"] == expected_source
+    hardware = manifest["hardware_lock"]
+    assert hardware["physical_gpu_index"] in (5, 7)
+    assert hardware["physical_gpu_uuid"]
     assert manifest["status"] == "completed"
     assert manifest["output_exists"] is True
     assert manifest["output_sha256"] == sha256(job["output"])
@@ -155,7 +158,17 @@ def main():
         audit_job(job, expected_source) for job in jobs
         if os.path.isfile(job["output"] + ".manifest.json")]
     selected = {}
+    launch = None
     if not missing:
+        launch_path = os.path.join(
+            ROOT, "results", "resnet_crossover", "p1_launch.json")
+        assert os.path.isfile(launch_path), "missing ResNet P1 launch lock"
+        launch = read_json(launch_path)
+        assert launch["stage"] == "p1"
+        assert launch["source"] == expected_source
+        assert launch["registry_sha256"] == registry_sha256(jobs)
+        assert launch["num_jobs"] == len(jobs)
+        assert launch["allowed_physical_gpus"] == [5, 7]
         for method in dict((job["method"], None) for job in jobs):
             candidates = [
                 record for record in records if record["method"] == method]
@@ -178,6 +191,7 @@ def main():
         "num_audited_records": len(records),
         "missing_experiments": missing,
         "test_policy": "none",
+        "launch_lock": launch,
         "records": records,
         "selected": selected,
     }
diff --git a/experiments/analyze_transformer_crossover_p1.py b/experiments/analyze_transformer_crossover_p1.py
index ab4c846..c1e33d6 100644
--- a/experiments/analyze_transformer_crossover_p1.py
+++ b/experiments/analyze_transformer_crossover_p1.py
@@ -9,7 +9,7 @@ import sys
 
 ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 sys.path.insert(0, ROOT)
-from experiments.transformer_crossover_grid import p1_jobs
+from experiments.transformer_crossover_grid import p1_jobs, registry_sha256
 
 
 TRAIN_HASH = (
@@ -183,7 +183,17 @@ def main():
         audit_job(job, expected_source) for job in jobs
         if os.path.isfile(job["output"] + ".manifest.json")]
     selected = {}
+    launch = None
     if not missing:
+        launch_path = os.path.join(
+            ROOT, "results", "transformer_crossover", "p1_launch.json")
+        assert os.path.isfile(launch_path), "missing Transformer P1 launch lock"
+        launch = read_json(launch_path)
+        assert launch["stage"] == "p1"
+        assert launch["source"] == expected_source
+        assert launch["registry_sha256"] == registry_sha256(jobs)
+        assert launch["num_jobs"] == len(jobs)
+        assert launch["allowed_physical_gpus"] == [5, 7]
         for method in dict((job["method"], None) for job in jobs):
             winner = choose([
                 record for record in records
@@ -208,6 +218,7 @@ def main():
         "num_audited_records": len(records),
         "missing_experiments": missing,
         "test_policy": "none",
+        "launch_lock": launch,
         "records": records,
         "selected": selected,
     }
diff --git a/experiments/resnet_crossover_grid.py b/experiments/resnet_crossover_grid.py
index 0967588..091f4e8 100644
--- a/experiments/resnet_crossover_grid.py
+++ b/experiments/resnet_crossover_grid.py
@@ -93,6 +93,76 @@ def p1_jobs():
     return [p1_command(method, rate) for method, rate in specifications]
 
 
+def registry_sha256(jobs):
+    encoded = json.dumps(
+        jobs, sort_keys=True, separators=(",", ":")).encode("utf-8")
+    return hashlib.sha256(encoded).hexdigest()
+
+
+def ensure_p1_launch(source, jobs):
+    path = os.path.join(RESULT_ROOT, "p1_launch.json")
+    expected = {
+        "stage": "p1",
+        "source": source,
+        "registry_sha256": registry_sha256(jobs),
+        "num_jobs": len(jobs),
+        "allowed_physical_gpus": [5, 7],
+    }
+    if os.path.isfile(path):
+        with open(path, encoding="utf-8") as handle:
+            existing = json.load(handle)
+        if existing != expected:
+            raise RuntimeError("ResNet P1 launch lock drift")
+        return path
+    os.makedirs(os.path.dirname(path), exist_ok=True)
+    with open(path, "w", encoding="utf-8") as handle:
+        json.dump(expected, handle, indent=2, sort_keys=True)
+        handle.write("\n")
+    return path
+
+
+def assert_source_unchanged(source):
+    if git_output("status", "--porcelain", "--untracked-files=no"):
+        raise RuntimeError("tracked source changed after launch")
+    if git_output("rev-parse", "HEAD") != source["git_commit"]:
+        raise RuntimeError("source commit changed after launch")
+    for relative, expected_hash in source["tracked_files"].items():
+        path = os.path.join(ROOT, relative)
+        if sha256(path) != expected_hash:
+            raise RuntimeError(
+                f"source file changed after launch: {relative}")
+
+
+def physical_gpu_report(dry_run=False):
+    visible = os.environ.get("CUDA_VISIBLE_DEVICES")
+    if dry_run:
+        return {
+            "cuda_visible_devices": visible,
+            "physical_gpu_index": None,
+            "physical_gpu_uuid": None,
+        }
+    if visible not in {"5", "7"}:
+        raise RuntimeError(
+            "formal ResNet jobs require CUDA_VISIBLE_DEVICES=5 or 7")
+    query = subprocess.run([
+        "nvidia-smi",
+        "--query-gpu=index,uuid,name",
+        "--format=csv,noheader,nounits",
+    ], check=True, capture_output=True, text=True).stdout.splitlines()
+    rows = {}
+    for line in query:
+        index, uuid, name = [value.strip() for value in line.split(",", 2)]
+        rows[index] = {"uuid": uuid, "name": name}
+    if visible not in rows:
+        raise RuntimeError(f"physical GPU {visible} not found")
+    return {
+        "cuda_visible_devices": visible,
+        "physical_gpu_index": int(visible),
+        "physical_gpu_uuid": rows[visible]["uuid"],
+        "physical_gpu_name": rows[visible]["name"],
+    }
+
+
 def p1_command(method, rate):
     name = f"resnet-p1-{method}-d20-lr{rate_tag(rate)}"
     output = os.path.join(RESULT_ROOT, "p1", name + ".json")
@@ -188,7 +258,9 @@ def p1_command(method, rate):
     }
 
 
-def run_job(job, source, dry_run):
+def run_job(job, source, gpu, dry_run):
+    if not dry_run:
+        assert_source_unchanged(source)
     manifest_path = job["output"] + ".manifest.json"
     if os.path.exists(manifest_path):
         print(f"preserving {job['experiment_name']}", flush=True)
@@ -215,6 +287,7 @@ def run_job(job, source, dry_run):
     manifest = {
         **job,
         "source": source,
+        "hardware_lock": gpu,
         "status": status,
         "return_code": return_code,
         "output_exists": output_exists,
@@ -241,7 +314,11 @@ def main():
     if not 0 <= args.shard_index < args.num_shards:
         raise ValueError("invalid shard")
     source = source_report()
+    gpu = physical_gpu_report(args.dry_run)
     jobs = p1_jobs()
+    if not args.dry_run:
+        launch = ensure_p1_launch(source, jobs)
+        print(f"P1 launch lock: {launch}", flush=True)
     if args.method:
         jobs = [job for job in jobs if job["method"] == args.method]
     jobs = [
@@ -250,7 +327,7 @@ def main():
     if not jobs:
         raise ValueError("no jobs selected")
     for job in jobs:
-        run_job(job, source, args.dry_run)
+        run_job(job, source, gpu, args.dry_run)
 
 
 if __name__ == "__main__":
diff --git a/experiments/transformer_crossover_grid.py b/experiments/transformer_crossover_grid.py
index 0fa8ba8..d37ded8 100644
--- a/experiments/transformer_crossover_grid.py
+++ b/experiments/transformer_crossover_grid.py
@@ -158,6 +158,46 @@ def p1_jobs():
     return jobs
 
 
+def registry_sha256(jobs):
+    encoded = json.dumps(
+        jobs, sort_keys=True, separators=(",", ":")).encode("utf-8")
+    return hashlib.sha256(encoded).hexdigest()
+
+
+def ensure_p1_launch(source, jobs):
+    path = os.path.join(RESULT_ROOT, "p1_launch.json")
+    expected = {
+        "stage": "p1",
+        "source": source,
+        "registry_sha256": registry_sha256(jobs),
+        "num_jobs": len(jobs),
+        "allowed_physical_gpus": [5, 7],
+    }
+    if os.path.isfile(path):
+        with open(path, encoding="utf-8") as handle:
+            existing = json.load(handle)
+        if existing != expected:
+            raise RuntimeError("Transformer P1 launch lock drift")
+        return path
+    os.makedirs(os.path.dirname(path), exist_ok=True)
+    with open(path, "w", encoding="utf-8") as handle:
+        json.dump(expected, handle, indent=2, sort_keys=True)
+        handle.write("\n")
+    return path
+
+
+def assert_source_unchanged(source):
+    if git_output("status", "--porcelain", "--untracked-files=no"):
+        raise RuntimeError("tracked source changed after launch")
+    if git_output("rev-parse", "HEAD") != source["git_commit"]:
+        raise RuntimeError("source commit changed after launch")
+    for relative, expected_hash in source["tracked_files"].items():
+        path = os.path.join(ROOT, relative)
+        if sha256(path) != expected_hash:
+            raise RuntimeError(
+                f"source file changed after launch: {relative}")
+
+
 def physical_gpu_report(dry_run=False):
     visible = os.environ.get("CUDA_VISIBLE_DEVICES")
     if dry_run:
@@ -189,6 +229,8 @@ def physical_gpu_report(dry_run=False):
 
 
 def run_job(job, source, gpu, dry_run):
+    if not dry_run:
+        assert_source_unchanged(source)
     manifest_path = job["output"] + ".manifest.json"
     if os.path.exists(manifest_path):
         print(f"preserving {job['experiment_name']}", flush=True)
@@ -243,6 +285,9 @@ def main():
     source = source_report()
     gpu = physical_gpu_report(args.dry_run)
     jobs = p1_jobs()
+    if not args.dry_run:
+        launch = ensure_p1_launch(source, jobs)
+        print(f"P1 launch lock: {launch}", flush=True)
     if args.method:
         jobs = [job for job in jobs if job["method"] == args.method]
     jobs = [