1 files changed, 79 insertions, 2 deletions

diff --git a/experiments/resnet_crossover_grid.py b/experiments/resnet_crossover_grid.py
index 0967588..091f4e8 100644
--- a/experiments/resnet_crossover_grid.py
+++ b/experiments/resnet_crossover_grid.py
@@ -93,6 +93,76 @@ def p1_jobs():
     return [p1_command(method, rate) for method, rate in specifications]
 
 
+def registry_sha256(jobs):
+    encoded = json.dumps(
+        jobs, sort_keys=True, separators=(",", ":")).encode("utf-8")
+    return hashlib.sha256(encoded).hexdigest()
+
+
+def ensure_p1_launch(source, jobs):
+    path = os.path.join(RESULT_ROOT, "p1_launch.json")
+    expected = {
+        "stage": "p1",
+        "source": source,
+        "registry_sha256": registry_sha256(jobs),
+        "num_jobs": len(jobs),
+        "allowed_physical_gpus": [5, 7],
+    }
+    if os.path.isfile(path):
+        with open(path, encoding="utf-8") as handle:
+            existing = json.load(handle)
+        if existing != expected:
+            raise RuntimeError("ResNet P1 launch lock drift")
+        return path
+    os.makedirs(os.path.dirname(path), exist_ok=True)
+    with open(path, "w", encoding="utf-8") as handle:
+        json.dump(expected, handle, indent=2, sort_keys=True)
+        handle.write("\n")
+    return path
+
+
+def assert_source_unchanged(source):
+    if git_output("status", "--porcelain", "--untracked-files=no"):
+        raise RuntimeError("tracked source changed after launch")
+    if git_output("rev-parse", "HEAD") != source["git_commit"]:
+        raise RuntimeError("source commit changed after launch")
+    for relative, expected_hash in source["tracked_files"].items():
+        path = os.path.join(ROOT, relative)
+        if sha256(path) != expected_hash:
+            raise RuntimeError(
+                f"source file changed after launch: {relative}")
+
+
+def physical_gpu_report(dry_run=False):
+    visible = os.environ.get("CUDA_VISIBLE_DEVICES")
+    if dry_run:
+        return {
+            "cuda_visible_devices": visible,
+            "physical_gpu_index": None,
+            "physical_gpu_uuid": None,
+        }
+    if visible not in {"5", "7"}:
+        raise RuntimeError(
+            "formal ResNet jobs require CUDA_VISIBLE_DEVICES=5 or 7")
+    query = subprocess.run([
+        "nvidia-smi",
+        "--query-gpu=index,uuid,name",
+        "--format=csv,noheader,nounits",
+    ], check=True, capture_output=True, text=True).stdout.splitlines()
+    rows = {}
+    for line in query:
+        index, uuid, name = [value.strip() for value in line.split(",", 2)]
+        rows[index] = {"uuid": uuid, "name": name}
+    if visible not in rows:
+        raise RuntimeError(f"physical GPU {visible} not found")
+    return {
+        "cuda_visible_devices": visible,
+        "physical_gpu_index": int(visible),
+        "physical_gpu_uuid": rows[visible]["uuid"],
+        "physical_gpu_name": rows[visible]["name"],
+    }
+
+
 def p1_command(method, rate):
     name = f"resnet-p1-{method}-d20-lr{rate_tag(rate)}"
     output = os.path.join(RESULT_ROOT, "p1", name + ".json")
@@ -188,7 +258,9 @@ def p1_command(method, rate):
     }
 
 
-def run_job(job, source, dry_run):
+def run_job(job, source, gpu, dry_run):
+    if not dry_run:
+        assert_source_unchanged(source)
     manifest_path = job["output"] + ".manifest.json"
     if os.path.exists(manifest_path):
         print(f"preserving {job['experiment_name']}", flush=True)
@@ -215,6 +287,7 @@ def run_job(job, source, dry_run):
     manifest = {
         **job,
         "source": source,
+        "hardware_lock": gpu,
         "status": status,
         "return_code": return_code,
         "output_exists": output_exists,
@@ -241,7 +314,11 @@ def main():
     if not 0 <= args.shard_index < args.num_shards:
         raise ValueError("invalid shard")
     source = source_report()
+    gpu = physical_gpu_report(args.dry_run)
     jobs = p1_jobs()
+    if not args.dry_run:
+        launch = ensure_p1_launch(source, jobs)
+        print(f"P1 launch lock: {launch}", flush=True)
     if args.method:
         jobs = [job for job in jobs if job["method"] == args.method]
     jobs = [
@@ -250,7 +327,7 @@ def main():
     if not jobs:
         raise ValueError("no jobs selected")
     for job in jobs:
-        run_job(job, source, args.dry_run)
+        run_job(job, source, gpu, args.dry_run)
 
 
 if __name__ == "__main__":