summaryrefslogtreecommitdiff
path: root/experiments/transformer_crossover_grid.py
diff options
context:
space:
mode:
Diffstat (limited to 'experiments/transformer_crossover_grid.py')
-rw-r--r--experiments/transformer_crossover_grid.py45
1 files changed, 45 insertions, 0 deletions
diff --git a/experiments/transformer_crossover_grid.py b/experiments/transformer_crossover_grid.py
index 0fa8ba8..d37ded8 100644
--- a/experiments/transformer_crossover_grid.py
+++ b/experiments/transformer_crossover_grid.py
@@ -158,6 +158,46 @@ def p1_jobs():
return jobs
+def registry_sha256(jobs):
+ encoded = json.dumps(
+ jobs, sort_keys=True, separators=(",", ":")).encode("utf-8")
+ return hashlib.sha256(encoded).hexdigest()
+
+
+def ensure_p1_launch(source, jobs):
+ path = os.path.join(RESULT_ROOT, "p1_launch.json")
+ expected = {
+ "stage": "p1",
+ "source": source,
+ "registry_sha256": registry_sha256(jobs),
+ "num_jobs": len(jobs),
+ "allowed_physical_gpus": [5, 7],
+ }
+ if os.path.isfile(path):
+ with open(path, encoding="utf-8") as handle:
+ existing = json.load(handle)
+ if existing != expected:
+ raise RuntimeError("Transformer P1 launch lock drift")
+ return path
+ os.makedirs(os.path.dirname(path), exist_ok=True)
+ with open(path, "w", encoding="utf-8") as handle:
+ json.dump(expected, handle, indent=2, sort_keys=True)
+ handle.write("\n")
+ return path
+
+
+def assert_source_unchanged(source):
+ if git_output("status", "--porcelain", "--untracked-files=no"):
+ raise RuntimeError("tracked source changed after launch")
+ if git_output("rev-parse", "HEAD") != source["git_commit"]:
+ raise RuntimeError("source commit changed after launch")
+ for relative, expected_hash in source["tracked_files"].items():
+ path = os.path.join(ROOT, relative)
+ if sha256(path) != expected_hash:
+ raise RuntimeError(
+ f"source file changed after launch: {relative}")
+
+
def physical_gpu_report(dry_run=False):
visible = os.environ.get("CUDA_VISIBLE_DEVICES")
if dry_run:
@@ -189,6 +229,8 @@ def physical_gpu_report(dry_run=False):
def run_job(job, source, gpu, dry_run):
+ if not dry_run:
+ assert_source_unchanged(source)
manifest_path = job["output"] + ".manifest.json"
if os.path.exists(manifest_path):
print(f"preserving {job['experiment_name']}", flush=True)
@@ -243,6 +285,9 @@ def main():
source = source_report()
gpu = physical_gpu_report(args.dry_run)
jobs = p1_jobs()
+ if not args.dry_run:
+ launch = ensure_p1_launch(source, jobs)
+ print(f"P1 launch lock: {launch}", flush=True)
if args.method:
jobs = [job for job in jobs if job["method"] == args.method]
jobs = [