diff options
Diffstat (limited to 'experiments/transformer_crossover_grid.py')
| -rw-r--r-- | experiments/transformer_crossover_grid.py | 45 |
1 files changed, 45 insertions, 0 deletions
diff --git a/experiments/transformer_crossover_grid.py b/experiments/transformer_crossover_grid.py index 0fa8ba8..d37ded8 100644 --- a/experiments/transformer_crossover_grid.py +++ b/experiments/transformer_crossover_grid.py @@ -158,6 +158,46 @@ def p1_jobs(): return jobs +def registry_sha256(jobs): + encoded = json.dumps( + jobs, sort_keys=True, separators=(",", ":")).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def ensure_p1_launch(source, jobs): + path = os.path.join(RESULT_ROOT, "p1_launch.json") + expected = { + "stage": "p1", + "source": source, + "registry_sha256": registry_sha256(jobs), + "num_jobs": len(jobs), + "allowed_physical_gpus": [5, 7], + } + if os.path.isfile(path): + with open(path, encoding="utf-8") as handle: + existing = json.load(handle) + if existing != expected: + raise RuntimeError("Transformer P1 launch lock drift") + return path + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8") as handle: + json.dump(expected, handle, indent=2, sort_keys=True) + handle.write("\n") + return path + + +def assert_source_unchanged(source): + if git_output("status", "--porcelain", "--untracked-files=no"): + raise RuntimeError("tracked source changed after launch") + if git_output("rev-parse", "HEAD") != source["git_commit"]: + raise RuntimeError("source commit changed after launch") + for relative, expected_hash in source["tracked_files"].items(): + path = os.path.join(ROOT, relative) + if sha256(path) != expected_hash: + raise RuntimeError( + f"source file changed after launch: {relative}") + + def physical_gpu_report(dry_run=False): visible = os.environ.get("CUDA_VISIBLE_DEVICES") if dry_run: @@ -189,6 +229,8 @@ def physical_gpu_report(dry_run=False): def run_job(job, source, gpu, dry_run): + if not dry_run: + assert_source_unchanged(source) manifest_path = job["output"] + ".manifest.json" if os.path.exists(manifest_path): print(f"preserving {job['experiment_name']}", flush=True) @@ -243,6 +285,9 @@ def main(): source = source_report() gpu = physical_gpu_report(args.dry_run) jobs = p1_jobs() + if not args.dry_run: + launch = ensure_p1_launch(source, jobs) + print(f"P1 launch lock: {launch}", flush=True) if args.method: jobs = [job for job in jobs if job["method"] == args.method] jobs = [ |
