diff options
| -rw-r--r-- | RESNET_CROSSOVER.md | 7 | ||||
| -rw-r--r-- | TRANSFORMER_CROSSOVER.md | 5 | ||||
| -rw-r--r-- | experiments/analyze_resnet_crossover_p1.py | 16 | ||||
| -rw-r--r-- | experiments/analyze_transformer_crossover_p1.py | 13 | ||||
| -rw-r--r-- | experiments/resnet_crossover_grid.py | 81 | ||||
| -rw-r--r-- | experiments/transformer_crossover_grid.py | 45 |
6 files changed, 162 insertions, 5 deletions
diff --git a/RESNET_CROSSOVER.md b/RESNET_CROSSOVER.md index 28c8395..bcb9fad 100644 --- a/RESNET_CROSSOVER.md +++ b/RESNET_CROSSOVER.md @@ -102,6 +102,13 @@ before launch. A capped or failed cell is recorded and the shard continues. The complete analyzer rejects missing/duplicate cells, source or architecture drift, test evaluation, ordinary-example drift, and silent failure omission. +The R1 selector itself uses the same source discipline: one pre-launch lock +binds its complete 19-job registry to a clean commit, tracked-file hashes, and +the physical-GPU5/7 allowlist. Every job rechecks the lock before starting, +and its manifest records the resolved physical UUID. Formal launch occurs +from a detached worktree so unrelated main-branch commits cannot mix +revisions inside a shard. + ## R3 untouched confirmation No R3 test endpoint opens until all 27 R2 records and the complete analyzer diff --git a/TRANSFORMER_CROSSOVER.md b/TRANSFORMER_CROSSOVER.md index df0cf0f..8be1e72 100644 --- a/TRANSFORMER_CROSSOVER.md +++ b/TRANSFORMER_CROSSOVER.md @@ -159,7 +159,10 @@ job but remain reported. No intermediate validation, test token, development pilot endpoint, or cross-method setting is used by the selector. The per-job cap is 48 hours. Only timan107 physical GPUs 5 and 7 are allowed. -A launch manifest must record the physical GPU UUID, source commit, tracked +Before either shard starts, one immutable launch lock binds the complete +27-job registry hash to the source commit, tracked-file hashes, environment +freeze, and allowed physical GPUs. Every subsequent cell rechecks that source +lock. Its manifest records the physical GPU UUID, source commit, tracked dirty state, environment freeze, dataset hashes, arguments, peak memory, wall time, and exit state. A failure advances the shard rather than deleting the cell. diff --git a/experiments/analyze_resnet_crossover_p1.py b/experiments/analyze_resnet_crossover_p1.py index cbce299..b030755 100644 --- a/experiments/analyze_resnet_crossover_p1.py +++ b/experiments/analyze_resnet_crossover_p1.py @@ -7,7 +7,7 @@ import json import math import os -from experiments.resnet_crossover_grid import p1_jobs +from experiments.resnet_crossover_grid import p1_jobs, registry_sha256 ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -86,6 +86,9 @@ def audit_job(job, expected_source): assert manifest[key] == job[key], ( f"{job['experiment_name']}: drift in {key}") assert manifest["source"] == expected_source + hardware = manifest["hardware_lock"] + assert hardware["physical_gpu_index"] in (5, 7) + assert hardware["physical_gpu_uuid"] assert manifest["status"] == "completed" assert manifest["output_exists"] is True assert manifest["output_sha256"] == sha256(job["output"]) @@ -155,7 +158,17 @@ def main(): audit_job(job, expected_source) for job in jobs if os.path.isfile(job["output"] + ".manifest.json")] selected = {} + launch = None if not missing: + launch_path = os.path.join( + ROOT, "results", "resnet_crossover", "p1_launch.json") + assert os.path.isfile(launch_path), "missing ResNet P1 launch lock" + launch = read_json(launch_path) + assert launch["stage"] == "p1" + assert launch["source"] == expected_source + assert launch["registry_sha256"] == registry_sha256(jobs) + assert launch["num_jobs"] == len(jobs) + assert launch["allowed_physical_gpus"] == [5, 7] for method in dict((job["method"], None) for job in jobs): candidates = [ record for record in records if record["method"] == method] @@ -178,6 +191,7 @@ def main(): "num_audited_records": len(records), "missing_experiments": missing, "test_policy": "none", + "launch_lock": launch, "records": records, "selected": selected, } diff --git a/experiments/analyze_transformer_crossover_p1.py b/experiments/analyze_transformer_crossover_p1.py index ab4c846..c1e33d6 100644 --- a/experiments/analyze_transformer_crossover_p1.py +++ b/experiments/analyze_transformer_crossover_p1.py @@ -9,7 +9,7 @@ import sys ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, ROOT) -from experiments.transformer_crossover_grid import p1_jobs +from experiments.transformer_crossover_grid import p1_jobs, registry_sha256 TRAIN_HASH = ( @@ -183,7 +183,17 @@ def main(): audit_job(job, expected_source) for job in jobs if os.path.isfile(job["output"] + ".manifest.json")] selected = {} + launch = None if not missing: + launch_path = os.path.join( + ROOT, "results", "transformer_crossover", "p1_launch.json") + assert os.path.isfile(launch_path), "missing Transformer P1 launch lock" + launch = read_json(launch_path) + assert launch["stage"] == "p1" + assert launch["source"] == expected_source + assert launch["registry_sha256"] == registry_sha256(jobs) + assert launch["num_jobs"] == len(jobs) + assert launch["allowed_physical_gpus"] == [5, 7] for method in dict((job["method"], None) for job in jobs): winner = choose([ record for record in records @@ -208,6 +218,7 @@ def main(): "num_audited_records": len(records), "missing_experiments": missing, "test_policy": "none", + "launch_lock": launch, "records": records, "selected": selected, } diff --git a/experiments/resnet_crossover_grid.py b/experiments/resnet_crossover_grid.py index 0967588..091f4e8 100644 --- a/experiments/resnet_crossover_grid.py +++ b/experiments/resnet_crossover_grid.py @@ -93,6 +93,76 @@ def p1_jobs(): return [p1_command(method, rate) for method, rate in specifications] +def registry_sha256(jobs): + encoded = json.dumps( + jobs, sort_keys=True, separators=(",", ":")).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def ensure_p1_launch(source, jobs): + path = os.path.join(RESULT_ROOT, "p1_launch.json") + expected = { + "stage": "p1", + "source": source, + "registry_sha256": registry_sha256(jobs), + "num_jobs": len(jobs), + "allowed_physical_gpus": [5, 7], + } + if os.path.isfile(path): + with open(path, encoding="utf-8") as handle: + existing = json.load(handle) + if existing != expected: + raise RuntimeError("ResNet P1 launch lock drift") + return path + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8") as handle: + json.dump(expected, handle, indent=2, sort_keys=True) + handle.write("\n") + return path + + +def assert_source_unchanged(source): + if git_output("status", "--porcelain", "--untracked-files=no"): + raise RuntimeError("tracked source changed after launch") + if git_output("rev-parse", "HEAD") != source["git_commit"]: + raise RuntimeError("source commit changed after launch") + for relative, expected_hash in source["tracked_files"].items(): + path = os.path.join(ROOT, relative) + if sha256(path) != expected_hash: + raise RuntimeError( + f"source file changed after launch: {relative}") + + +def physical_gpu_report(dry_run=False): + visible = os.environ.get("CUDA_VISIBLE_DEVICES") + if dry_run: + return { + "cuda_visible_devices": visible, + "physical_gpu_index": None, + "physical_gpu_uuid": None, + } + if visible not in {"5", "7"}: + raise RuntimeError( + "formal ResNet jobs require CUDA_VISIBLE_DEVICES=5 or 7") + query = subprocess.run([ + "nvidia-smi", + "--query-gpu=index,uuid,name", + "--format=csv,noheader,nounits", + ], check=True, capture_output=True, text=True).stdout.splitlines() + rows = {} + for line in query: + index, uuid, name = [value.strip() for value in line.split(",", 2)] + rows[index] = {"uuid": uuid, "name": name} + if visible not in rows: + raise RuntimeError(f"physical GPU {visible} not found") + return { + "cuda_visible_devices": visible, + "physical_gpu_index": int(visible), + "physical_gpu_uuid": rows[visible]["uuid"], + "physical_gpu_name": rows[visible]["name"], + } + + def p1_command(method, rate): name = f"resnet-p1-{method}-d20-lr{rate_tag(rate)}" output = os.path.join(RESULT_ROOT, "p1", name + ".json") @@ -188,7 +258,9 @@ def p1_command(method, rate): } -def run_job(job, source, dry_run): +def run_job(job, source, gpu, dry_run): + if not dry_run: + assert_source_unchanged(source) manifest_path = job["output"] + ".manifest.json" if os.path.exists(manifest_path): print(f"preserving {job['experiment_name']}", flush=True) @@ -215,6 +287,7 @@ def run_job(job, source, dry_run): manifest = { **job, "source": source, + "hardware_lock": gpu, "status": status, "return_code": return_code, "output_exists": output_exists, @@ -241,7 +314,11 @@ def main(): if not 0 <= args.shard_index < args.num_shards: raise ValueError("invalid shard") source = source_report() + gpu = physical_gpu_report(args.dry_run) jobs = p1_jobs() + if not args.dry_run: + launch = ensure_p1_launch(source, jobs) + print(f"P1 launch lock: {launch}", flush=True) if args.method: jobs = [job for job in jobs if job["method"] == args.method] jobs = [ @@ -250,7 +327,7 @@ def main(): if not jobs: raise ValueError("no jobs selected") for job in jobs: - run_job(job, source, args.dry_run) + run_job(job, source, gpu, args.dry_run) if __name__ == "__main__": diff --git a/experiments/transformer_crossover_grid.py b/experiments/transformer_crossover_grid.py index 0fa8ba8..d37ded8 100644 --- a/experiments/transformer_crossover_grid.py +++ b/experiments/transformer_crossover_grid.py @@ -158,6 +158,46 @@ def p1_jobs(): return jobs +def registry_sha256(jobs): + encoded = json.dumps( + jobs, sort_keys=True, separators=(",", ":")).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def ensure_p1_launch(source, jobs): + path = os.path.join(RESULT_ROOT, "p1_launch.json") + expected = { + "stage": "p1", + "source": source, + "registry_sha256": registry_sha256(jobs), + "num_jobs": len(jobs), + "allowed_physical_gpus": [5, 7], + } + if os.path.isfile(path): + with open(path, encoding="utf-8") as handle: + existing = json.load(handle) + if existing != expected: + raise RuntimeError("Transformer P1 launch lock drift") + return path + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8") as handle: + json.dump(expected, handle, indent=2, sort_keys=True) + handle.write("\n") + return path + + +def assert_source_unchanged(source): + if git_output("status", "--porcelain", "--untracked-files=no"): + raise RuntimeError("tracked source changed after launch") + if git_output("rev-parse", "HEAD") != source["git_commit"]: + raise RuntimeError("source commit changed after launch") + for relative, expected_hash in source["tracked_files"].items(): + path = os.path.join(ROOT, relative) + if sha256(path) != expected_hash: + raise RuntimeError( + f"source file changed after launch: {relative}") + + def physical_gpu_report(dry_run=False): visible = os.environ.get("CUDA_VISIBLE_DEVICES") if dry_run: @@ -189,6 +229,8 @@ def physical_gpu_report(dry_run=False): def run_job(job, source, gpu, dry_run): + if not dry_run: + assert_source_unchanged(source) manifest_path = job["output"] + ".manifest.json" if os.path.exists(manifest_path): print(f"preserving {job['experiment_name']}", flush=True) @@ -243,6 +285,9 @@ def main(): source = source_report() gpu = physical_gpu_report(args.dry_run) jobs = p1_jobs() + if not args.dry_run: + launch = ensure_p1_launch(source, jobs) + print(f"P1 launch lock: {launch}", flush=True) if args.method: jobs = [job for job in jobs if job["method"] == args.method] jobs = [ |