diff options
Diffstat (limited to 'experiments/analyze_transformer_crossover_t2.py')
| -rw-r--r-- | experiments/analyze_transformer_crossover_t2.py | 30 |
1 files changed, 21 insertions, 9 deletions
diff --git a/experiments/analyze_transformer_crossover_t2.py b/experiments/analyze_transformer_crossover_t2.py index 5eeb226..74c3ef6 100644 --- a/experiments/analyze_transformer_crossover_t2.py +++ b/experiments/analyze_transformer_crossover_t2.py @@ -18,6 +18,7 @@ from experiments.transformer_crossover_t2 import ( selector_report, t2_jobs, ) +from experiments.crossover_hardware import assert_hardware_report TRAIN_HASH = ( @@ -79,7 +80,7 @@ def expected_work(record, job): return work -def audit_completed(job, manifest, source): +def audit_completed(job, manifest, source, hardware_policy): assert manifest["output_exists"] is True assert manifest["output_sha256"] == sha256(job["output"]) record = read_json(job["output"]) @@ -164,8 +165,13 @@ def audit_completed(job, manifest, source): ) assert record["work"]["logical_task_loss_queries"] == 0 hardware = record["hardware"] - assert hardware["cuda_visible_devices"] in ("5", "7") - assert hardware["device_name"] == "NVIDIA GeForce GTX 1080" + locked_hardware = manifest["hardware_lock"] + assert_hardware_report(locked_hardware, hardware_policy) + assert ( + hardware["cuda_visible_devices"] + == locked_hardware["cuda_visible_devices"] + ) + assert hardware["device_name"] == locked_hardware["physical_gpu_name"] assert hardware["peak_memory_allocated_bytes"] is not None assert hardware["peak_memory_reserved_bytes"] is not None return { @@ -189,7 +195,7 @@ def audit_completed(job, manifest, source): } -def audit_job(job, source, selector): +def audit_job(job, source, selector, hardware_policy): manifest_path = job["output"] + ".manifest.json" if not os.path.isfile(manifest_path): raise AssertionError(f"missing T2 manifest: {job['experiment_name']}") @@ -211,8 +217,7 @@ def audit_job(job, source, selector): assert manifest["source"] == source assert manifest["selector"] == selector hardware = manifest["hardware_lock"] - assert hardware["physical_gpu_index"] in (5, 7) - assert hardware["physical_gpu_uuid"] + assert_hardware_report(hardware, hardware_policy) common = { "cell_id": f"transformer{job['depth']}::{job['method']}", "method": job["method"], @@ -228,7 +233,7 @@ def audit_job(job, source, selector): if manifest["status"] == "completed": return { **common, - **audit_completed(job, manifest, source), + **audit_completed(job, manifest, source, hardware_policy), } assert manifest["status"] in { "timeout", "nonzero_exit", "missing_output" @@ -271,9 +276,15 @@ def main(): assert launch["selector"] == selector assert launch["registry_sha256"] == registry_sha256(jobs) assert launch["num_jobs"] == 27 - assert launch["allowed_physical_gpus"] == [5, 7] + hardware_policy = launch["hardware_policy"] + assert ( + launch["allowed_physical_gpus"] + == hardware_policy["allowed_physical_gpu_indices"] + ) source = launch["source"] - records = [audit_job(job, source, selector) for job in jobs] + records = [ + audit_job(job, source, selector, hardware_policy) for job in jobs + ] assert len(records) == 27 assert len({record["cell_id"] for record in records}) == 27 assert { @@ -294,6 +305,7 @@ def main(): "test_policy": "none", "source": source, "selector": selector, + "hardware_policy": hardware_policy, "launch_lock": { "path": os.path.relpath(launch_path, ROOT), "sha256": sha256(launch_path), |