experiment: serialize crossover launch locks

5 files changed, 55 insertions, 5 deletions

diff --git a/experiments/crossover_artifact_validation_smoke.py b/experiments/crossover_artifact_validation_smoke.py
index e267832..f410a3e 100644
--- a/experiments/crossover_artifact_validation_smoke.py
+++ b/experiments/crossover_artifact_validation_smoke.py
@@ -44,6 +44,15 @@ def main():
         "allowed_physical_gpus": [5, 7],
     }
     validate(launch, "resnet_p1_launch", "abc")
+    transformer_launch = copy.deepcopy(launch)
+    transformer_launch["num_jobs"] = 27
+    validate(transformer_launch, "transformer_p1_launch", "abc")
+    r2_launch = copy.deepcopy(transformer_launch)
+    r2_launch["stage"] = "r2"
+    validate(r2_launch, "resnet_r2_launch", "abc")
+    t2_launch = copy.deepcopy(transformer_launch)
+    t2_launch["stage"] = "t2"
+    validate(t2_launch, "transformer_t2_launch", "abc")
     bad_launch = copy.deepcopy(launch)
     bad_launch["allowed_physical_gpus"] = [0, 1]
     must_fail(bad_launch, "resnet_p1_launch", "abc")
diff --git a/experiments/handoff_plain_to_resnet_p1.sh b/experiments/handoff_plain_to_resnet_p1.sh
index c5656a2..04fcdea 100755
--- a/experiments/handoff_plain_to_resnet_p1.sh
+++ b/experiments/handoff_plain_to_resnet_p1.sh
@@ -12,6 +12,7 @@ fi
 expected_commit="$1"
 resnet_root="/scratch/yurenh2/sdil-runs/resnet-p1"
 python_bin="/home/yurenh2/miniconda3/envs/ep_pascal/bin/python3"
+validator="/home/yurenh2/sdil/experiments/validate_crossover_artifact.py"
 gpu5_uuid="GPU-f4aed756-228e-7522-94e7-fb543c02cc74"
 gpu7_uuid="GPU-df3d5933-c3e9-9ea1-d595-9d2a3b68c14b"
 
@@ -74,6 +75,13 @@ fi
 mkdir -p "$resnet_root/results/resnet_crossover"
 tmux new-session -d -s resnet_p1_gpu5 \
   "cd '$resnet_root' && env CUDA_VISIBLE_DEVICES=5 '$python_bin' experiments/resnet_crossover_grid.py --stage p1 --num-shards 2 --shard-index 0 > results/resnet_crossover/p1_gpu5.log 2>&1"
+resnet_launch="$resnet_root/results/resnet_crossover/p1_launch.json"
+until "$python_bin" "$validator" \
+    --path "$resnet_launch" \
+    --kind resnet_p1_launch \
+    --expected-commit "$expected_commit" >/dev/null 2>&1; do
+  sleep 1
+done
 tmux new-session -d -s resnet_p1_gpu7 \
   "cd '$resnet_root' && env CUDA_VISIBLE_DEVICES=7 '$python_bin' experiments/resnet_crossover_grid.py --stage p1 --num-shards 2 --shard-index 1 > results/resnet_crossover/p1_gpu7.log 2>&1"
 tmux has-session -t resnet_p1_gpu5
diff --git a/experiments/handoff_resnet_p1_to_transformer_p1.sh b/experiments/handoff_resnet_p1_to_transformer_p1.sh
index c3424d0..7376bf3 100755
--- a/experiments/handoff_resnet_p1_to_transformer_p1.sh
+++ b/experiments/handoff_resnet_p1_to_transformer_p1.sh
@@ -113,6 +113,12 @@ fi
 mkdir -p "$transformer_root/results/transformer_crossover"
 tmux new-session -d -s transformer_p1_gpu5 \
   "cd '$transformer_root' && env CUDA_VISIBLE_DEVICES=5 '$python_bin' experiments/transformer_crossover_grid.py --stage p1 --num-shards 2 --shard-index 0 > results/transformer_crossover/p1_gpu5.log 2>&1"
+until "$python_bin" "$validator" \
+    --path "$transformer_launch" \
+    --kind transformer_p1_launch \
+    --expected-commit "$expected_transformer_commit" >/dev/null 2>&1; do
+  sleep 1
+done
 tmux new-session -d -s transformer_p1_gpu7 \
   "cd '$transformer_root' && env CUDA_VISIBLE_DEVICES=7 '$python_bin' experiments/transformer_crossover_grid.py --stage p1 --num-shards 2 --shard-index 1 > results/transformer_crossover/p1_gpu7.log 2>&1"
 tmux has-session -t transformer_p1_gpu5
diff --git a/experiments/handoff_selectors_to_full_crossover.sh b/experiments/handoff_selectors_to_full_crossover.sh
index e87fea8..12670db 100755
--- a/experiments/handoff_selectors_to_full_crossover.sh
+++ b/experiments/handoff_selectors_to_full_crossover.sh
@@ -138,6 +138,13 @@ if tmux has-session -t resnet_r2_gpu5 2>/dev/null ||
 fi
 tmux new-session -d -s resnet_r2_gpu5 \
   "cd '$resnet_r2_root' && env CUDA_VISIBLE_DEVICES=5 '$python_bin' experiments/resnet_crossover_r2.py --num-shards 2 --shard-index 0 > results/resnet_crossover/r2_gpu5.log 2>&1"
+resnet_r2_launch="$resnet_r2_root/results/resnet_crossover/r2_launch.json"
+until "$python_bin" "$validator" \
+    --path "$resnet_r2_launch" \
+    --kind resnet_r2_launch \
+    --expected-commit "$resnet_r2_commit" >/dev/null 2>&1; do
+  sleep 1
+done
 tmux new-session -d -s resnet_r2_gpu7 \
   "cd '$resnet_r2_root' && env CUDA_VISIBLE_DEVICES=7 '$python_bin' experiments/resnet_crossover_r2.py --num-shards 2 --shard-index 1 > results/resnet_crossover/r2_gpu7.log 2>&1"
 tmux has-session -t resnet_r2_gpu5
@@ -171,6 +178,13 @@ if tmux has-session -t transformer_t2_gpu5 2>/dev/null ||
 fi
 tmux new-session -d -s transformer_t2_gpu5 \
   "cd '$transformer_t2_root' && env CUDA_VISIBLE_DEVICES=5 '$python_bin' experiments/transformer_crossover_t2.py --num-shards 2 --shard-index 0 > results/transformer_crossover/t2_gpu5.log 2>&1"
+transformer_t2_launch="$transformer_t2_root/results/transformer_crossover/t2_launch.json"
+until "$python_bin" "$validator" \
+    --path "$transformer_t2_launch" \
+    --kind transformer_t2_launch \
+    --expected-commit "$transformer_t2_commit" >/dev/null 2>&1; do
+  sleep 1
+done
 tmux new-session -d -s transformer_t2_gpu7 \
   "cd '$transformer_t2_root' && env CUDA_VISIBLE_DEVICES=7 '$python_bin' experiments/transformer_crossover_t2.py --num-shards 2 --shard-index 1 > results/transformer_crossover/t2_gpu7.log 2>&1"
 tmux has-session -t transformer_t2_gpu5
diff --git a/experiments/validate_crossover_artifact.py b/experiments/validate_crossover_artifact.py
index 8b82ec4..360c654 100644
--- a/experiments/validate_crossover_artifact.py
+++ b/experiments/validate_crossover_artifact.py
@@ -22,13 +22,23 @@ def validate_audit(report, count):
     assert report["test_policy"] == "none"
 
 
+def validate_launch(report, stage, count, expected_commit):
+    assert expected_commit
+    assert report["stage"] == stage
+    assert report["source"]["git_commit"] == expected_commit
+    assert report["num_jobs"] == count
+    assert report["allowed_physical_gpus"] == [5, 7]
+
+
 def validate(report, kind, expected_commit=None):
     if kind == "resnet_p1_launch":
-        assert expected_commit
-        assert report["stage"] == "p1"
-        assert report["source"]["git_commit"] == expected_commit
-        assert report["num_jobs"] == 19
-        assert report["allowed_physical_gpus"] == [5, 7]
+        validate_launch(report, "p1", 19, expected_commit)
+    elif kind == "transformer_p1_launch":
+        validate_launch(report, "p1", 27, expected_commit)
+    elif kind == "resnet_r2_launch":
+        validate_launch(report, "r2", 27, expected_commit)
+    elif kind == "transformer_t2_launch":
+        validate_launch(report, "t2", 27, expected_commit)
     elif kind == "resnet_p1_selector":
         validate_selector(report, "resnet_crossover_p1", 19)
     elif kind == "transformer_p1_selector":
@@ -54,6 +64,9 @@ def main():
         required=True,
         choices=(
             "resnet_p1_launch",
+            "transformer_p1_launch",
+            "resnet_r2_launch",
+            "transformer_t2_launch",
             "resnet_p1_selector",
             "transformer_p1_selector",
             "resnet_r2_audit",