diff options
Diffstat (limited to 'experiments/handoff_plain_to_resnet_p1.sh')
| -rwxr-xr-x | experiments/handoff_plain_to_resnet_p1.sh | 8 |
1 files changed, 8 insertions, 0 deletions
diff --git a/experiments/handoff_plain_to_resnet_p1.sh b/experiments/handoff_plain_to_resnet_p1.sh index c5656a2..04fcdea 100755 --- a/experiments/handoff_plain_to_resnet_p1.sh +++ b/experiments/handoff_plain_to_resnet_p1.sh @@ -12,6 +12,7 @@ fi expected_commit="$1" resnet_root="/scratch/yurenh2/sdil-runs/resnet-p1" python_bin="/home/yurenh2/miniconda3/envs/ep_pascal/bin/python3" +validator="/home/yurenh2/sdil/experiments/validate_crossover_artifact.py" gpu5_uuid="GPU-f4aed756-228e-7522-94e7-fb543c02cc74" gpu7_uuid="GPU-df3d5933-c3e9-9ea1-d595-9d2a3b68c14b" @@ -74,6 +75,13 @@ fi mkdir -p "$resnet_root/results/resnet_crossover" tmux new-session -d -s resnet_p1_gpu5 \ "cd '$resnet_root' && env CUDA_VISIBLE_DEVICES=5 '$python_bin' experiments/resnet_crossover_grid.py --stage p1 --num-shards 2 --shard-index 0 > results/resnet_crossover/p1_gpu5.log 2>&1" +resnet_launch="$resnet_root/results/resnet_crossover/p1_launch.json" +until "$python_bin" "$validator" \ + --path "$resnet_launch" \ + --kind resnet_p1_launch \ + --expected-commit "$expected_commit" >/dev/null 2>&1; do + sleep 1 +done tmux new-session -d -s resnet_p1_gpu7 \ "cd '$resnet_root' && env CUDA_VISIBLE_DEVICES=7 '$python_bin' experiments/resnet_crossover_grid.py --stage p1 --num-shards 2 --shard-index 1 > results/resnet_crossover/p1_gpu7.log 2>&1" tmux has-session -t resnet_p1_gpu5 |