diff options
Diffstat (limited to 'experiments/handoff_selectors_to_full_crossover.sh')
| -rwxr-xr-x | experiments/handoff_selectors_to_full_crossover.sh | 14 |
1 files changed, 14 insertions, 0 deletions
diff --git a/experiments/handoff_selectors_to_full_crossover.sh b/experiments/handoff_selectors_to_full_crossover.sh index e87fea8..12670db 100755 --- a/experiments/handoff_selectors_to_full_crossover.sh +++ b/experiments/handoff_selectors_to_full_crossover.sh @@ -138,6 +138,13 @@ if tmux has-session -t resnet_r2_gpu5 2>/dev/null || fi tmux new-session -d -s resnet_r2_gpu5 \ "cd '$resnet_r2_root' && env CUDA_VISIBLE_DEVICES=5 '$python_bin' experiments/resnet_crossover_r2.py --num-shards 2 --shard-index 0 > results/resnet_crossover/r2_gpu5.log 2>&1" +resnet_r2_launch="$resnet_r2_root/results/resnet_crossover/r2_launch.json" +until "$python_bin" "$validator" \ + --path "$resnet_r2_launch" \ + --kind resnet_r2_launch \ + --expected-commit "$resnet_r2_commit" >/dev/null 2>&1; do + sleep 1 +done tmux new-session -d -s resnet_r2_gpu7 \ "cd '$resnet_r2_root' && env CUDA_VISIBLE_DEVICES=7 '$python_bin' experiments/resnet_crossover_r2.py --num-shards 2 --shard-index 1 > results/resnet_crossover/r2_gpu7.log 2>&1" tmux has-session -t resnet_r2_gpu5 @@ -171,6 +178,13 @@ if tmux has-session -t transformer_t2_gpu5 2>/dev/null || fi tmux new-session -d -s transformer_t2_gpu5 \ "cd '$transformer_t2_root' && env CUDA_VISIBLE_DEVICES=5 '$python_bin' experiments/transformer_crossover_t2.py --num-shards 2 --shard-index 0 > results/transformer_crossover/t2_gpu5.log 2>&1" +transformer_t2_launch="$transformer_t2_root/results/transformer_crossover/t2_launch.json" +until "$python_bin" "$validator" \ + --path "$transformer_t2_launch" \ + --kind transformer_t2_launch \ + --expected-commit "$transformer_t2_commit" >/dev/null 2>&1; do + sleep 1 +done tmux new-session -d -s transformer_t2_gpu7 \ "cd '$transformer_t2_root' && env CUDA_VISIBLE_DEVICES=7 '$python_bin' experiments/transformer_crossover_t2.py --num-shards 2 --shard-index 1 > results/transformer_crossover/t2_gpu7.log 2>&1" tmux has-session -t transformer_t2_gpu5 |