diff options
| -rwxr-xr-x | experiments/handoff_plain_to_resnet_p1.sh | 81 |
1 files changed, 81 insertions, 0 deletions
diff --git a/experiments/handoff_plain_to_resnet_p1.sh b/experiments/handoff_plain_to_resnet_p1.sh new file mode 100755 index 0000000..a7a1acb --- /dev/null +++ b/experiments/handoff_plain_to_resnet_p1.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# Wait for the complete plain-CNN P2 shards, then launch the frozen ResNet P1 +# selector on the only two authorized physical GPUs. This script never stops +# an existing process and refuses source or GPU-identity drift. +set -euo pipefail + +if [ "$#" -ne 1 ]; then + echo "usage: $0 EXPECTED_RESNET_COMMIT" >&2 + exit 2 +fi + +expected_commit="$1" +resnet_root="/home/yurenh2/sdil-resnet-p1" +python_bin="/home/yurenh2/miniconda3/envs/ep_pascal/bin/python3" +gpu5_uuid="GPU-f4aed756-228e-7522-94e7-fb543c02cc74" +gpu7_uuid="GPU-df3d5933-c3e9-9ea1-d595-9d2a3b68c14b" + +require_frozen_source() { + test -d "$resnet_root/.git" -o -f "$resnet_root/.git" + test "$(git -C "$resnet_root" rev-parse HEAD)" = "$expected_commit" + test -z "$(git -C "$resnet_root" status --porcelain --untracked-files=no)" + test -x "$python_bin" +} + +gpu_uuid() { + nvidia-smi \ + --query-gpu=index,uuid \ + --format=csv,noheader,nounits | + awk -F',' -v wanted="$1" ' + { + gsub(/^[ \t]+|[ \t]+$/, "", $1) + gsub(/^[ \t]+|[ \t]+$/, "", $2) + if ($1 == wanted) print $2 + }' +} + +gpu_has_compute_process() { + nvidia-smi \ + --query-compute-apps=gpu_uuid \ + --format=csv,noheader,nounits 2>/dev/null | + awk -v wanted="$1" ' + { + gsub(/^[ \t]+|[ \t]+$/, "", $0) + if ($0 == wanted) found=1 + } + END { exit(found ? 0 : 1) }' +} + +require_frozen_source +test "$(gpu_uuid 5)" = "$gpu5_uuid" +test "$(gpu_uuid 7)" = "$gpu7_uuid" + +echo "handoff waiting for complete plain P2 shards at $(date --iso-8601=seconds)" +while tmux has-session -t plain_p2_gpu5 2>/dev/null || + tmux has-session -t plain_p2_gpu7 2>/dev/null; do + sleep 60 +done + +echo "plain P2 sessions ended; waiting for authorized GPUs to become idle" +while gpu_has_compute_process "$gpu5_uuid" || + gpu_has_compute_process "$gpu7_uuid"; do + sleep 60 +done + +require_frozen_source +test "$(gpu_uuid 5)" = "$gpu5_uuid" +test "$(gpu_uuid 7)" = "$gpu7_uuid" +if tmux has-session -t resnet_p1_gpu5 2>/dev/null || + tmux has-session -t resnet_p1_gpu7 2>/dev/null; then + echo "a ResNet P1 session already exists; refusing duplicate launch" >&2 + exit 1 +fi + +mkdir -p "$resnet_root/results/resnet_crossover" +tmux new-session -d -s resnet_p1_gpu5 \ + "cd '$resnet_root' && env CUDA_VISIBLE_DEVICES=5 '$python_bin' experiments/resnet_crossover_grid.py --stage p1 --num-shards 2 --shard-index 0 > results/resnet_crossover/p1_gpu5.log 2>&1" +tmux new-session -d -s resnet_p1_gpu7 \ + "cd '$resnet_root' && env CUDA_VISIBLE_DEVICES=7 '$python_bin' experiments/resnet_crossover_grid.py --stage p1 --num-shards 2 --shard-index 1 > results/resnet_crossover/p1_gpu7.log 2>&1" +tmux has-session -t resnet_p1_gpu5 +tmux has-session -t resnet_p1_gpu7 +echo "launched frozen ResNet P1 shards at $(date --iso-8601=seconds)" |