1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
|
#!/usr/bin/env bash
# Wait for the complete plain-CNN P2 shards, then launch the frozen ResNet P1
# selector on the only two authorized physical GPUs. This script never stops
# an existing process and refuses source or GPU-identity drift.
set -euo pipefail
if [ "$#" -ne 1 ]; then
echo "usage: $0 EXPECTED_RESNET_COMMIT" >&2
exit 2
fi
expected_commit="$1"
resnet_root="/home/yurenh2/sdil-resnet-p1"
python_bin="/home/yurenh2/miniconda3/envs/ep_pascal/bin/python3"
gpu5_uuid="GPU-f4aed756-228e-7522-94e7-fb543c02cc74"
gpu7_uuid="GPU-df3d5933-c3e9-9ea1-d595-9d2a3b68c14b"
require_frozen_source() {
test -d "$resnet_root/.git" -o -f "$resnet_root/.git"
test "$(git -C "$resnet_root" rev-parse HEAD)" = "$expected_commit"
test -z "$(git -C "$resnet_root" status --porcelain --untracked-files=no)"
test -x "$python_bin"
}
gpu_uuid() {
nvidia-smi \
--query-gpu=index,uuid \
--format=csv,noheader,nounits |
awk -F',' -v wanted="$1" '
{
gsub(/^[ \t]+|[ \t]+$/, "", $1)
gsub(/^[ \t]+|[ \t]+$/, "", $2)
if ($1 == wanted) print $2
}'
}
gpu_has_compute_process() {
nvidia-smi \
--query-compute-apps=gpu_uuid \
--format=csv,noheader,nounits 2>/dev/null |
awk -v wanted="$1" '
{
gsub(/^[ \t]+|[ \t]+$/, "", $0)
if ($0 == wanted) found=1
}
END { exit(found ? 0 : 1) }'
}
require_frozen_source
test "$(gpu_uuid 5)" = "$gpu5_uuid"
test "$(gpu_uuid 7)" = "$gpu7_uuid"
echo "handoff waiting for complete plain P2 shards at $(date --iso-8601=seconds)"
while tmux has-session -t plain_p2_gpu5 2>/dev/null ||
tmux has-session -t plain_p2_gpu7 2>/dev/null; do
sleep 60
done
echo "plain P2 sessions ended; waiting for authorized GPUs to become idle"
while gpu_has_compute_process "$gpu5_uuid" ||
gpu_has_compute_process "$gpu7_uuid"; do
sleep 60
done
require_frozen_source
test "$(gpu_uuid 5)" = "$gpu5_uuid"
test "$(gpu_uuid 7)" = "$gpu7_uuid"
if tmux has-session -t resnet_p1_gpu5 2>/dev/null ||
tmux has-session -t resnet_p1_gpu7 2>/dev/null; then
echo "a ResNet P1 session already exists; refusing duplicate launch" >&2
exit 1
fi
mkdir -p "$resnet_root/results/resnet_crossover"
tmux new-session -d -s resnet_p1_gpu5 \
"cd '$resnet_root' && env CUDA_VISIBLE_DEVICES=5 '$python_bin' experiments/resnet_crossover_grid.py --stage p1 --num-shards 2 --shard-index 0 > results/resnet_crossover/p1_gpu5.log 2>&1"
tmux new-session -d -s resnet_p1_gpu7 \
"cd '$resnet_root' && env CUDA_VISIBLE_DEVICES=7 '$python_bin' experiments/resnet_crossover_grid.py --stage p1 --num-shards 2 --shard-index 1 > results/resnet_crossover/p1_gpu7.log 2>&1"
tmux has-session -t resnet_p1_gpu5
tmux has-session -t resnet_p1_gpu7
echo "launched frozen ResNet P1 shards at $(date --iso-8601=seconds)"
|
