1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
|
#!/usr/bin/env bash
# After both P1 selectors are complete, bind them into frozen R2/T2
# worktrees, run the full 27+27 cells sequentially, audit every record, and
# commit the generated evidence on the corresponding run branches.
set -euo pipefail
if [ "$#" -ne 3 ]; then
echo "usage: $0 EXPECTED_P1_COMMIT EXPECTED_R2_BASE EXPECTED_T2_BASE" >&2
exit 2
fi
expected_p1_commit="$1"
expected_r2_base="$2"
expected_t2_base="$3"
resnet_p1_root="/home/yurenh2/sdil-resnet-p1"
transformer_p1_root="/home/yurenh2/sdil-transformer-p1"
resnet_r2_root="/home/yurenh2/sdil-resnet-r2"
transformer_t2_root="/home/yurenh2/sdil-transformer-t2"
plain_root="/scratch/yurenh2/sdil-crossover/dualprop_vgg16"
python_bin="/home/yurenh2/miniconda3/envs/ep_pascal/bin/python3"
gpu5_uuid="GPU-f4aed756-228e-7522-94e7-fb543c02cc74"
gpu7_uuid="GPU-df3d5933-c3e9-9ea1-d595-9d2a3b68c14b"
require_source() {
local root="$1"
local expected_commit="$2"
test "$(git -C "$root" rev-parse HEAD)" = "$expected_commit"
test -z "$(git -C "$root" status --porcelain --untracked-files=no)"
}
gpu_uuid() {
nvidia-smi \
--query-gpu=index,uuid \
--format=csv,noheader,nounits |
awk -F',' -v wanted="$1" '
{
gsub(/^[ \t]+|[ \t]+$/, "", $1)
gsub(/^[ \t]+|[ \t]+$/, "", $2)
if ($1 == wanted) print $2
}'
}
gpu_has_compute_process() {
nvidia-smi \
--query-compute-apps=gpu_uuid \
--format=csv,noheader,nounits 2>/dev/null |
awk -v wanted="$1" '
{
gsub(/^[ \t]+|[ \t]+$/, "", $0)
if ($0 == wanted) found=1
}
END { exit(found ? 0 : 1) }'
}
wait_for_idle_gpus() {
while gpu_has_compute_process "$gpu5_uuid" ||
gpu_has_compute_process "$gpu7_uuid"; do
sleep 60
done
}
manifest_count() {
local result_root="$1"
find "$result_root" -maxdepth 1 -type f \
-name '*.manifest.json' -print 2>/dev/null | wc -l
}
wait_for_file() {
while [ ! -f "$1" ]; do
sleep 60
done
}
validate_selector() {
local path="$1"
local stage="$2"
local count="$3"
jq -e --arg stage "$stage" --argjson count "$count" '
.gate == "pass" and
.stage == $stage and
.num_expected_records == $count and
.num_audited_records == $count and
.missing_experiments == [] and
(.selected | length) == 9 and
.test_policy == "none"
' "$path" >/dev/null
}
test -x "$python_bin"
test "$(gpu_uuid 5)" = "$gpu5_uuid"
test "$(gpu_uuid 7)" = "$gpu7_uuid"
require_source "$resnet_p1_root" "$expected_p1_commit"
require_source "$transformer_p1_root" "$expected_p1_commit"
require_source "$resnet_r2_root" "$expected_r2_base"
require_source "$transformer_t2_root" "$expected_t2_base"
resnet_p1_selector="$resnet_p1_root/results/resnet_crossover/p1_selector.json"
transformer_p1_selector="$transformer_p1_root/results/transformer_crossover/p1_selector.json"
echo "waiting for both audited P1 selectors at $(date --iso-8601=seconds)"
wait_for_file "$resnet_p1_selector"
wait_for_file "$transformer_p1_selector"
validate_selector "$resnet_p1_selector" "resnet_crossover_p1" 19
validate_selector \
"$transformer_p1_selector" "transformer_crossover_p1" 27
resnet_r2_selector="$resnet_r2_root/results/resnet_crossover/p1_selector.json"
transformer_t2_selector="$transformer_t2_root/results/transformer_crossover/p1_selector.json"
test ! -e "$resnet_r2_selector"
test ! -e "$transformer_t2_selector"
mkdir -p "$(dirname "$resnet_r2_selector")"
mkdir -p "$(dirname "$transformer_t2_selector")"
(
cd "$resnet_p1_root"
"$python_bin" experiments/analyze_resnet_crossover_p1.py \
--out "$resnet_r2_selector"
) >"$resnet_r2_root/results/resnet_crossover/p1_import.log" 2>&1
(
cd "$transformer_p1_root"
"$python_bin" experiments/analyze_transformer_crossover_p1.py \
--out "$transformer_t2_selector"
) >"$transformer_t2_root/results/transformer_crossover/p1_import.log" 2>&1
validate_selector "$resnet_r2_selector" "resnet_crossover_p1" 19
validate_selector \
"$transformer_t2_selector" "transformer_crossover_p1" 27
git -C "$resnet_r2_root" add -f \
results/resnet_crossover/p1_selector.json
git -C "$resnet_r2_root" commit -m \
"experiment: freeze ResNet crossover selector"
git -C "$transformer_t2_root" add -f \
results/transformer_crossover/p1_selector.json
git -C "$transformer_t2_root" commit -m \
"experiment: freeze Transformer crossover selector"
resnet_r2_commit="$(git -C "$resnet_r2_root" rev-parse HEAD)"
transformer_t2_commit="$(git -C "$transformer_t2_root" rev-parse HEAD)"
require_source "$resnet_r2_root" "$resnet_r2_commit"
require_source "$transformer_t2_root" "$transformer_t2_commit"
echo "selectors frozen; launching ResNet R2 at $(date --iso-8601=seconds)"
wait_for_idle_gpus
if tmux has-session -t resnet_r2_gpu5 2>/dev/null ||
tmux has-session -t resnet_r2_gpu7 2>/dev/null; then
echo "ResNet R2 session already exists; refusing duplicate launch" >&2
exit 1
fi
tmux new-session -d -s resnet_r2_gpu5 \
"cd '$resnet_r2_root' && env CUDA_VISIBLE_DEVICES=5 '$python_bin' experiments/resnet_crossover_r2.py --num-shards 2 --shard-index 0 > results/resnet_crossover/r2_gpu5.log 2>&1"
tmux new-session -d -s resnet_r2_gpu7 \
"cd '$resnet_r2_root' && env CUDA_VISIBLE_DEVICES=7 '$python_bin' experiments/resnet_crossover_r2.py --num-shards 2 --shard-index 1 > results/resnet_crossover/r2_gpu7.log 2>&1"
tmux has-session -t resnet_r2_gpu5
tmux has-session -t resnet_r2_gpu7
while tmux has-session -t resnet_r2_gpu5 2>/dev/null ||
tmux has-session -t resnet_r2_gpu7 2>/dev/null; do
sleep 60
done
test "$(manifest_count \
"$resnet_r2_root/results/resnet_crossover/r2")" -eq 27
require_source "$resnet_r2_root" "$resnet_r2_commit"
(
cd "$resnet_r2_root"
"$python_bin" experiments/analyze_resnet_crossover_r2.py \
--out results/resnet_crossover/r2_audit.json
) >"$resnet_r2_root/results/resnet_crossover/r2_audit.log" 2>&1
jq -e '
.audit_status == "passed" and
.complete_grid == true and
.failure_retaining == true and
.num_audited_cells == 27 and
.test_policy == "none"
' "$resnet_r2_root/results/resnet_crossover/r2_audit.json" >/dev/null
git -C "$resnet_r2_root" add -f results/resnet_crossover
git -C "$resnet_r2_root" commit -m \
"results: record complete ResNet crossover"
echo "ResNet R2 audited and committed at $(date --iso-8601=seconds)"
wait_for_idle_gpus
require_source "$transformer_t2_root" "$transformer_t2_commit"
if tmux has-session -t transformer_t2_gpu5 2>/dev/null ||
tmux has-session -t transformer_t2_gpu7 2>/dev/null; then
echo "Transformer T2 session already exists; refusing duplicate launch" >&2
exit 1
fi
tmux new-session -d -s transformer_t2_gpu5 \
"cd '$transformer_t2_root' && env CUDA_VISIBLE_DEVICES=5 '$python_bin' experiments/transformer_crossover_t2.py --num-shards 2 --shard-index 0 > results/transformer_crossover/t2_gpu5.log 2>&1"
tmux new-session -d -s transformer_t2_gpu7 \
"cd '$transformer_t2_root' && env CUDA_VISIBLE_DEVICES=7 '$python_bin' experiments/transformer_crossover_t2.py --num-shards 2 --shard-index 1 > results/transformer_crossover/t2_gpu7.log 2>&1"
tmux has-session -t transformer_t2_gpu5
tmux has-session -t transformer_t2_gpu7
while tmux has-session -t transformer_t2_gpu5 2>/dev/null ||
tmux has-session -t transformer_t2_gpu7 2>/dev/null; do
sleep 60
done
test "$(manifest_count \
"$transformer_t2_root/results/transformer_crossover/t2")" -eq 27
require_source "$transformer_t2_root" "$transformer_t2_commit"
(
cd "$transformer_t2_root"
"$python_bin" experiments/analyze_transformer_crossover_t2.py \
--out results/transformer_crossover/t2_audit.json
) >"$transformer_t2_root/results/transformer_crossover/t2_audit.log" 2>&1
jq -e '
.audit_status == "passed" and
.complete_grid == true and
.failure_retaining == true and
.num_audited_cells == 27 and
.test_policy == "none"
' "$transformer_t2_root/results/transformer_crossover/t2_audit.json" \
>/dev/null
plain_audit="$transformer_t2_root/results/plain_cnn_p2_audit.json"
(
cd "$plain_root"
"$python_bin" analyze_p2.py --output "$plain_audit"
)
"$python_bin" "$transformer_t2_root/experiments/analyze_crossover_81.py" \
--plain "$plain_audit" \
--resnet "$resnet_r2_root/results/resnet_crossover/r2_audit.json" \
--transformer \
"$transformer_t2_root/results/transformer_crossover/t2_audit.json" \
--out "$transformer_t2_root/results/crossover_81_audit.json"
jq -e '
.audit_status == "passed" and
.complete_grid == true and
.failure_retaining == true and
.num_audited_cells == 81 and
.test_policy == "none"
' "$transformer_t2_root/results/crossover_81_audit.json" >/dev/null
git -C "$transformer_t2_root" add -f \
results/transformer_crossover \
results/plain_cnn_p2_audit.json \
results/crossover_81_audit.json
git -C "$transformer_t2_root" commit -m \
"results: record complete 81-cell crossover"
echo "complete 81-cell crossover audited and committed at $(date --iso-8601=seconds)"
|
