experiment: queue complete 81-cell crossover

1 files changed, 233 insertions, 0 deletions

diff --git a/experiments/handoff_selectors_to_full_crossover.sh b/experiments/handoff_selectors_to_full_crossover.sh
new file mode 100755
index 0000000..04d1b5d
--- /dev/null
+++ b/experiments/handoff_selectors_to_full_crossover.sh
@@ -0,0 +1,233 @@
+#!/usr/bin/env bash
+# After both P1 selectors are complete, bind them into frozen R2/T2
+# worktrees, run the full 27+27 cells sequentially, audit every record, and
+# commit the generated evidence on the corresponding run branches.
+set -euo pipefail
+
+if [ "$#" -ne 3 ]; then
+  echo "usage: $0 EXPECTED_P1_COMMIT EXPECTED_R2_BASE EXPECTED_T2_BASE" >&2
+  exit 2
+fi
+
+expected_p1_commit="$1"
+expected_r2_base="$2"
+expected_t2_base="$3"
+resnet_p1_root="/home/yurenh2/sdil-resnet-p1"
+transformer_p1_root="/home/yurenh2/sdil-transformer-p1"
+resnet_r2_root="/home/yurenh2/sdil-resnet-r2"
+transformer_t2_root="/home/yurenh2/sdil-transformer-t2"
+plain_root="/scratch/yurenh2/sdil-crossover/dualprop_vgg16"
+python_bin="/home/yurenh2/miniconda3/envs/ep_pascal/bin/python3"
+gpu5_uuid="GPU-f4aed756-228e-7522-94e7-fb543c02cc74"
+gpu7_uuid="GPU-df3d5933-c3e9-9ea1-d595-9d2a3b68c14b"
+
+require_source() {
+  local root="$1"
+  local expected_commit="$2"
+  test "$(git -C "$root" rev-parse HEAD)" = "$expected_commit"
+  test -z "$(git -C "$root" status --porcelain --untracked-files=no)"
+}
+
+gpu_uuid() {
+  nvidia-smi \
+    --query-gpu=index,uuid \
+    --format=csv,noheader,nounits |
+    awk -F',' -v wanted="$1" '
+      {
+        gsub(/^[ \t]+|[ \t]+$/, "", $1)
+        gsub(/^[ \t]+|[ \t]+$/, "", $2)
+        if ($1 == wanted) print $2
+      }'
+}
+
+gpu_has_compute_process() {
+  nvidia-smi \
+    --query-compute-apps=gpu_uuid \
+    --format=csv,noheader,nounits 2>/dev/null |
+    awk -v wanted="$1" '
+      {
+        gsub(/^[ \t]+|[ \t]+$/, "", $0)
+        if ($0 == wanted) found=1
+      }
+      END { exit(found ? 0 : 1) }'
+}
+
+wait_for_idle_gpus() {
+  while gpu_has_compute_process "$gpu5_uuid" ||
+        gpu_has_compute_process "$gpu7_uuid"; do
+    sleep 60
+  done
+}
+
+manifest_count() {
+  local result_root="$1"
+  find "$result_root" -maxdepth 1 -type f \
+    -name '*.manifest.json' -print 2>/dev/null | wc -l
+}
+
+wait_for_file() {
+  while [ ! -f "$1" ]; do
+    sleep 60
+  done
+}
+
+validate_selector() {
+  local path="$1"
+  local stage="$2"
+  local count="$3"
+  jq -e --arg stage "$stage" --argjson count "$count" '
+    .gate == "pass" and
+    .stage == $stage and
+    .num_expected_records == $count and
+    .num_audited_records == $count and
+    .missing_experiments == [] and
+    (.selected | length) == 9 and
+    .test_policy == "none"
+  ' "$path" >/dev/null
+}
+
+test -x "$python_bin"
+test "$(gpu_uuid 5)" = "$gpu5_uuid"
+test "$(gpu_uuid 7)" = "$gpu7_uuid"
+require_source "$resnet_p1_root" "$expected_p1_commit"
+require_source "$transformer_p1_root" "$expected_p1_commit"
+require_source "$resnet_r2_root" "$expected_r2_base"
+require_source "$transformer_t2_root" "$expected_t2_base"
+
+resnet_p1_selector="$resnet_p1_root/results/resnet_crossover/p1_selector.json"
+transformer_p1_selector="$transformer_p1_root/results/transformer_crossover/p1_selector.json"
+echo "waiting for both audited P1 selectors at $(date --iso-8601=seconds)"
+wait_for_file "$resnet_p1_selector"
+wait_for_file "$transformer_p1_selector"
+validate_selector "$resnet_p1_selector" "resnet_crossover_p1" 19
+validate_selector \
+  "$transformer_p1_selector" "transformer_crossover_p1" 27
+
+resnet_r2_selector="$resnet_r2_root/results/resnet_crossover/p1_selector.json"
+transformer_t2_selector="$transformer_t2_root/results/transformer_crossover/p1_selector.json"
+test ! -e "$resnet_r2_selector"
+test ! -e "$transformer_t2_selector"
+mkdir -p "$(dirname "$resnet_r2_selector")"
+mkdir -p "$(dirname "$transformer_t2_selector")"
+(
+  cd "$resnet_p1_root"
+  "$python_bin" experiments/analyze_resnet_crossover_p1.py \
+    --out "$resnet_r2_selector"
+) >"$resnet_r2_root/results/resnet_crossover/p1_import.log" 2>&1
+(
+  cd "$transformer_p1_root"
+  "$python_bin" experiments/analyze_transformer_crossover_p1.py \
+    --out "$transformer_t2_selector"
+) >"$transformer_t2_root/results/transformer_crossover/p1_import.log" 2>&1
+validate_selector "$resnet_r2_selector" "resnet_crossover_p1" 19
+validate_selector \
+  "$transformer_t2_selector" "transformer_crossover_p1" 27
+git -C "$resnet_r2_root" add -f \
+  results/resnet_crossover/p1_selector.json
+git -C "$resnet_r2_root" commit -m \
+  "experiment: freeze ResNet crossover selector"
+git -C "$transformer_t2_root" add -f \
+  results/transformer_crossover/p1_selector.json
+git -C "$transformer_t2_root" commit -m \
+  "experiment: freeze Transformer crossover selector"
+resnet_r2_commit="$(git -C "$resnet_r2_root" rev-parse HEAD)"
+transformer_t2_commit="$(git -C "$transformer_t2_root" rev-parse HEAD)"
+require_source "$resnet_r2_root" "$resnet_r2_commit"
+require_source "$transformer_t2_root" "$transformer_t2_commit"
+echo "selectors frozen; launching ResNet R2 at $(date --iso-8601=seconds)"
+
+wait_for_idle_gpus
+if tmux has-session -t resnet_r2_gpu5 2>/dev/null ||
+   tmux has-session -t resnet_r2_gpu7 2>/dev/null; then
+  echo "ResNet R2 session already exists; refusing duplicate launch" >&2
+  exit 1
+fi
+tmux new-session -d -s resnet_r2_gpu5 \
+  "cd '$resnet_r2_root' && env CUDA_VISIBLE_DEVICES=5 '$python_bin' experiments/resnet_crossover_r2.py --num-shards 2 --shard-index 0 > results/resnet_crossover/r2_gpu5.log 2>&1"
+tmux new-session -d -s resnet_r2_gpu7 \
+  "cd '$resnet_r2_root' && env CUDA_VISIBLE_DEVICES=7 '$python_bin' experiments/resnet_crossover_r2.py --num-shards 2 --shard-index 1 > results/resnet_crossover/r2_gpu7.log 2>&1"
+tmux has-session -t resnet_r2_gpu5
+tmux has-session -t resnet_r2_gpu7
+while tmux has-session -t resnet_r2_gpu5 2>/dev/null ||
+      tmux has-session -t resnet_r2_gpu7 2>/dev/null; do
+  sleep 60
+done
+test "$(manifest_count \
+  "$resnet_r2_root/results/resnet_crossover/r2")" -eq 27
+require_source "$resnet_r2_root" "$resnet_r2_commit"
+(
+  cd "$resnet_r2_root"
+  "$python_bin" experiments/analyze_resnet_crossover_r2.py \
+    --out results/resnet_crossover/r2_audit.json
+) >"$resnet_r2_root/results/resnet_crossover/r2_audit.log" 2>&1
+jq -e '
+  .audit_status == "passed" and
+  .complete_grid == true and
+  .failure_retaining == true and
+  .num_audited_cells == 27 and
+  .test_policy == "none"
+' "$resnet_r2_root/results/resnet_crossover/r2_audit.json" >/dev/null
+git -C "$resnet_r2_root" add -f results/resnet_crossover
+git -C "$resnet_r2_root" commit -m \
+  "results: record complete ResNet crossover"
+echo "ResNet R2 audited and committed at $(date --iso-8601=seconds)"
+
+wait_for_idle_gpus
+require_source "$transformer_t2_root" "$transformer_t2_commit"
+if tmux has-session -t transformer_t2_gpu5 2>/dev/null ||
+   tmux has-session -t transformer_t2_gpu7 2>/dev/null; then
+  echo "Transformer T2 session already exists; refusing duplicate launch" >&2
+  exit 1
+fi
+tmux new-session -d -s transformer_t2_gpu5 \
+  "cd '$transformer_t2_root' && env CUDA_VISIBLE_DEVICES=5 '$python_bin' experiments/transformer_crossover_t2.py --num-shards 2 --shard-index 0 > results/transformer_crossover/t2_gpu5.log 2>&1"
+tmux new-session -d -s transformer_t2_gpu7 \
+  "cd '$transformer_t2_root' && env CUDA_VISIBLE_DEVICES=7 '$python_bin' experiments/transformer_crossover_t2.py --num-shards 2 --shard-index 1 > results/transformer_crossover/t2_gpu7.log 2>&1"
+tmux has-session -t transformer_t2_gpu5
+tmux has-session -t transformer_t2_gpu7
+while tmux has-session -t transformer_t2_gpu5 2>/dev/null ||
+      tmux has-session -t transformer_t2_gpu7 2>/dev/null; do
+  sleep 60
+done
+test "$(manifest_count \
+  "$transformer_t2_root/results/transformer_crossover/t2")" -eq 27
+require_source "$transformer_t2_root" "$transformer_t2_commit"
+(
+  cd "$transformer_t2_root"
+  "$python_bin" experiments/analyze_transformer_crossover_t2.py \
+    --out results/transformer_crossover/t2_audit.json
+) >"$transformer_t2_root/results/transformer_crossover/t2_audit.log" 2>&1
+jq -e '
+  .audit_status == "passed" and
+  .complete_grid == true and
+  .failure_retaining == true and
+  .num_audited_cells == 27 and
+  .test_policy == "none"
+' "$transformer_t2_root/results/transformer_crossover/t2_audit.json" \
+  >/dev/null
+
+plain_audit="$transformer_t2_root/results/plain_cnn_p2_audit.json"
+(
+  cd "$plain_root"
+  "$python_bin" analyze_p2.py --output "$plain_audit"
+)
+"$python_bin" "$transformer_t2_root/experiments/analyze_crossover_81.py" \
+  --plain "$plain_audit" \
+  --resnet "$resnet_r2_root/results/resnet_crossover/r2_audit.json" \
+  --transformer \
+    "$transformer_t2_root/results/transformer_crossover/t2_audit.json" \
+  --out "$transformer_t2_root/results/crossover_81_audit.json"
+jq -e '
+  .audit_status == "passed" and
+  .complete_grid == true and
+  .failure_retaining == true and
+  .num_audited_cells == 81 and
+  .test_policy == "none"
+' "$transformer_t2_root/results/crossover_81_audit.json" >/dev/null
+git -C "$transformer_t2_root" add -f \
+  results/transformer_crossover \
+  results/plain_cnn_p2_audit.json \
+  results/crossover_81_audit.json
+git -C "$transformer_t2_root" commit -m \
+  "results: record complete 81-cell crossover"
+echo "complete 81-cell crossover audited and committed at $(date --iso-8601=seconds)"