#!/usr/bin/env bash
# Wait for the complete plain-CNN P2 shards, then launch the frozen ResNet P1
# selector on the only two authorized physical GPUs. This script never stops
# an existing process and refuses source or GPU-identity drift.
set -euo pipefail

if [ "$#" -ne 1 ]; then
  echo "usage: $0 EXPECTED_RESNET_COMMIT" >&2
  exit 2
fi

expected_commit="$1"
resnet_root="/scratch/yurenh2/sdil-runs/resnet-p1"
python_bin="/home/yurenh2/miniconda3/envs/ep_pascal/bin/python3"
validator="/home/yurenh2/sdil/experiments/validate_crossover_artifact.py"
gpu5_uuid="GPU-f4aed756-228e-7522-94e7-fb543c02cc74"
gpu7_uuid="GPU-df3d5933-c3e9-9ea1-d595-9d2a3b68c14b"

require_frozen_source() {
  test -d "$resnet_root/.git" -o -f "$resnet_root/.git"
  test "$(git -C "$resnet_root" rev-parse HEAD)" = "$expected_commit"
  test -z "$(git -C "$resnet_root" status --porcelain --untracked-files=no)"
  test -x "$python_bin"
}

gpu_uuid() {
  nvidia-smi \
    --query-gpu=index,uuid \
    --format=csv,noheader,nounits |
    awk -F',' -v wanted="$1" '
      {
        gsub(/^[ \t]+|[ \t]+$/, "", $1)
        gsub(/^[ \t]+|[ \t]+$/, "", $2)
        if ($1 == wanted) print $2
      }'
}

gpu_has_compute_process() {
  nvidia-smi \
    --query-compute-apps=gpu_uuid \
    --format=csv,noheader,nounits 2>/dev/null |
    awk -v wanted="$1" '
      {
        gsub(/^[ \t]+|[ \t]+$/, "", $0)
        if ($0 == wanted) found=1
      }
      END { exit(found ? 0 : 1) }'
}

require_frozen_source
test "$(gpu_uuid 5)" = "$gpu5_uuid"
test "$(gpu_uuid 7)" = "$gpu7_uuid"

echo "handoff waiting for complete plain P2 shards at $(date --iso-8601=seconds)"
while tmux has-session -t plain_p2_gpu5 2>/dev/null ||
      tmux has-session -t plain_p2_gpu7 2>/dev/null; do
  sleep 60
done

echo "plain P2 sessions ended; waiting for authorized GPUs to become idle"
while gpu_has_compute_process "$gpu5_uuid" ||
      gpu_has_compute_process "$gpu7_uuid"; do
  sleep 60
done

require_frozen_source
test "$(gpu_uuid 5)" = "$gpu5_uuid"
test "$(gpu_uuid 7)" = "$gpu7_uuid"
if tmux has-session -t resnet_p1_gpu5 2>/dev/null ||
   tmux has-session -t resnet_p1_gpu7 2>/dev/null; then
  echo "a ResNet P1 session already exists; refusing duplicate launch" >&2
  exit 1
fi

mkdir -p "$resnet_root/results/resnet_crossover"
tmux new-session -d -s resnet_p1_gpu5 \
  "cd '$resnet_root' && env CUDA_VISIBLE_DEVICES=5 '$python_bin' experiments/resnet_crossover_grid.py --stage p1 --num-shards 2 --shard-index 0 > results/resnet_crossover/p1_gpu5.log 2>&1"
resnet_launch="$resnet_root/results/resnet_crossover/p1_launch.json"
until "$python_bin" "$validator" \
    --path "$resnet_launch" \
    --kind resnet_p1_launch \
    --expected-commit "$expected_commit" >/dev/null 2>&1; do
  sleep 1
done
tmux new-session -d -s resnet_p1_gpu7 \
  "cd '$resnet_root' && env CUDA_VISIBLE_DEVICES=7 '$python_bin' experiments/resnet_crossover_grid.py --stage p1 --num-shards 2 --shard-index 1 > results/resnet_crossover/p1_gpu7.log 2>&1"
tmux has-session -t resnet_p1_gpu5
tmux has-session -t resnet_p1_gpu7
echo "launched frozen ResNet P1 shards at $(date --iso-8601=seconds)"