#!/usr/bin/env bash
set -eo pipefail

FLOSS_DIR="/home/yurenh2/rrm/research/flossing"
CONDA_SH="/home/yurenh2/miniconda3/etc/profile.d/conda.sh"
LOG="${FLOSS_DIR}/monitor_dynamics_experiments.log"
REPORT="${FLOSS_DIR}/dynamics_experiment_report.md"

PATTERN='step3_M_volume_cf_26040_lstar_neg015|step7_C_hrm_engelken_interfloss_kl10|step7_D_trm_engelken_interfloss_kl10|step7_I_hrm_engelken_interfloss_kl100_short|launch_dynamics_variants_queue|step7_E_hrm_late|step7_F_trm_late|step7_G_hrm_volume|step7_H_trm_volume|step8_A_hrm_basin|step8_B_trm_basin|step9_[A-Z]_|launch_trajectory_perturb_queue|launch_trajectory_sampling_long'

cd "${FLOSS_DIR}"
source "${CONDA_SH}"
conda activate rrm

echo "[$(date --iso-8601=seconds)] monitor started" >> "${LOG}"
echo "[$(date --iso-8601=seconds)] report target: ${REPORT}" >> "${LOG}"

last_snapshot=""
while true; do
  active="$(pgrep -af "${PATTERN}" || true)"
  if [[ -z "${active}" ]]; then
    echo "[$(date --iso-8601=seconds)] no active monitored processes; generating final report" >> "${LOG}"
    python analyze_dynamics_experiments.py > dynamics_experiment_report.stdout 2>&1
    echo "[$(date --iso-8601=seconds)] final report generated" >> "${LOG}"
    exit 0
  fi

  snapshot="$(echo "${active}" | sed 's/  */ /g' | cut -c1-260)"
  if [[ "${snapshot}" != "${last_snapshot}" ]]; then
    echo "[$(date --iso-8601=seconds)] active processes:" >> "${LOG}"
    echo "${snapshot}" >> "${LOG}"
    last_snapshot="${snapshot}"
  fi

  # Keep a rolling partial report useful while long queues are still running.
  python analyze_dynamics_experiments.py > dynamics_experiment_report.stdout 2>&1 || true
  sleep 300
done