1 files changed, 38 insertions, 0 deletions
diff --git a/research/flossing/monitor_dynamics_experiments.sh b/research/flossing/monitor_dynamics_experiments.sh
new file mode 100755
index 0000000..b7e74b9
--- /dev/null
+++ b/research/flossing/monitor_dynamics_experiments.sh
@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+set -eo pipefail
+
+FLOSS_DIR="/home/yurenh2/rrm/research/flossing"
+CONDA_SH="/home/yurenh2/miniconda3/etc/profile.d/conda.sh"
+LOG="${FLOSS_DIR}/monitor_dynamics_experiments.log"
+REPORT="${FLOSS_DIR}/dynamics_experiment_report.md"
+
+PATTERN='step3_M_volume_cf_26040_lstar_neg015|step7_C_hrm_engelken_interfloss_kl10|step7_D_trm_engelken_interfloss_kl10|step7_I_hrm_engelken_interfloss_kl100_short|launch_dynamics_variants_queue|step7_E_hrm_late|step7_F_trm_late|step7_G_hrm_volume|step7_H_trm_volume|step8_A_hrm_basin|step8_B_trm_basin|step9_[A-Z]_|launch_trajectory_perturb_queue|launch_trajectory_sampling_long'
+
+cd "${FLOSS_DIR}"
+source "${CONDA_SH}"
+conda activate rrm
+
+echo "[$(date --iso-8601=seconds)] monitor started" >> "${LOG}"
+echo "[$(date --iso-8601=seconds)] report target: ${REPORT}" >> "${LOG}"
+
+last_snapshot=""
+while true; do
+  active="$(pgrep -af "${PATTERN}" || true)"
+  if [[ -z "${active}" ]]; then
+    echo "[$(date --iso-8601=seconds)] no active monitored processes; generating final report" >> "${LOG}"
+    python analyze_dynamics_experiments.py > dynamics_experiment_report.stdout 2>&1
+    echo "[$(date --iso-8601=seconds)] final report generated" >> "${LOG}"
+    exit 0
+  fi
+
+  snapshot="$(echo "${active}" | sed 's/  */ /g' | cut -c1-260)"
+  if [[ "${snapshot}" != "${last_snapshot}" ]]; then
+    echo "[$(date --iso-8601=seconds)] active processes:" >> "${LOG}"
+    echo "${snapshot}" >> "${LOG}"
+    last_snapshot="${snapshot}"
+  fi
+
+  # Keep a rolling partial report useful while long queues are still running.
+  python analyze_dynamics_experiments.py > dynamics_experiment_report.stdout 2>&1 || true
+  sleep 300
+done