blob: b7e74b9f032b1a82234c99abf332bde649e8362c (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
|
#!/usr/bin/env bash
set -eo pipefail
FLOSS_DIR="/home/yurenh2/rrm/research/flossing"
CONDA_SH="/home/yurenh2/miniconda3/etc/profile.d/conda.sh"
LOG="${FLOSS_DIR}/monitor_dynamics_experiments.log"
REPORT="${FLOSS_DIR}/dynamics_experiment_report.md"
PATTERN='step3_M_volume_cf_26040_lstar_neg015|step7_C_hrm_engelken_interfloss_kl10|step7_D_trm_engelken_interfloss_kl10|step7_I_hrm_engelken_interfloss_kl100_short|launch_dynamics_variants_queue|step7_E_hrm_late|step7_F_trm_late|step7_G_hrm_volume|step7_H_trm_volume|step8_A_hrm_basin|step8_B_trm_basin|step9_[A-Z]_|launch_trajectory_perturb_queue|launch_trajectory_sampling_long'
cd "${FLOSS_DIR}"
source "${CONDA_SH}"
conda activate rrm
echo "[$(date --iso-8601=seconds)] monitor started" >> "${LOG}"
echo "[$(date --iso-8601=seconds)] report target: ${REPORT}" >> "${LOG}"
last_snapshot=""
while true; do
active="$(pgrep -af "${PATTERN}" || true)"
if [[ -z "${active}" ]]; then
echo "[$(date --iso-8601=seconds)] no active monitored processes; generating final report" >> "${LOG}"
python analyze_dynamics_experiments.py > dynamics_experiment_report.stdout 2>&1
echo "[$(date --iso-8601=seconds)] final report generated" >> "${LOG}"
exit 0
fi
snapshot="$(echo "${active}" | sed 's/ */ /g' | cut -c1-260)"
if [[ "${snapshot}" != "${last_snapshot}" ]]; then
echo "[$(date --iso-8601=seconds)] active processes:" >> "${LOG}"
echo "${snapshot}" >> "${LOG}"
last_snapshot="${snapshot}"
fi
# Keep a rolling partial report useful while long queues are still running.
python analyze_dynamics_experiments.py > dynamics_experiment_report.stdout 2>&1 || true
sleep 300
done
|
