blob: 4a44936758ba5ee05a25f13415cf997b290ccc25 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
|
#!/usr/bin/env bash
set -euo pipefail
ROOT=/home/yurenh2/rrm
PY=/home/yurenh2/miniconda3/envs/rrm/bin/python
cd "${ROOT}"
DIR=research/flossing/directional_lyap_perturb_eps_sweep
BASE_CKPT="${ROOT}/trm/checkpoints/Sudoku-extreme-1k-aug-1000-ACT-torch/pretrain_mlp_t_sudoku_official_gbs768_repro"
MULTI_CKPT="${ROOT}/trm/checkpoints/Sudoku-extreme-1k-aug-1000-ACT-torch/pretrain_mlp_t_sudoku_official_gbs768_multi4_loguniform_repro"
SIGMAS='0,0.001,0.003,0.01,0.03,0.1'
AFTERS='0,4,8,12'
N=1000
CAND=8
BS=16
SEED=20260608
wait_pid_file() {
local pf="$1"
local pid
pid=$(cat "${pf}")
echo "watch ${pf}: ${pid}"
while kill -0 "${pid}" 2>/dev/null; do
sleep 60
done
echo "done ${pf}: ${pid}"
}
launch_multi_eps003_on_gpu0() {
local out=trm_multi4_best_step35805_n1000_c8_fdeps003
local log="${DIR}/logs/${out}.log"
local pidf="${DIR}/logs/${out}.pid"
setsid bash -c "cd '${ROOT}' && export CUDA_VISIBLE_DEVICES='0' PYTHONUNBUFFERED=1 && exec '${PY}' research/flossing/directional_lyap_perturb_robustness.py --ckpt-root '${MULTI_CKPT}' --ckpt-name step_35805 --label trm_multi4_best --n-samples '${N}' --batch-size '${BS}' --candidates '${CAND}' --fd-eps 0.03 --sigmas '${SIGMAS}' --perturb-afters '${AFTERS}' --seed '${SEED}' --out-prefix '${DIR}/${out}'" \
> "${log}" 2>&1 < /dev/null &
echo $! > "${pidf}"
echo "launched ${out}: $(cat "${pidf}")"
}
wait_pid_file "${DIR}/logs/trm_baseline_best_step58590_n1000_c8_fdeps001.pid"
launch_multi_eps003_on_gpu0
wait_pid_file "${DIR}/logs/trm_multi4_best_step35805_n1000_c8_fdeps001.pid"
wait_pid_file "${DIR}/logs/trm_baseline_best_step58590_n1000_c8_fdeps003.pid"
wait_pid_file "${DIR}/logs/trm_multi4_best_step35805_n1000_c8_fdeps003.pid"
"${PY}" research/flossing/plot_directional_lyap_perturb.py \
--summaries \
"${DIR}/trm_baseline_best_step58590_n1000_c8_fdeps001.summary.csv" \
"${DIR}/trm_multi4_best_step35805_n1000_c8_fdeps001.summary.csv" \
"${DIR}/trm_baseline_best_step58590_n1000_c8_fdeps003.summary.csv" \
"${DIR}/trm_multi4_best_step35805_n1000_c8_fdeps003.summary.csv" \
--out-dir "${DIR}/plots" \
--slice-sigma 0.03
nvidia-smi --query-gpu=index,memory.used,memory.total,utilization.gpu --format=csv,noheader,nounits \
> "${DIR}/plots/final_gpu_status.txt"
|
