diff options
| author | Yuren Hao <yurenh2@illinois.edu> | 2026-07-04 22:35:16 -0500 |
|---|---|---|
| committer | Yuren Hao <yurenh2@illinois.edu> | 2026-07-04 22:35:16 -0500 |
| commit | b11d9c6da6ce32471e1c25a6f1b5e7a0a568774d (patch) | |
| tree | ec296d92ba99d51434c3ec716c21da41e96083d6 | |
| parent | 6e78420da6e613964d93da06156b556e1a91caef (diff) | |
magic-s2000 study: reg_delay/noadaptc flags, 4-arm queue v2, redx trajectory audit
- lt_ep_train: --reg_delay N (reg-free early phase: resreg/jr/floss/adaptc off
for first N steps) + --noadaptc (kill hidden jacreg==0 damping feedback that
would pollute single-reg ablation arms)
- queue v2: 4 arms delay-first (abl_delay = reg-free 2k -> proven pair)
- eig_traj/2/3: ARPACK audit of redx_traj — the run crossed the edge EARLY and
oscillated (s1000 rotating-unstable, s1400 excursion mu=+2.1 self-recovered,
s2000 the ONLY stable snapshot mu=-0.02, s2100/s2200 already back out) =>
s2000 is a post-excursion STABILITY-DIP capture, dip width <100 steps;
learning survives mild instability (val fell through unstable stretches).
lead_rho cold-40 under-reads clusters — NOT a classifier; ARPACK for audits.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014FAPDWQ49M5Ye3NpTndTpn
| -rw-r--r-- | ep_run/eig_traj.log | 20 | ||||
| -rw-r--r-- | ep_run/eig_traj.py | 44 | ||||
| -rw-r--r-- | ep_run/eig_traj2.log | 9 | ||||
| -rw-r--r-- | ep_run/eig_traj2.py | 39 | ||||
| -rw-r--r-- | ep_run/eig_traj3.py | 39 | ||||
| -rw-r--r-- | ep_run/lt_ep_train.py | 18 | ||||
| -rwxr-xr-x | ep_run/runs/abl3_queue.sh | 33 |
7 files changed, 184 insertions, 18 deletions
diff --git a/ep_run/eig_traj.log b/ep_run/eig_traj.log new file mode 100644 index 0000000..37ee8c5 --- /dev/null +++ b/ep_run/eig_traj.log @@ -0,0 +1,20 @@ + ckpt rho@400 Re_mu res@150 res@400 val +/home/yurenh2/miniconda3/lib/python3.13/site-packages/torch/autograd/graph.py:865: UserWarning: Attempting to run cuBLAS, but there was no current CUDA context! Attempting to set the primary context... (Triggered internally at /pytorch/aten/src/ATen/cuda/CublasHandlePool.cpp:330.) + return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass +s600 0.97709 -0.2291 2.75e-01 9.57e-02 4.9611 +s800 0.98169 -0.1831 5.13e-01 1.54e-01 4.2029 +s1000 0.98278 -0.1722 3.45e+00 1.61e+00 3.9009 +s1200 0.97454 -0.2546 6.77e-01 5.27e-02 3.6971 +s1400 1.18812 +1.8812 1.78e+01 1.66e+01 3.6155 +s1600 0.97216 -0.2784 4.56e-01 3.00e-02 3.3450 +s1800 0.97043 -0.2959 2.63e+00 2.72e+00 3.2221 +s2000 0.97087 -0.2913 2.58e+00 4.64e-02 3.1644 +s2200 0.97327 -0.2674 3.80e+00 1.21e+00 3.0150 +s2400 0.96983 -0.3017 2.90e+00 7.92e-02 2.9451 +s2600 0.97321 -0.2680 3.75e+00 1.57e+00 2.8785 +s2800 0.97357 -0.2686 5.54e+00 9.81e-01 2.8715 +s3000 0.97211 -0.2789 3.99e+00 8.35e-02 2.7709 +s3200 0.99111 -0.0889 1.15e+01 8.34e+00 2.7815 +s3400 0.99111 -0.0889 1.15e+01 8.34e+00 2.7815 +s3600 0.99111 -0.0889 1.15e+01 8.34e+00 2.7815 +EIG_TRAJ_DONE diff --git a/ep_run/eig_traj.py b/ep_run/eig_traj.py new file mode 100644 index 0000000..4cf14a3 --- /dev/null +++ b/ep_run/eig_traj.py @@ -0,0 +1,44 @@ +"""E1 of the magic-s2000 study: trajectory fingerprint over the redx every-100-step checkpoints. +redx recipe = frozen jr 0.1, NO resreg (predates it) — it rode free, made the golden s2000 (val 3.13), +and blew at step 3300 (CE 2.74 -> 41). Question: does rho(step) show a monotone approach to the edge, +with s2000 sitting in a stable-but-critical sweet window before the ~s3200 crossing? That would make +'edge operator' the mechanism of the magic warm start — and abl_delay the way to manufacture it. +Per ckpt: rho/Re_mu of the forward map at the DEEP state (400-step relax; z_T1=150 readings are +state-contaminated per eig_v2_depth), res at 150 (training protocol) and 400, val CE (nb=4). +""" +import torch +from pathlib import Path +import lt_ep_train as L +from eig_control import lead_rho + +T1, DEEP, EPS, B, C = 150, 400, 0.1, 6, 1.0 +STEPS = list(range(600, 3700, 200)) + + +def load(path): + torch.manual_seed(0) + blk = L.EQBlock(512, 16, 256, 256, c=C, attn_mode='thick'); blk.qknorm = True + ck = torch.load(path, map_location=L.dev) + with torch.no_grad(): + for p, w in zip(blk.allp, ck['allp']): + p.copy_(w.to(L.dev)) + return blk + + +print(f"{'ckpt':>6} {'rho@400':>9} {'Re_mu':>8} {'res@150':>9} {'res@400':>9} {'val':>8}", flush=True) +for s in STEPS: + p = Path(f'runs/redx_traj/s{s}.pt') + if not p.exists(): + print(f"s{s:<5} MISSING", flush=True); continue + blk = load(p) + torch.manual_seed(42) # SAME batch for every ckpt + idx, _ = L.get_batch('train', B, 256) + xin = blk.embed(idx).detach() + z150 = L.relax(blk, xin.clone(), xin, T1, EPS) + r150 = (L.relax(blk, z150, xin, 1, EPS) - z150).norm().item() + z400 = L.relax(blk, z150, xin, DEEP - T1, EPS) + r400 = (L.relax(blk, z400, xin, 1, EPS) - z400).norm().item() + _, rho, mu = lead_rho(blk, z400, EPS, C, {}, iters=40) + val = L.evaluate(blk, T1, EPS, nb=4) + print(f"s{s:<5} {rho:>9.5f} {mu:>+8.4f} {r150:>9.2e} {r400:>9.2e} {val:>8.4f}", flush=True) +print("EIG_TRAJ_DONE", flush=True) diff --git a/ep_run/eig_traj2.log b/ep_run/eig_traj2.log new file mode 100644 index 0000000..60fe028 --- /dev/null +++ b/ep_run/eig_traj2.log @@ -0,0 +1,9 @@ +/home/yurenh2/miniconda3/lib/python3.13/site-packages/torch/autograd/graph.py:865: UserWarning: Attempting to run cuBLAS, but there was no current CUDA context! Attempting to set the primary context... (Triggered internally at /pytorch/aten/src/ATen/cuda/CublasHandlePool.cpp:330.) + return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass +s1000 |l|=1.03113(mu=+0.1944+1.548j) |l|=1.03113(mu=+0.1944-1.548j) |l|=1.00482(mu=+0.0482+0.000j) +s1400 |l|=1.20728(mu=+2.0728+0.000j) |l|=1.20201(mu=+2.0201+0.000j) |l|=1.18788(mu=+1.8788+0.000j) +s2000 |l|=0.99798(mu=-0.0202+0.000j) |l|=0.99788(mu=-0.0212+0.000j) |l|=0.99753(mu=-0.0247+0.000j) +s2400 |l|=1.00559(mu=+0.0559+0.000j) |l|=1.00018(mu=+0.0018+0.000j) |l|=0.99530(mu=-0.0470+0.000j) +s2800 |l|=1.01613(mu=+0.1613+0.000j) |l|=1.00169(mu=+0.0169+0.000j) |l|=0.98866(mu=-0.4329+2.493j) +s3200 |l|=1.03371(mu=+0.1271-2.073j) |l|=1.03371(mu=+0.1271+2.073j) |l|=1.02563(mu=+0.1254-1.633j) +EIG_TRAJ2_DONE diff --git a/ep_run/eig_traj2.py b/ep_run/eig_traj2.py new file mode 100644 index 0000000..bae22b7 --- /dev/null +++ b/ep_run/eig_traj2.py @@ -0,0 +1,39 @@ +"""E1b: ARPACK gold-standard re-measurement of the redx trajectory at 6 key ckpts (lead_rho's cold 2-D +iteration under-reads near-unity clusters by ~0.02-0.03, so absolute mu from eig_traj.py is suspect). +Top-3 |lam| of the forward map M = I + eps*J_F at the 400-step deep state, same seed-42 batch.""" +import numpy as np, torch, scipy.sparse.linalg as sla +from torch.autograd.functional import jvp +import lt_ep_train as L + +EPS, B, C = 0.1, 6, 1.0 +KEY = [1000, 1400, 2000, 2400, 2800, 3200] + +for s in KEY: + torch.manual_seed(0) + blk = L.EQBlock(512, 16, 256, 256, c=C, attn_mode='thick'); blk.qknorm = True + ck = torch.load(f'runs/redx_traj/s{s}.pt', map_location=L.dev) + with torch.no_grad(): + for p, w in zip(blk.allp, ck['allp']): + p.copy_(w.to(L.dev)) + torch.manual_seed(42) + idx, _ = L.get_batch('train', B, 256) + xin = blk.embed(idx).detach() + z = L.relax(blk, xin.clone(), xin, 400, EPS) + sh, n = z.shape, z.numel() + kk = 1.0 - EPS * (1.0 + C) + + def mv(x, z=z, sh=sh): + v = torch.from_numpy(np.asarray(x, dtype=np.float32)).to(L.dev).view(sh) + with torch.no_grad(): + Mv = kk * v + EPS * jvp(blk.nc_force, z, v)[1] + return Mv.reshape(-1).double().cpu().numpy() + + A = sla.LinearOperator((n, n), matvec=mv, dtype=np.float64) + try: + vals = sorted(sla.eigs(A, k=3, which='LM', return_eigenvectors=False, maxiter=2000, tol=1e-4), + key=lambda x: -abs(x)) + out = " ".join(f"|l|={abs(l):.5f}(mu={(l.real-1)/EPS:+.4f}{l.imag/EPS:+.3f}j)" for l in vals) + except Exception as e: + out = f"ARPACK-fail {type(e).__name__}" + print(f"s{s:<5} {out}", flush=True) +print("EIG_TRAJ2_DONE", flush=True) diff --git a/ep_run/eig_traj3.py b/ep_run/eig_traj3.py new file mode 100644 index 0000000..1d864b6 --- /dev/null +++ b/ep_run/eig_traj3.py @@ -0,0 +1,39 @@ +"""E1b: ARPACK gold-standard re-measurement of the redx trajectory at 6 key ckpts (lead_rho's cold 2-D +iteration under-reads near-unity clusters by ~0.02-0.03, so absolute mu from eig_traj.py is suspect). +Top-3 |lam| of the forward map M = I + eps*J_F at the 400-step deep state, same seed-42 batch.""" +import numpy as np, torch, scipy.sparse.linalg as sla +from torch.autograd.functional import jvp +import lt_ep_train as L + +EPS, B, C = 0.1, 6, 1.0 +KEY = [2100, 2200, 2300] + +for s in KEY: + torch.manual_seed(0) + blk = L.EQBlock(512, 16, 256, 256, c=C, attn_mode='thick'); blk.qknorm = True + ck = torch.load(f'runs/redx_traj/s{s}.pt', map_location=L.dev) + with torch.no_grad(): + for p, w in zip(blk.allp, ck['allp']): + p.copy_(w.to(L.dev)) + torch.manual_seed(42) + idx, _ = L.get_batch('train', B, 256) + xin = blk.embed(idx).detach() + z = L.relax(blk, xin.clone(), xin, 400, EPS) + sh, n = z.shape, z.numel() + kk = 1.0 - EPS * (1.0 + C) + + def mv(x, z=z, sh=sh): + v = torch.from_numpy(np.asarray(x, dtype=np.float32)).to(L.dev).view(sh) + with torch.no_grad(): + Mv = kk * v + EPS * jvp(blk.nc_force, z, v)[1] + return Mv.reshape(-1).double().cpu().numpy() + + A = sla.LinearOperator((n, n), matvec=mv, dtype=np.float64) + try: + vals = sorted(sla.eigs(A, k=3, which='LM', return_eigenvectors=False, maxiter=2000, tol=1e-4), + key=lambda x: -abs(x)) + out = " ".join(f"|l|={abs(l):.5f}(mu={(l.real-1)/EPS:+.4f}{l.imag/EPS:+.3f}j)" for l in vals) + except Exception as e: + out = f"ARPACK-fail {type(e).__name__}" + print(f"s{s:<5} {out}", flush=True) +print("EIG_TRAJ2_DONE", flush=True) diff --git a/ep_run/lt_ep_train.py b/ep_run/lt_ep_train.py index 99a1811..d56af1d 100644 --- a/ep_run/lt_ep_train.py +++ b/ep_run/lt_ep_train.py @@ -407,6 +407,8 @@ def main(): ap.add_argument('--floss_q', type=int, default=10) # unroll horizon (steps past z_T1, with graph, sub-batch) ap.add_argument('--floss_rho', type=float, default=0.995) # per-step contraction target (one-sided; matches eig_margin) ap.add_argument('--floss_bsub', type=int, default=4) # sub-batch rows for the graphed unroll (memory) + ap.add_argument('--reg_delay', type=int, default=0) # reg-free first N steps (resreg/jacreg/floss/adaptc all off) — magic-s2000 test + ap.add_argument('--noadaptc', action='store_true') # kill the hidden jacreg==0 damping feedback (clean single-reg arms) ap.add_argument('--diag_cos', type=int, default=0) # #1: every N steps, log cos(EP grad, exact BPTT grad) + res ap.add_argument('--fingerprint', action='store_true') # load --init_ckpt, print (res,cos,abscissa,val) fingerprint, exit ap.add_argument('--opt', choices=['adamw', 'lion', 'lionlars', 'sgdm', 'sgdsai'], default='adamw') @@ -558,12 +560,18 @@ def main(): idx, y = get_batch('train', cfg.B, cfg.T) if cfg.mode == 'ep': sw = hw_swap() if hw_on else None - grads, res = ep_step(blk, idx, y, cfg.T1, cfg.T2, cfg.eps, cfg.beta, jr, cfg.holo, cfg.hr, - cfg.t1max, cfg.res_est, cfg.t2sel, cfg.corr_every, cfg.res_gate, cfg.resreg, - cfg.eigreg, cfg.eig_margin, cfg.floss, cfg.floss_q, cfg.floss_rho, cfg.floss_bsub) + dly = step < cfg.reg_delay # reg-free early phase (magic-ckpt hypothesis: reach the edge unleashed) + grads, res = ep_step(blk, idx, y, cfg.T1, cfg.T2, cfg.eps, cfg.beta, 0.0 if dly else jr, + cfg.holo, cfg.hr, + cfg.t1max, cfg.res_est, cfg.t2sel, cfg.corr_every, cfg.res_gate, + 0.0 if dly else cfg.resreg, + cfg.eigreg, cfg.eig_margin, 0.0 if dly else cfg.floss, + cfg.floss_q, cfg.floss_rho, cfg.floss_bsub) if sw is not None: hw_restore(sw) - if cfg.jacreg > 0: # continuous controller: drive residual -> res_target (smooth) + if dly: # delay phase: no controller, no damping feedback either + pass + elif cfg.jacreg > 0: # continuous controller: drive residual -> res_target (smooth) flo = cfg.jacreg if cfg.jr_floor is None else cfg.jr_floor if cfg.jr_lrcouple: flo *= sched.get_last_lr()[0] / cfg.lr @@ -573,7 +581,7 @@ def main(): rtgt = math.exp((1 - u) * math.log(cfg.res_target) + u * math.log(cfg.rt_final)) rs = res if rs is None else cfg.res_ema * rs + (1 - cfg.res_ema) * res jr = min(cfg.jr_max, max(flo, jr * math.exp(0.3 * math.log((rs + 1e-9) / rtgt)))) - else: # damping feedback (no jacreg) + elif not cfg.noadaptc: # damping feedback (no jacreg) — DISABLE for clean single-reg ablations if res > 1e-3: blk.c = min(cfg.ccap, blk.c * 1.3) elif res < 2e-4: diff --git a/ep_run/runs/abl3_queue.sh b/ep_run/runs/abl3_queue.sh index e7c6bde..6fc47ad 100755 --- a/ep_run/runs/abl3_queue.sh +++ b/ep_run/runs/abl3_queue.sh @@ -1,31 +1,38 @@ #!/bin/bash -# Three-arm from-scratch reg ablation (docs/campaign/FINDINGS.md 2026-07-03 verdict): -# arm0 abl_floss — floss-only (graded finite-horizon LE penalty; the "one fundamental reg" candidate) -# arm1 abl_resreg — resreg-only (never cleanly run before; jacreg fully off) -# arm2 abl_pair — proven pair (resreg 0.2 + FROZEN jr 0.1; the ★2.09 recipe = control arm) +# FOUR-arm from-scratch reg study (FINDINGS 2026-07-03 verdict + the magic-s2000 hypothesis 2026-07-05): +# arm0 abl_delay — reg-FREE first 2000 steps then the proven pair (--reg_delay 2000). Tests: the redx +# run (frozen jr 0.1, NO resreg) reached the stability EDGE by s2000 and that edge +# operator is the golden warm start; safe recipes over-damp early and plateau. If +# delay-then-leash reproduces an s2000-like operator from scratch, the magic ckpt +# becomes manufacturable. +# arm1 abl_pair — proven pair from step 0 (resreg 0.2 + FROZEN jr 0.1; ★2.09 recipe) = delay's control +# arm2 abl_floss — floss-only (graded finite-horizon LE penalty), --noadaptc (hidden damping feedback OFF) +# arm3 abl_resreg — resreg-only, --noadaptc # All arms share every other flag with the proven ep_resreg_scratch cmd (EP_BELOW210:97-101), same seed. # Queue: poll GPUs 0/1/3 (GPU2 = japardi2 NV-Embed server, DO NOT TOUCH); a slot is free when # mem.used < 38 GB AND util < 30% for 3 consecutive 60 s polls; launch the next arm per freed slot. cd /home/yurenh2/ept/ep_run || exit 1 LOG=runs/abl3_queue.log -echo "[$(date)] queue runner up (pid $$)" >> "$LOG" +echo "[$(date)] queue runner v2 up (pid $$, 4 arms, delay-first)" >> "$LOG" launch () { # $1 = gpu id, $2 = arm index case $2 in - 0) CUDA_VISIBLE_DEVICES=$1 nohup python3 lt_ep_train.py --mode ep --attn_mode thick --B 24 --C 512 --H 16 --T 256 --c 1.0 --jacreg 0 --resreg 0 --floss 0.2 --holo 2 --hr 0.02 --t2sel 40 --track --pema 0.999 --t1max 300 --res_est 1e-4 --res_gate 0 --qknorm --resinit 0.1 --warmup 800 --T1 150 --T2 20 --lr 6e-4 --wsd 0.25 --steps 32000 --log 200 --save_every 500 --abort_res 0.3 --data data/tinystories_bpe --ckpt runs/abl_floss.pt --state runs/abl_floss.state > runs/abl_floss.log 2>&1 & - echo "[$(date)] abl_floss (floss-only) -> GPU$1 pid $!" >> "$LOG" ;; - 1) CUDA_VISIBLE_DEVICES=$1 nohup python3 lt_ep_train.py --mode ep --attn_mode thick --B 24 --C 512 --H 16 --T 256 --c 1.0 --jacreg 0 --resreg 0.2 --holo 2 --hr 0.02 --t2sel 40 --track --pema 0.999 --t1max 300 --res_est 1e-4 --res_gate 0 --qknorm --resinit 0.1 --warmup 800 --T1 150 --T2 20 --lr 6e-4 --wsd 0.25 --steps 32000 --log 200 --save_every 500 --abort_res 0.3 --data data/tinystories_bpe --ckpt runs/abl_resreg.pt --state runs/abl_resreg.state > runs/abl_resreg.log 2>&1 & - echo "[$(date)] abl_resreg (resreg-only) -> GPU$1 pid $!" >> "$LOG" ;; - 2) CUDA_VISIBLE_DEVICES=$1 nohup python3 lt_ep_train.py --mode ep --attn_mode thick --B 24 --C 512 --H 16 --T 256 --c 1.0 --jacreg 0.1 --jr_floor 0.1 --jr_max 0.1 --resreg 0.2 --holo 2 --hr 0.02 --t2sel 40 --track --pema 0.999 --t1max 300 --res_est 1e-4 --res_gate 0 --qknorm --resinit 0.1 --warmup 800 --T1 150 --T2 20 --lr 6e-4 --wsd 0.25 --steps 32000 --log 200 --save_every 500 --abort_res 0.3 --data data/tinystories_bpe --ckpt runs/abl_pair.pt --state runs/abl_pair.state > runs/abl_pair.log 2>&1 & + 0) CUDA_VISIBLE_DEVICES=$1 nohup python3 lt_ep_train.py --mode ep --attn_mode thick --B 24 --C 512 --H 16 --T 256 --c 1.0 --jacreg 0.1 --jr_floor 0.1 --jr_max 0.1 --resreg 0.2 --reg_delay 2000 --holo 2 --hr 0.02 --t2sel 40 --track --pema 0.999 --t1max 300 --res_est 1e-4 --res_gate 0 --qknorm --resinit 0.1 --warmup 800 --T1 150 --T2 20 --lr 6e-4 --wsd 0.25 --steps 32000 --log 200 --save_every 500 --abort_res 0.3 --data data/tinystories_bpe --ckpt runs/abl_delay.pt --state runs/abl_delay.state > runs/abl_delay.log 2>&1 & + echo "[$(date)] abl_delay (reg-free 2k -> pair) -> GPU$1 pid $!" >> "$LOG" ;; + 1) CUDA_VISIBLE_DEVICES=$1 nohup python3 lt_ep_train.py --mode ep --attn_mode thick --B 24 --C 512 --H 16 --T 256 --c 1.0 --jacreg 0.1 --jr_floor 0.1 --jr_max 0.1 --resreg 0.2 --holo 2 --hr 0.02 --t2sel 40 --track --pema 0.999 --t1max 300 --res_est 1e-4 --res_gate 0 --qknorm --resinit 0.1 --warmup 800 --T1 150 --T2 20 --lr 6e-4 --wsd 0.25 --steps 32000 --log 200 --save_every 500 --abort_res 0.3 --data data/tinystories_bpe --ckpt runs/abl_pair.pt --state runs/abl_pair.state > runs/abl_pair.log 2>&1 & echo "[$(date)] abl_pair (proven pair, control) -> GPU$1 pid $!" >> "$LOG" ;; + 2) CUDA_VISIBLE_DEVICES=$1 nohup python3 lt_ep_train.py --mode ep --attn_mode thick --B 24 --C 512 --H 16 --T 256 --c 1.0 --jacreg 0 --resreg 0 --floss 0.2 --noadaptc --holo 2 --hr 0.02 --t2sel 40 --track --pema 0.999 --t1max 300 --res_est 1e-4 --res_gate 0 --qknorm --resinit 0.1 --warmup 800 --T1 150 --T2 20 --lr 6e-4 --wsd 0.25 --steps 32000 --log 200 --save_every 500 --abort_res 0.3 --data data/tinystories_bpe --ckpt runs/abl_floss.pt --state runs/abl_floss.state > runs/abl_floss.log 2>&1 & + echo "[$(date)] abl_floss (floss-only, noadaptc) -> GPU$1 pid $!" >> "$LOG" ;; + 3) CUDA_VISIBLE_DEVICES=$1 nohup python3 lt_ep_train.py --mode ep --attn_mode thick --B 24 --C 512 --H 16 --T 256 --c 1.0 --jacreg 0 --resreg 0.2 --noadaptc --holo 2 --hr 0.02 --t2sel 40 --track --pema 0.999 --t1max 300 --res_est 1e-4 --res_gate 0 --qknorm --resinit 0.1 --warmup 800 --T1 150 --T2 20 --lr 6e-4 --wsd 0.25 --steps 32000 --log 200 --save_every 500 --abort_res 0.3 --data data/tinystories_bpe --ckpt runs/abl_resreg.pt --state runs/abl_resreg.state > runs/abl_resreg.log 2>&1 & + echo "[$(date)] abl_resreg (resreg-only, noadaptc) -> GPU$1 pid $!" >> "$LOG" ;; esac } i=0 declare -A CNT USED -while [ $i -lt 3 ]; do +while [ $i -lt 4 ]; do for g in 0 1 3; do - [ $i -ge 3 ] && break + [ $i -ge 4 ] && break [ -n "${USED[$g]}" ] && continue read -r mem util <<< "$(nvidia-smi --query-gpu=memory.used,utilization.gpu --format=csv,noheader,nounits -i "$g" 2>/dev/null | awk -F',' '{gsub(/ /,""); print $1" "$2}')" if [ -n "$mem" ] && [ "$mem" -lt 38000 ] && [ "$util" -lt 30 ] 2>/dev/null; then @@ -42,4 +49,4 @@ while [ $i -lt 3 ]; do done sleep 60 done -echo "[$(date)] all three arms launched — queue runner exiting" >> "$LOG" +echo "[$(date)] all four arms launched — queue runner exiting" >> "$LOG" |
