summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorYuren Hao <yurenh2@illinois.edu>2026-07-04 22:35:16 -0500
committerYuren Hao <yurenh2@illinois.edu>2026-07-04 22:35:16 -0500
commitb11d9c6da6ce32471e1c25a6f1b5e7a0a568774d (patch)
treeec296d92ba99d51434c3ec716c21da41e96083d6
parent6e78420da6e613964d93da06156b556e1a91caef (diff)
magic-s2000 study: reg_delay/noadaptc flags, 4-arm queue v2, redx trajectory audit
- lt_ep_train: --reg_delay N (reg-free early phase: resreg/jr/floss/adaptc off for first N steps) + --noadaptc (kill hidden jacreg==0 damping feedback that would pollute single-reg ablation arms) - queue v2: 4 arms delay-first (abl_delay = reg-free 2k -> proven pair) - eig_traj/2/3: ARPACK audit of redx_traj — the run crossed the edge EARLY and oscillated (s1000 rotating-unstable, s1400 excursion mu=+2.1 self-recovered, s2000 the ONLY stable snapshot mu=-0.02, s2100/s2200 already back out) => s2000 is a post-excursion STABILITY-DIP capture, dip width <100 steps; learning survives mild instability (val fell through unstable stretches). lead_rho cold-40 under-reads clusters — NOT a classifier; ARPACK for audits. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014FAPDWQ49M5Ye3NpTndTpn
-rw-r--r--ep_run/eig_traj.log20
-rw-r--r--ep_run/eig_traj.py44
-rw-r--r--ep_run/eig_traj2.log9
-rw-r--r--ep_run/eig_traj2.py39
-rw-r--r--ep_run/eig_traj3.py39
-rw-r--r--ep_run/lt_ep_train.py18
-rwxr-xr-xep_run/runs/abl3_queue.sh33
7 files changed, 184 insertions, 18 deletions
diff --git a/ep_run/eig_traj.log b/ep_run/eig_traj.log
new file mode 100644
index 0000000..37ee8c5
--- /dev/null
+++ b/ep_run/eig_traj.log
@@ -0,0 +1,20 @@
+ ckpt rho@400 Re_mu res@150 res@400 val
+/home/yurenh2/miniconda3/lib/python3.13/site-packages/torch/autograd/graph.py:865: UserWarning: Attempting to run cuBLAS, but there was no current CUDA context! Attempting to set the primary context... (Triggered internally at /pytorch/aten/src/ATen/cuda/CublasHandlePool.cpp:330.)
+ return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
+s600 0.97709 -0.2291 2.75e-01 9.57e-02 4.9611
+s800 0.98169 -0.1831 5.13e-01 1.54e-01 4.2029
+s1000 0.98278 -0.1722 3.45e+00 1.61e+00 3.9009
+s1200 0.97454 -0.2546 6.77e-01 5.27e-02 3.6971
+s1400 1.18812 +1.8812 1.78e+01 1.66e+01 3.6155
+s1600 0.97216 -0.2784 4.56e-01 3.00e-02 3.3450
+s1800 0.97043 -0.2959 2.63e+00 2.72e+00 3.2221
+s2000 0.97087 -0.2913 2.58e+00 4.64e-02 3.1644
+s2200 0.97327 -0.2674 3.80e+00 1.21e+00 3.0150
+s2400 0.96983 -0.3017 2.90e+00 7.92e-02 2.9451
+s2600 0.97321 -0.2680 3.75e+00 1.57e+00 2.8785
+s2800 0.97357 -0.2686 5.54e+00 9.81e-01 2.8715
+s3000 0.97211 -0.2789 3.99e+00 8.35e-02 2.7709
+s3200 0.99111 -0.0889 1.15e+01 8.34e+00 2.7815
+s3400 0.99111 -0.0889 1.15e+01 8.34e+00 2.7815
+s3600 0.99111 -0.0889 1.15e+01 8.34e+00 2.7815
+EIG_TRAJ_DONE
diff --git a/ep_run/eig_traj.py b/ep_run/eig_traj.py
new file mode 100644
index 0000000..4cf14a3
--- /dev/null
+++ b/ep_run/eig_traj.py
@@ -0,0 +1,44 @@
+"""E1 of the magic-s2000 study: trajectory fingerprint over the redx every-100-step checkpoints.
+redx recipe = frozen jr 0.1, NO resreg (predates it) — it rode free, made the golden s2000 (val 3.13),
+and blew at step 3300 (CE 2.74 -> 41). Question: does rho(step) show a monotone approach to the edge,
+with s2000 sitting in a stable-but-critical sweet window before the ~s3200 crossing? That would make
+'edge operator' the mechanism of the magic warm start — and abl_delay the way to manufacture it.
+Per ckpt: rho/Re_mu of the forward map at the DEEP state (400-step relax; z_T1=150 readings are
+state-contaminated per eig_v2_depth), res at 150 (training protocol) and 400, val CE (nb=4).
+"""
+import torch
+from pathlib import Path
+import lt_ep_train as L
+from eig_control import lead_rho
+
+T1, DEEP, EPS, B, C = 150, 400, 0.1, 6, 1.0
+STEPS = list(range(600, 3700, 200))
+
+
+def load(path):
+ torch.manual_seed(0)
+ blk = L.EQBlock(512, 16, 256, 256, c=C, attn_mode='thick'); blk.qknorm = True
+ ck = torch.load(path, map_location=L.dev)
+ with torch.no_grad():
+ for p, w in zip(blk.allp, ck['allp']):
+ p.copy_(w.to(L.dev))
+ return blk
+
+
+print(f"{'ckpt':>6} {'rho@400':>9} {'Re_mu':>8} {'res@150':>9} {'res@400':>9} {'val':>8}", flush=True)
+for s in STEPS:
+ p = Path(f'runs/redx_traj/s{s}.pt')
+ if not p.exists():
+ print(f"s{s:<5} MISSING", flush=True); continue
+ blk = load(p)
+ torch.manual_seed(42) # SAME batch for every ckpt
+ idx, _ = L.get_batch('train', B, 256)
+ xin = blk.embed(idx).detach()
+ z150 = L.relax(blk, xin.clone(), xin, T1, EPS)
+ r150 = (L.relax(blk, z150, xin, 1, EPS) - z150).norm().item()
+ z400 = L.relax(blk, z150, xin, DEEP - T1, EPS)
+ r400 = (L.relax(blk, z400, xin, 1, EPS) - z400).norm().item()
+ _, rho, mu = lead_rho(blk, z400, EPS, C, {}, iters=40)
+ val = L.evaluate(blk, T1, EPS, nb=4)
+ print(f"s{s:<5} {rho:>9.5f} {mu:>+8.4f} {r150:>9.2e} {r400:>9.2e} {val:>8.4f}", flush=True)
+print("EIG_TRAJ_DONE", flush=True)
diff --git a/ep_run/eig_traj2.log b/ep_run/eig_traj2.log
new file mode 100644
index 0000000..60fe028
--- /dev/null
+++ b/ep_run/eig_traj2.log
@@ -0,0 +1,9 @@
+/home/yurenh2/miniconda3/lib/python3.13/site-packages/torch/autograd/graph.py:865: UserWarning: Attempting to run cuBLAS, but there was no current CUDA context! Attempting to set the primary context... (Triggered internally at /pytorch/aten/src/ATen/cuda/CublasHandlePool.cpp:330.)
+ return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
+s1000 |l|=1.03113(mu=+0.1944+1.548j) |l|=1.03113(mu=+0.1944-1.548j) |l|=1.00482(mu=+0.0482+0.000j)
+s1400 |l|=1.20728(mu=+2.0728+0.000j) |l|=1.20201(mu=+2.0201+0.000j) |l|=1.18788(mu=+1.8788+0.000j)
+s2000 |l|=0.99798(mu=-0.0202+0.000j) |l|=0.99788(mu=-0.0212+0.000j) |l|=0.99753(mu=-0.0247+0.000j)
+s2400 |l|=1.00559(mu=+0.0559+0.000j) |l|=1.00018(mu=+0.0018+0.000j) |l|=0.99530(mu=-0.0470+0.000j)
+s2800 |l|=1.01613(mu=+0.1613+0.000j) |l|=1.00169(mu=+0.0169+0.000j) |l|=0.98866(mu=-0.4329+2.493j)
+s3200 |l|=1.03371(mu=+0.1271-2.073j) |l|=1.03371(mu=+0.1271+2.073j) |l|=1.02563(mu=+0.1254-1.633j)
+EIG_TRAJ2_DONE
diff --git a/ep_run/eig_traj2.py b/ep_run/eig_traj2.py
new file mode 100644
index 0000000..bae22b7
--- /dev/null
+++ b/ep_run/eig_traj2.py
@@ -0,0 +1,39 @@
+"""E1b: ARPACK gold-standard re-measurement of the redx trajectory at 6 key ckpts (lead_rho's cold 2-D
+iteration under-reads near-unity clusters by ~0.02-0.03, so absolute mu from eig_traj.py is suspect).
+Top-3 |lam| of the forward map M = I + eps*J_F at the 400-step deep state, same seed-42 batch."""
+import numpy as np, torch, scipy.sparse.linalg as sla
+from torch.autograd.functional import jvp
+import lt_ep_train as L
+
+EPS, B, C = 0.1, 6, 1.0
+KEY = [1000, 1400, 2000, 2400, 2800, 3200]
+
+for s in KEY:
+ torch.manual_seed(0)
+ blk = L.EQBlock(512, 16, 256, 256, c=C, attn_mode='thick'); blk.qknorm = True
+ ck = torch.load(f'runs/redx_traj/s{s}.pt', map_location=L.dev)
+ with torch.no_grad():
+ for p, w in zip(blk.allp, ck['allp']):
+ p.copy_(w.to(L.dev))
+ torch.manual_seed(42)
+ idx, _ = L.get_batch('train', B, 256)
+ xin = blk.embed(idx).detach()
+ z = L.relax(blk, xin.clone(), xin, 400, EPS)
+ sh, n = z.shape, z.numel()
+ kk = 1.0 - EPS * (1.0 + C)
+
+ def mv(x, z=z, sh=sh):
+ v = torch.from_numpy(np.asarray(x, dtype=np.float32)).to(L.dev).view(sh)
+ with torch.no_grad():
+ Mv = kk * v + EPS * jvp(blk.nc_force, z, v)[1]
+ return Mv.reshape(-1).double().cpu().numpy()
+
+ A = sla.LinearOperator((n, n), matvec=mv, dtype=np.float64)
+ try:
+ vals = sorted(sla.eigs(A, k=3, which='LM', return_eigenvectors=False, maxiter=2000, tol=1e-4),
+ key=lambda x: -abs(x))
+ out = " ".join(f"|l|={abs(l):.5f}(mu={(l.real-1)/EPS:+.4f}{l.imag/EPS:+.3f}j)" for l in vals)
+ except Exception as e:
+ out = f"ARPACK-fail {type(e).__name__}"
+ print(f"s{s:<5} {out}", flush=True)
+print("EIG_TRAJ2_DONE", flush=True)
diff --git a/ep_run/eig_traj3.py b/ep_run/eig_traj3.py
new file mode 100644
index 0000000..1d864b6
--- /dev/null
+++ b/ep_run/eig_traj3.py
@@ -0,0 +1,39 @@
+"""E1b: ARPACK gold-standard re-measurement of the redx trajectory at 6 key ckpts (lead_rho's cold 2-D
+iteration under-reads near-unity clusters by ~0.02-0.03, so absolute mu from eig_traj.py is suspect).
+Top-3 |lam| of the forward map M = I + eps*J_F at the 400-step deep state, same seed-42 batch."""
+import numpy as np, torch, scipy.sparse.linalg as sla
+from torch.autograd.functional import jvp
+import lt_ep_train as L
+
+EPS, B, C = 0.1, 6, 1.0
+KEY = [2100, 2200, 2300]
+
+for s in KEY:
+ torch.manual_seed(0)
+ blk = L.EQBlock(512, 16, 256, 256, c=C, attn_mode='thick'); blk.qknorm = True
+ ck = torch.load(f'runs/redx_traj/s{s}.pt', map_location=L.dev)
+ with torch.no_grad():
+ for p, w in zip(blk.allp, ck['allp']):
+ p.copy_(w.to(L.dev))
+ torch.manual_seed(42)
+ idx, _ = L.get_batch('train', B, 256)
+ xin = blk.embed(idx).detach()
+ z = L.relax(blk, xin.clone(), xin, 400, EPS)
+ sh, n = z.shape, z.numel()
+ kk = 1.0 - EPS * (1.0 + C)
+
+ def mv(x, z=z, sh=sh):
+ v = torch.from_numpy(np.asarray(x, dtype=np.float32)).to(L.dev).view(sh)
+ with torch.no_grad():
+ Mv = kk * v + EPS * jvp(blk.nc_force, z, v)[1]
+ return Mv.reshape(-1).double().cpu().numpy()
+
+ A = sla.LinearOperator((n, n), matvec=mv, dtype=np.float64)
+ try:
+ vals = sorted(sla.eigs(A, k=3, which='LM', return_eigenvectors=False, maxiter=2000, tol=1e-4),
+ key=lambda x: -abs(x))
+ out = " ".join(f"|l|={abs(l):.5f}(mu={(l.real-1)/EPS:+.4f}{l.imag/EPS:+.3f}j)" for l in vals)
+ except Exception as e:
+ out = f"ARPACK-fail {type(e).__name__}"
+ print(f"s{s:<5} {out}", flush=True)
+print("EIG_TRAJ2_DONE", flush=True)
diff --git a/ep_run/lt_ep_train.py b/ep_run/lt_ep_train.py
index 99a1811..d56af1d 100644
--- a/ep_run/lt_ep_train.py
+++ b/ep_run/lt_ep_train.py
@@ -407,6 +407,8 @@ def main():
ap.add_argument('--floss_q', type=int, default=10) # unroll horizon (steps past z_T1, with graph, sub-batch)
ap.add_argument('--floss_rho', type=float, default=0.995) # per-step contraction target (one-sided; matches eig_margin)
ap.add_argument('--floss_bsub', type=int, default=4) # sub-batch rows for the graphed unroll (memory)
+ ap.add_argument('--reg_delay', type=int, default=0) # reg-free first N steps (resreg/jacreg/floss/adaptc all off) — magic-s2000 test
+ ap.add_argument('--noadaptc', action='store_true') # kill the hidden jacreg==0 damping feedback (clean single-reg arms)
ap.add_argument('--diag_cos', type=int, default=0) # #1: every N steps, log cos(EP grad, exact BPTT grad) + res
ap.add_argument('--fingerprint', action='store_true') # load --init_ckpt, print (res,cos,abscissa,val) fingerprint, exit
ap.add_argument('--opt', choices=['adamw', 'lion', 'lionlars', 'sgdm', 'sgdsai'], default='adamw')
@@ -558,12 +560,18 @@ def main():
idx, y = get_batch('train', cfg.B, cfg.T)
if cfg.mode == 'ep':
sw = hw_swap() if hw_on else None
- grads, res = ep_step(blk, idx, y, cfg.T1, cfg.T2, cfg.eps, cfg.beta, jr, cfg.holo, cfg.hr,
- cfg.t1max, cfg.res_est, cfg.t2sel, cfg.corr_every, cfg.res_gate, cfg.resreg,
- cfg.eigreg, cfg.eig_margin, cfg.floss, cfg.floss_q, cfg.floss_rho, cfg.floss_bsub)
+ dly = step < cfg.reg_delay # reg-free early phase (magic-ckpt hypothesis: reach the edge unleashed)
+ grads, res = ep_step(blk, idx, y, cfg.T1, cfg.T2, cfg.eps, cfg.beta, 0.0 if dly else jr,
+ cfg.holo, cfg.hr,
+ cfg.t1max, cfg.res_est, cfg.t2sel, cfg.corr_every, cfg.res_gate,
+ 0.0 if dly else cfg.resreg,
+ cfg.eigreg, cfg.eig_margin, 0.0 if dly else cfg.floss,
+ cfg.floss_q, cfg.floss_rho, cfg.floss_bsub)
if sw is not None:
hw_restore(sw)
- if cfg.jacreg > 0: # continuous controller: drive residual -> res_target (smooth)
+ if dly: # delay phase: no controller, no damping feedback either
+ pass
+ elif cfg.jacreg > 0: # continuous controller: drive residual -> res_target (smooth)
flo = cfg.jacreg if cfg.jr_floor is None else cfg.jr_floor
if cfg.jr_lrcouple:
flo *= sched.get_last_lr()[0] / cfg.lr
@@ -573,7 +581,7 @@ def main():
rtgt = math.exp((1 - u) * math.log(cfg.res_target) + u * math.log(cfg.rt_final))
rs = res if rs is None else cfg.res_ema * rs + (1 - cfg.res_ema) * res
jr = min(cfg.jr_max, max(flo, jr * math.exp(0.3 * math.log((rs + 1e-9) / rtgt))))
- else: # damping feedback (no jacreg)
+ elif not cfg.noadaptc: # damping feedback (no jacreg) — DISABLE for clean single-reg ablations
if res > 1e-3:
blk.c = min(cfg.ccap, blk.c * 1.3)
elif res < 2e-4:
diff --git a/ep_run/runs/abl3_queue.sh b/ep_run/runs/abl3_queue.sh
index e7c6bde..6fc47ad 100755
--- a/ep_run/runs/abl3_queue.sh
+++ b/ep_run/runs/abl3_queue.sh
@@ -1,31 +1,38 @@
#!/bin/bash
-# Three-arm from-scratch reg ablation (docs/campaign/FINDINGS.md 2026-07-03 verdict):
-# arm0 abl_floss — floss-only (graded finite-horizon LE penalty; the "one fundamental reg" candidate)
-# arm1 abl_resreg — resreg-only (never cleanly run before; jacreg fully off)
-# arm2 abl_pair — proven pair (resreg 0.2 + FROZEN jr 0.1; the ★2.09 recipe = control arm)
+# FOUR-arm from-scratch reg study (FINDINGS 2026-07-03 verdict + the magic-s2000 hypothesis 2026-07-05):
+# arm0 abl_delay — reg-FREE first 2000 steps then the proven pair (--reg_delay 2000). Tests: the redx
+# run (frozen jr 0.1, NO resreg) reached the stability EDGE by s2000 and that edge
+# operator is the golden warm start; safe recipes over-damp early and plateau. If
+# delay-then-leash reproduces an s2000-like operator from scratch, the magic ckpt
+# becomes manufacturable.
+# arm1 abl_pair — proven pair from step 0 (resreg 0.2 + FROZEN jr 0.1; ★2.09 recipe) = delay's control
+# arm2 abl_floss — floss-only (graded finite-horizon LE penalty), --noadaptc (hidden damping feedback OFF)
+# arm3 abl_resreg — resreg-only, --noadaptc
# All arms share every other flag with the proven ep_resreg_scratch cmd (EP_BELOW210:97-101), same seed.
# Queue: poll GPUs 0/1/3 (GPU2 = japardi2 NV-Embed server, DO NOT TOUCH); a slot is free when
# mem.used < 38 GB AND util < 30% for 3 consecutive 60 s polls; launch the next arm per freed slot.
cd /home/yurenh2/ept/ep_run || exit 1
LOG=runs/abl3_queue.log
-echo "[$(date)] queue runner up (pid $$)" >> "$LOG"
+echo "[$(date)] queue runner v2 up (pid $$, 4 arms, delay-first)" >> "$LOG"
launch () { # $1 = gpu id, $2 = arm index
case $2 in
- 0) CUDA_VISIBLE_DEVICES=$1 nohup python3 lt_ep_train.py --mode ep --attn_mode thick --B 24 --C 512 --H 16 --T 256 --c 1.0 --jacreg 0 --resreg 0 --floss 0.2 --holo 2 --hr 0.02 --t2sel 40 --track --pema 0.999 --t1max 300 --res_est 1e-4 --res_gate 0 --qknorm --resinit 0.1 --warmup 800 --T1 150 --T2 20 --lr 6e-4 --wsd 0.25 --steps 32000 --log 200 --save_every 500 --abort_res 0.3 --data data/tinystories_bpe --ckpt runs/abl_floss.pt --state runs/abl_floss.state > runs/abl_floss.log 2>&1 &
- echo "[$(date)] abl_floss (floss-only) -> GPU$1 pid $!" >> "$LOG" ;;
- 1) CUDA_VISIBLE_DEVICES=$1 nohup python3 lt_ep_train.py --mode ep --attn_mode thick --B 24 --C 512 --H 16 --T 256 --c 1.0 --jacreg 0 --resreg 0.2 --holo 2 --hr 0.02 --t2sel 40 --track --pema 0.999 --t1max 300 --res_est 1e-4 --res_gate 0 --qknorm --resinit 0.1 --warmup 800 --T1 150 --T2 20 --lr 6e-4 --wsd 0.25 --steps 32000 --log 200 --save_every 500 --abort_res 0.3 --data data/tinystories_bpe --ckpt runs/abl_resreg.pt --state runs/abl_resreg.state > runs/abl_resreg.log 2>&1 &
- echo "[$(date)] abl_resreg (resreg-only) -> GPU$1 pid $!" >> "$LOG" ;;
- 2) CUDA_VISIBLE_DEVICES=$1 nohup python3 lt_ep_train.py --mode ep --attn_mode thick --B 24 --C 512 --H 16 --T 256 --c 1.0 --jacreg 0.1 --jr_floor 0.1 --jr_max 0.1 --resreg 0.2 --holo 2 --hr 0.02 --t2sel 40 --track --pema 0.999 --t1max 300 --res_est 1e-4 --res_gate 0 --qknorm --resinit 0.1 --warmup 800 --T1 150 --T2 20 --lr 6e-4 --wsd 0.25 --steps 32000 --log 200 --save_every 500 --abort_res 0.3 --data data/tinystories_bpe --ckpt runs/abl_pair.pt --state runs/abl_pair.state > runs/abl_pair.log 2>&1 &
+ 0) CUDA_VISIBLE_DEVICES=$1 nohup python3 lt_ep_train.py --mode ep --attn_mode thick --B 24 --C 512 --H 16 --T 256 --c 1.0 --jacreg 0.1 --jr_floor 0.1 --jr_max 0.1 --resreg 0.2 --reg_delay 2000 --holo 2 --hr 0.02 --t2sel 40 --track --pema 0.999 --t1max 300 --res_est 1e-4 --res_gate 0 --qknorm --resinit 0.1 --warmup 800 --T1 150 --T2 20 --lr 6e-4 --wsd 0.25 --steps 32000 --log 200 --save_every 500 --abort_res 0.3 --data data/tinystories_bpe --ckpt runs/abl_delay.pt --state runs/abl_delay.state > runs/abl_delay.log 2>&1 &
+ echo "[$(date)] abl_delay (reg-free 2k -> pair) -> GPU$1 pid $!" >> "$LOG" ;;
+ 1) CUDA_VISIBLE_DEVICES=$1 nohup python3 lt_ep_train.py --mode ep --attn_mode thick --B 24 --C 512 --H 16 --T 256 --c 1.0 --jacreg 0.1 --jr_floor 0.1 --jr_max 0.1 --resreg 0.2 --holo 2 --hr 0.02 --t2sel 40 --track --pema 0.999 --t1max 300 --res_est 1e-4 --res_gate 0 --qknorm --resinit 0.1 --warmup 800 --T1 150 --T2 20 --lr 6e-4 --wsd 0.25 --steps 32000 --log 200 --save_every 500 --abort_res 0.3 --data data/tinystories_bpe --ckpt runs/abl_pair.pt --state runs/abl_pair.state > runs/abl_pair.log 2>&1 &
echo "[$(date)] abl_pair (proven pair, control) -> GPU$1 pid $!" >> "$LOG" ;;
+ 2) CUDA_VISIBLE_DEVICES=$1 nohup python3 lt_ep_train.py --mode ep --attn_mode thick --B 24 --C 512 --H 16 --T 256 --c 1.0 --jacreg 0 --resreg 0 --floss 0.2 --noadaptc --holo 2 --hr 0.02 --t2sel 40 --track --pema 0.999 --t1max 300 --res_est 1e-4 --res_gate 0 --qknorm --resinit 0.1 --warmup 800 --T1 150 --T2 20 --lr 6e-4 --wsd 0.25 --steps 32000 --log 200 --save_every 500 --abort_res 0.3 --data data/tinystories_bpe --ckpt runs/abl_floss.pt --state runs/abl_floss.state > runs/abl_floss.log 2>&1 &
+ echo "[$(date)] abl_floss (floss-only, noadaptc) -> GPU$1 pid $!" >> "$LOG" ;;
+ 3) CUDA_VISIBLE_DEVICES=$1 nohup python3 lt_ep_train.py --mode ep --attn_mode thick --B 24 --C 512 --H 16 --T 256 --c 1.0 --jacreg 0 --resreg 0.2 --noadaptc --holo 2 --hr 0.02 --t2sel 40 --track --pema 0.999 --t1max 300 --res_est 1e-4 --res_gate 0 --qknorm --resinit 0.1 --warmup 800 --T1 150 --T2 20 --lr 6e-4 --wsd 0.25 --steps 32000 --log 200 --save_every 500 --abort_res 0.3 --data data/tinystories_bpe --ckpt runs/abl_resreg.pt --state runs/abl_resreg.state > runs/abl_resreg.log 2>&1 &
+ echo "[$(date)] abl_resreg (resreg-only, noadaptc) -> GPU$1 pid $!" >> "$LOG" ;;
esac
}
i=0
declare -A CNT USED
-while [ $i -lt 3 ]; do
+while [ $i -lt 4 ]; do
for g in 0 1 3; do
- [ $i -ge 3 ] && break
+ [ $i -ge 4 ] && break
[ -n "${USED[$g]}" ] && continue
read -r mem util <<< "$(nvidia-smi --query-gpu=memory.used,utilization.gpu --format=csv,noheader,nounits -i "$g" 2>/dev/null | awk -F',' '{gsub(/ /,""); print $1" "$2}')"
if [ -n "$mem" ] && [ "$mem" -lt 38000 ] && [ "$util" -lt 30 ] 2>/dev/null; then
@@ -42,4 +49,4 @@ while [ $i -lt 3 ]; do
done
sleep 60
done
-echo "[$(date)] all three arms launched — queue runner exiting" >> "$LOG"
+echo "[$(date)] all four arms launched — queue runner exiting" >> "$LOG"