From b11d9c6da6ce32471e1c25a6f1b5e7a0a568774d Mon Sep 17 00:00:00 2001
From: Yuren Hao <yurenh2@illinois.edu>
Date: Sat, 4 Jul 2026 22:35:16 -0500
Subject: magic-s2000 study: reg_delay/noadaptc flags, 4-arm queue v2, redx
 trajectory audit
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- lt_ep_train: --reg_delay N (reg-free early phase: resreg/jr/floss/adaptc off
  for first N steps) + --noadaptc (kill hidden jacreg==0 damping feedback that
  would pollute single-reg ablation arms)
- queue v2: 4 arms delay-first (abl_delay = reg-free 2k -> proven pair)
- eig_traj/2/3: ARPACK audit of redx_traj — the run crossed the edge EARLY and
  oscillated (s1000 rotating-unstable, s1400 excursion mu=+2.1 self-recovered,
  s2000 the ONLY stable snapshot mu=-0.02, s2100/s2200 already back out) =>
  s2000 is a post-excursion STABILITY-DIP capture, dip width <100 steps;
  learning survives mild instability (val fell through unstable stretches).
  lead_rho cold-40 under-reads clusters — NOT a classifier; ARPACK for audits.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014FAPDWQ49M5Ye3NpTndTpn
---
 ep_run/runs/abl3_queue.sh | 33 ++++++++++++++++++++-------------
 1 file changed, 20 insertions(+), 13 deletions(-)

(limited to 'ep_run/runs')

diff --git a/ep_run/runs/abl3_queue.sh b/ep_run/runs/abl3_queue.sh
index e7c6bde..6fc47ad 100755
--- a/ep_run/runs/abl3_queue.sh
+++ b/ep_run/runs/abl3_queue.sh
@@ -1,31 +1,38 @@
 #!/bin/bash
-# Three-arm from-scratch reg ablation (docs/campaign/FINDINGS.md 2026-07-03 verdict):
-#   arm0 abl_floss  — floss-only  (graded finite-horizon LE penalty; the "one fundamental reg" candidate)
-#   arm1 abl_resreg — resreg-only (never cleanly run before; jacreg fully off)
-#   arm2 abl_pair   — proven pair (resreg 0.2 + FROZEN jr 0.1; the ★2.09 recipe = control arm)
+# FOUR-arm from-scratch reg study (FINDINGS 2026-07-03 verdict + the magic-s2000 hypothesis 2026-07-05):
+#   arm0 abl_delay  — reg-FREE first 2000 steps then the proven pair (--reg_delay 2000).  Tests: the redx
+#                     run (frozen jr 0.1, NO resreg) reached the stability EDGE by s2000 and that edge
+#                     operator is the golden warm start; safe recipes over-damp early and plateau. If
+#                     delay-then-leash reproduces an s2000-like operator from scratch, the magic ckpt
+#                     becomes manufacturable.
+#   arm1 abl_pair   — proven pair from step 0 (resreg 0.2 + FROZEN jr 0.1; ★2.09 recipe) = delay's control
+#   arm2 abl_floss  — floss-only (graded finite-horizon LE penalty), --noadaptc (hidden damping feedback OFF)
+#   arm3 abl_resreg — resreg-only, --noadaptc
 # All arms share every other flag with the proven ep_resreg_scratch cmd (EP_BELOW210:97-101), same seed.
 # Queue: poll GPUs 0/1/3 (GPU2 = japardi2 NV-Embed server, DO NOT TOUCH); a slot is free when
 # mem.used < 38 GB AND util < 30% for 3 consecutive 60 s polls; launch the next arm per freed slot.
 cd /home/yurenh2/ept/ep_run || exit 1
 LOG=runs/abl3_queue.log
-echo "[$(date)] queue runner up (pid $$)" >> "$LOG"
+echo "[$(date)] queue runner v2 up (pid $$, 4 arms, delay-first)" >> "$LOG"
 
 launch () {  # $1 = gpu id, $2 = arm index
   case $2 in
-  0) CUDA_VISIBLE_DEVICES=$1 nohup python3 lt_ep_train.py --mode ep --attn_mode thick --B 24 --C 512 --H 16 --T 256 --c 1.0 --jacreg 0 --resreg 0 --floss 0.2 --holo 2 --hr 0.02 --t2sel 40 --track --pema 0.999 --t1max 300 --res_est 1e-4 --res_gate 0 --qknorm --resinit 0.1 --warmup 800 --T1 150 --T2 20 --lr 6e-4 --wsd 0.25 --steps 32000 --log 200 --save_every 500 --abort_res 0.3 --data data/tinystories_bpe --ckpt runs/abl_floss.pt --state runs/abl_floss.state > runs/abl_floss.log 2>&1 &
-     echo "[$(date)] abl_floss (floss-only) -> GPU$1 pid $!" >> "$LOG" ;;
-  1) CUDA_VISIBLE_DEVICES=$1 nohup python3 lt_ep_train.py --mode ep --attn_mode thick --B 24 --C 512 --H 16 --T 256 --c 1.0 --jacreg 0 --resreg 0.2 --holo 2 --hr 0.02 --t2sel 40 --track --pema 0.999 --t1max 300 --res_est 1e-4 --res_gate 0 --qknorm --resinit 0.1 --warmup 800 --T1 150 --T2 20 --lr 6e-4 --wsd 0.25 --steps 32000 --log 200 --save_every 500 --abort_res 0.3 --data data/tinystories_bpe --ckpt runs/abl_resreg.pt --state runs/abl_resreg.state > runs/abl_resreg.log 2>&1 &
-     echo "[$(date)] abl_resreg (resreg-only) -> GPU$1 pid $!" >> "$LOG" ;;
-  2) CUDA_VISIBLE_DEVICES=$1 nohup python3 lt_ep_train.py --mode ep --attn_mode thick --B 24 --C 512 --H 16 --T 256 --c 1.0 --jacreg 0.1 --jr_floor 0.1 --jr_max 0.1 --resreg 0.2 --holo 2 --hr 0.02 --t2sel 40 --track --pema 0.999 --t1max 300 --res_est 1e-4 --res_gate 0 --qknorm --resinit 0.1 --warmup 800 --T1 150 --T2 20 --lr 6e-4 --wsd 0.25 --steps 32000 --log 200 --save_every 500 --abort_res 0.3 --data data/tinystories_bpe --ckpt runs/abl_pair.pt --state runs/abl_pair.state > runs/abl_pair.log 2>&1 &
+  0) CUDA_VISIBLE_DEVICES=$1 nohup python3 lt_ep_train.py --mode ep --attn_mode thick --B 24 --C 512 --H 16 --T 256 --c 1.0 --jacreg 0.1 --jr_floor 0.1 --jr_max 0.1 --resreg 0.2 --reg_delay 2000 --holo 2 --hr 0.02 --t2sel 40 --track --pema 0.999 --t1max 300 --res_est 1e-4 --res_gate 0 --qknorm --resinit 0.1 --warmup 800 --T1 150 --T2 20 --lr 6e-4 --wsd 0.25 --steps 32000 --log 200 --save_every 500 --abort_res 0.3 --data data/tinystories_bpe --ckpt runs/abl_delay.pt --state runs/abl_delay.state > runs/abl_delay.log 2>&1 &
+     echo "[$(date)] abl_delay (reg-free 2k -> pair) -> GPU$1 pid $!" >> "$LOG" ;;
+  1) CUDA_VISIBLE_DEVICES=$1 nohup python3 lt_ep_train.py --mode ep --attn_mode thick --B 24 --C 512 --H 16 --T 256 --c 1.0 --jacreg 0.1 --jr_floor 0.1 --jr_max 0.1 --resreg 0.2 --holo 2 --hr 0.02 --t2sel 40 --track --pema 0.999 --t1max 300 --res_est 1e-4 --res_gate 0 --qknorm --resinit 0.1 --warmup 800 --T1 150 --T2 20 --lr 6e-4 --wsd 0.25 --steps 32000 --log 200 --save_every 500 --abort_res 0.3 --data data/tinystories_bpe --ckpt runs/abl_pair.pt --state runs/abl_pair.state > runs/abl_pair.log 2>&1 &
      echo "[$(date)] abl_pair (proven pair, control) -> GPU$1 pid $!" >> "$LOG" ;;
+  2) CUDA_VISIBLE_DEVICES=$1 nohup python3 lt_ep_train.py --mode ep --attn_mode thick --B 24 --C 512 --H 16 --T 256 --c 1.0 --jacreg 0 --resreg 0 --floss 0.2 --noadaptc --holo 2 --hr 0.02 --t2sel 40 --track --pema 0.999 --t1max 300 --res_est 1e-4 --res_gate 0 --qknorm --resinit 0.1 --warmup 800 --T1 150 --T2 20 --lr 6e-4 --wsd 0.25 --steps 32000 --log 200 --save_every 500 --abort_res 0.3 --data data/tinystories_bpe --ckpt runs/abl_floss.pt --state runs/abl_floss.state > runs/abl_floss.log 2>&1 &
+     echo "[$(date)] abl_floss (floss-only, noadaptc) -> GPU$1 pid $!" >> "$LOG" ;;
+  3) CUDA_VISIBLE_DEVICES=$1 nohup python3 lt_ep_train.py --mode ep --attn_mode thick --B 24 --C 512 --H 16 --T 256 --c 1.0 --jacreg 0 --resreg 0.2 --noadaptc --holo 2 --hr 0.02 --t2sel 40 --track --pema 0.999 --t1max 300 --res_est 1e-4 --res_gate 0 --qknorm --resinit 0.1 --warmup 800 --T1 150 --T2 20 --lr 6e-4 --wsd 0.25 --steps 32000 --log 200 --save_every 500 --abort_res 0.3 --data data/tinystories_bpe --ckpt runs/abl_resreg.pt --state runs/abl_resreg.state > runs/abl_resreg.log 2>&1 &
+     echo "[$(date)] abl_resreg (resreg-only, noadaptc) -> GPU$1 pid $!" >> "$LOG" ;;
   esac
 }
 
 i=0
 declare -A CNT USED
-while [ $i -lt 3 ]; do
+while [ $i -lt 4 ]; do
   for g in 0 1 3; do
-    [ $i -ge 3 ] && break
+    [ $i -ge 4 ] && break
     [ -n "${USED[$g]}" ] && continue
     read -r mem util <<< "$(nvidia-smi --query-gpu=memory.used,utilization.gpu --format=csv,noheader,nounits -i "$g" 2>/dev/null | awk -F',' '{gsub(/ /,""); print $1" "$2}')"
     if [ -n "$mem" ] && [ "$mem" -lt 38000 ] && [ "$util" -lt 30 ] 2>/dev/null; then
@@ -42,4 +49,4 @@ while [ $i -lt 3 ]; do
   done
   sleep 60
 done
-echo "[$(date)] all three arms launched — queue runner exiting" >> "$LOG"
+echo "[$(date)] all four arms launched — queue runner exiting" >> "$LOG"
-- 
cgit v1.2.3