1 files changed, 52 insertions, 0 deletions
diff --git a/ep_run/t2fix_rho_prober.py b/ep_run/t2fix_rho_prober.py
new file mode 100644
index 0000000..ad9d1e8
--- /dev/null
+++ b/ep_run/t2fix_rho_prober.py
@@ -0,0 +1,52 @@
+import time, os, re, glob, math, pickle, subprocess
+from pathlib import Path
+import torch
+import lt_ep_train as L
+from lt_ep_train import EQBlock
+os.chdir("/home/yurenh2/ept/ep_run")
+L.DD=Path('data/tinystories_bpe'); L.vocab=pickle.load(open(L.DD/'meta.pkl','rb'))['vocab_size']
+dev='cuda'; eps=0.1; B=8; T=256; N=800; OUT="runs/t2fix_rho.log"
+torch.manual_seed(1234); idx,y=L.get_batch('val',B,T); idx=idx.to(dev) if hasattr(idx,'to') else idx
+def measure(ckpt):
+    blk=EQBlock(512,16,256,256,s=1.0,c=1.0,attn_mode='thick'); blk.qknorm=True
+    ck=torch.load(ckpt,map_location=dev)
+    with torch.no_grad():
+        for p,w in zip(blk.allp,ck['allp']): p.copy_(w.to(dev))
+        xin=blk.embed(idx).detach(); z=xin.clone(); ress=[]
+        for t in range(N):
+            z2=z+eps*blk.force(z,xin).detach()
+            r=(z2-z).norm().item()/(z.norm().item()+1e-9); ress.append(r); z=z2
+            if not math.isfinite(r) or r>1e2: break
+    win=[ress[i] for i in range(len(ress)) if 1e-6<ress[i]<1e-1] or ress[-200:]
+    rats=[win[i+1]/win[i] for i in range(len(win)-1) if win[i]>0]
+    return (math.exp(sum(math.log(x) for x in rats)/len(rats)) if rats else float('nan')), ress[-1]
+def alive(): return subprocess.run(["pgrep","-f","ckpt runs/ep_t2fix.pt"],capture_output=True).returncode==0
+def valof(step):
+    try:
+        for l in open("runs/ep_t2fix.log"):
+            if l.startswith("step"):
+                m=re.search(r"step\s+(\d+)/.*val CE ([\d.]+)", l)
+                if m and int(m.group(1))==step: return m.group(2)
+    except Exception: pass
+    return "?"
+open(OUT,"a").write("# ep_t2fix(t2sel=160) rho-tracking | refs: BPTT=0.982 ; redx val3.4->0.988, val2.74->0.998(blew)\n")
+seen=set(); fired=None; t0=time.time()
+while fired is None and time.time()-t0<24*3600:
+    time.sleep(60)
+    for ck in sorted(glob.glob("runs/t2fix_traj/s*.pt"), key=lambda p:int(re.search(r's(\d+)',p).group(1))):
+        step=int(re.search(r's(\d+)',ck).group(1))
+        if step in seen: continue
+        seen.add(step)
+        try: rho,fr=measure(ck)
+        except Exception: rho,fr=float('nan'),float('nan')
+        v=valof(step); line=f"step {step} val {v}: rho={rho:.4f} final_res={fr:.2e}"
+        open(OUT,"a").write(line+"\n"); print(line,flush=True)
+        try:
+            vf=float(v)
+            if vf<3.3 and isinstance(rho,float) and not math.isnan(rho):
+                if rho<0.992: fired=f"ep_t2fix val {v}: rho={rho:.4f} STAYS LOW (vs redx ~0.998 here) -> better gradient holds contraction (FIX WORKING)"; break
+                if rho>0.997: fired=f"ep_t2fix val {v}: rho={rho:.4f} DRIFTED to threshold like redx -> gradient does NOT defend rho (your objection holds)"; break
+        except Exception: pass
+    if not alive(): fired=f"ep_t2fix exited; trajectory in {OUT}"; break
+print("=== EP_T2FIX RHO VERDICT ==="); print(fired or "24h timeout")
+for l in open(OUT): print(l.rstrip())