1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
|
import time, os, re, torch, pickle, numpy as np, subprocess
from pathlib import Path
from scipy.sparse.linalg import eigs, LinearOperator
import lt_ep_train as L
from lt_ep_train import EQBlock, relax
os.chdir("/home/yurenh2/ept/ep_run")
L.DD=Path('data/tinystories_bpe'); L.vocab=pickle.load(open(L.DD/'meta.pkl','rb'))['vocab_size']
dev='cuda'; B=24; T=256; eps=0.1
CK='runs/ep_resreg_warm.pt'; LOG='runs/ep_resreg_warm.log'; OUT='runs/resreg_warm_probe.log'
torch.manual_seed(0); idx,y=L.get_batch('val',B,T); idx=idx.to(dev) if hasattr(idx,'to') else idx
def alive(): return subprocess.run(["pgrep","-f","ckpt runs/ep_resreg_warm.pt"],capture_output=True).returncode==0
def curinfo():
try:
ls=[l for l in open(LOG) if l.startswith("step")]
m=re.search(r"step\s+(\d+)",ls[-1]); b=re.search(r"best ([\d.]+)",ls[-1]); v=re.search(r"val CE ([\d.]+)",ls[-1])
return (m.group(1), b.group(1), v.group(1))
except Exception: return ("?","?","?")
def probe():
ck=torch.load(CK,map_location=dev)
blk=EQBlock(512,16,256,256,s=1.0,c=1.0,attn_mode='thick'); blk.qknorm=True; blk.fnoise=0.0
with torch.no_grad():
for p,w in zip(blk.allp,ck['allp']): p.copy_(w.to(dev))
xin=blk.embed(idx).detach()
z=relax(blk,xin.clone(),xin,300,eps).detach()
g=(blk.force(z,xin).detach().norm()/z.norm()).item()
F0=blk.force(z,xin).detach(); h=1e-3*z.norm().item(); N=z.numel()
def Jv(v):
vt=torch.from_numpy(v).float().to(dev).view_as(z); nv=vt.norm().item()
if nv<1e-20: return np.zeros_like(v)
return ((blk.force(z+h*(vt/nv),xin).detach()-F0)/h*nv).view(-1).cpu().numpy()
lam=eigs(LinearOperator((N,N),matvec=lambda v:v+eps*Jv(v),dtype=np.float32),k=6,which='LM',return_eigenvectors=False,maxiter=300,tol=1e-3)
mu=(lam-1)/eps; top=mu[np.argmax(mu.real)]
return g, float(top.real), float(abs(top.imag))
t0=time.time()
hdr="=== resreg_warm g/Reμ probe loop (every 20min) — track if g holds (bounded) or grows (toward blow) ==="
print(hdr,flush=True); open(OUT,'a').write(hdr+"\n")
while time.time()-t0 < 24*3600:
if not alive():
line=f"[+{int(time.time()-t0)}s] ep_resreg_warm DEAD/done -> stop"; print(line,flush=True); open(OUT,'a').write(line+"\n"); break
try:
st,bb,vv=curinfo(); g,re_mu,im_mu=probe()
line=f"[+{int((time.time()-t0)/60)}min] step{st} best{bb} valCE{vv} | g_floor={g:.2e} Re_mu={re_mu:+.3f} Im={im_mu:.3f}"
print(line,flush=True); open(OUT,'a').write(line+"\n")
except Exception as e:
print("probe skip:",repr(e),flush=True)
time.sleep(1200)
print("loop end",flush=True)
|
