1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
|
import time, os, re, subprocess
os.chdir("/home/yurenh2/ept/ep_run"); LOG="runs/ep_eps05.log"
def alive(): return subprocess.run(["pgrep","-f","ckpt runs/ep_eps05.pt"],capture_output=True).returncode==0
def latest():
try: ls=[l for l in open(LOG) if l.startswith("step")]
except Exception: return None
if not ls: return None
l=ls[-1]
ms=re.search(r"step\s+(\d+)", l); mv=re.search(r"val CE ([\d.eE+-]+)", l)
mb=re.search(r"best ([\d.eE+-]+)", l); mr=re.search(r"res=([\d.eE+-]+)", l)
if not (ms and mv and mb and mr): return None
return (int(ms.group(1)), float(mv.group(1)), float(mb.group(1)), float(mr.group(1)))
fired=None; best_seen=99.0; best_step=0; hi_res=0.0; t0=time.time()
while fired is None and time.time()-t0 < 12*3600:
time.sleep(120)
d=latest()
if not alive(): fired=f"ep_eps05 EXITED; last {d}"; break
if not d: continue
step,val,best,res = d
if best < best_seen: best_seen=best; best_step=step
if res>hi_res: hi_res=res
if val>10 or res>0.3: fired=f"ep_eps05 DIVERGED @step{step} val{val:.2f} res{res:.1e} -> wall MOVED LOWER: eps=0.05 blew ~{best:.2f} (between 2.74 and 2.09)"; break
if best <= 2.09: fired=f"ep_eps05 PASSED 2.09 (early wall)! step{step} best{best:.4f} val{val:.4f} res{res:.1e} -> integration fix beats the early wall"; break
if res>4e-2: fired=f"ep_eps05 res CLIMBING: res{res:.1e} (peak {hi_res:.1e}) @step{step} best{best:.4f} -> early warning, wall may be moving lower (NOT blown yet)"; break
if step-best_step >= 1500 and best>2.09: fired=f"ep_eps05 STALLED: best{best:.4f} no-improve {step-best_step} steps res{res:.1e} -> eps=0.05 floor ~{best:.2f}"; break
print("=== EP_EPS05 TRACK2 (2.74->2.09) ==="); print(fired or "12h timeout"); print("last:", latest(), "peak_res", round(hi_res,4))
|
