1 files changed, 24 insertions, 0 deletions
diff --git a/ep_run/ep_eps05_track.py b/ep_run/ep_eps05_track.py
new file mode 100644
index 0000000..b888e24
--- /dev/null
+++ b/ep_run/ep_eps05_track.py
@@ -0,0 +1,24 @@
+import time, os, re, subprocess
+os.chdir("/home/yurenh2/ept/ep_run"); LOG="runs/ep_eps05.log"
+def alive(): return subprocess.run(["pgrep","-f","ckpt runs/ep_eps05.pt"],capture_output=True).returncode==0
+def latest():
+    try: ls=[l for l in open(LOG) if l.startswith("step")]
+    except Exception: return None
+    if not ls: return None
+    l=ls[-1]
+    ms=re.search(r"step\s+(\d+)", l); mv=re.search(r"val CE ([\d.eE+-]+)", l)
+    mb=re.search(r"best ([\d.eE+-]+)", l); mr=re.search(r"res=([\d.eE+-]+)", l)
+    if not (ms and mv and mb and mr): return None
+    return (int(ms.group(1)), float(mv.group(1)), float(mb.group(1)), float(mr.group(1)))
+fired=None; best_seen=99.0; best_step=0; t0=time.time()
+while fired is None and time.time()-t0 < 8*3600:
+    time.sleep(120)
+    d=latest()
+    if not alive(): fired=f"ep_eps05 EXITED; last {d}"; break
+    if not d: continue
+    step,val,best,res = d
+    if best < best_seen: best_seen=best; best_step=step
+    if res>0.3 or val>15: fired=f"ep_eps05 DIVERGED @step{step} val{val:.2f} res{res:.1e} -> blew BETWEEN 3.13 and 2.74; smaller eps did NOT fully fix it"; break
+    if best <= 2.74: fired=f"ep_eps05 CROSSED 2.74 (redx blow point): step{step} best{best:.4f} val{val:.4f} res{res:.1e} -> integration fix carried it PAST redx wall"; break
+    if step-best_step >= 1200 and best > 2.74: fired=f"ep_eps05 STALLED: best{best:.4f}, no improve {step-best_step} steps, res{res:.1e} (not blowing) -> eps=0.05 wall ~{best:.2f}?"; break
+print("=== EP_EPS05 TRACK (->2.74) ==="); print(fired or "8h timeout"); print("last:", latest())