import time, os, re, subprocess os.chdir("/home/yurenh2/ept/ep_run"); LOG="runs/ep_rr_scratch.log"; TARGET=2.70 def alive(): return subprocess.run(["pgrep","-f","ckpt runs/ep_rr_scratch.pt"],capture_output=True).returncode==0 def latest(): try: ls=[l for l in open(LOG) if l.startswith("step")] except Exception: return None if not ls: return None l=ls[-1] ms=re.search(r"step\s+(\d+)",l); mv=re.search(r"val CE ([\d.eE+-]+)",l); mb=re.search(r"best ([\d.eE+-]+)",l) mr=re.search(r"res=([\d.eE+-]+)",l); mi=re.search(r"([\d.]+) it/s",l) if not (ms and mv and mb and mr): return None return (int(ms.group(1)),float(mv.group(1)),float(mb.group(1)),float(mr.group(1)), mi.group(1) if mi else "?") fired=None; t0=time.time() while fired is None and time.time()-t0<23*3600: time.sleep(120) d=latest() if not alive(): fired=f"ep_rr_scratch EXITED; last {d}"; break if not d: continue step,val,best,res,its=d if (val>12 or res>0.3) and step>400: fired=f"ep_rr_scratch BLEW @step{step} val{val:.2f} res{res:.1e} best{best:.4f} ({its} it/s)"; break if best<=TARGET: fired=f"ep_rr_scratch reached {TARGET}(from-scratch 干净降入墙区): step{step} best{best:.4f} val{val:.4f} res{res:.1e} {its}it/s"; break print(f"=== EP_RR_SCRATCH -> {TARGET} ==="); print(fired or "23h timeout"); print("last:",latest())