summaryrefslogtreecommitdiff
path: root/ep_run/dipfarm_freezer.py
diff options
context:
space:
mode:
authorYuren Hao <yurenh2@illinois.edu>2026-07-06 07:15:30 -0500
committerYuren Hao <yurenh2@illinois.edu>2026-07-06 07:15:30 -0500
commit488c50e1bdbf8f420b2ad5b4021a7f950d121835 (patch)
tree3605b3c92af8fd7c0c18ab5982bee084a858b015 /ep_run/dipfarm_freezer.py
parent8f43de671cf03ee20d5d652b6cd3ba575982a436 (diff)
--holoavg shipped (gate 0.913->0.936 @t2sel160, batch2 +0.051) + farm wave-2 tooling
trend-aware stop + plateau averaging recovers the semi-convergence victims exactly as predicted. Estimator pack now: holofast+sdpa+t2sel80(+holoavg). Also: bp_lm stdinit/beta2/sched flags (anchor archaeology), dipfarm_freezer, staged tol_sweep.sh (gated on hr2 verdict), bp_sweep.sh. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014FAPDWQ49M5Ye3NpTndTpn
Diffstat (limited to 'ep_run/dipfarm_freezer.py')
-rw-r--r--ep_run/dipfarm_freezer.py38
1 files changed, 38 insertions, 0 deletions
diff --git a/ep_run/dipfarm_freezer.py b/ep_run/dipfarm_freezer.py
new file mode 100644
index 0000000..49cdead
--- /dev/null
+++ b/ep_run/dipfarm_freezer.py
@@ -0,0 +1,38 @@
+"""Consolidated trajectory freezer for the 4-seed dip farm (runs anywhere on the shared NFS):
+copies runs/dipfarm_sN.pt -> runs/dipfarm_sN_traj/s<step>.pt whenever a new step line appears.
+Feeds the ARPACK dip-screening (stability-dip statistics: rate, width, screenable yield)."""
+import time, os, re, shutil
+
+os.chdir("/home/yurenh2/ept/ep_run")
+SEEDS = (1, 2, 3, 4)
+seen = {s: set() for s in SEEDS}
+for s in SEEDS:
+ os.makedirs(f"runs/dipfarm_s{s}_traj", exist_ok=True)
+t0 = time.time()
+while time.time() - t0 < 24 * 3600:
+ time.sleep(20)
+ done = 0
+ for s in SEEDS:
+ log, ck = f"runs/dipfarm_s{s}.log", f"runs/dipfarm_s{s}.pt"
+ try:
+ ls = [l for l in open(log) if l.startswith("step")]
+ except Exception:
+ continue
+ if not ls:
+ continue
+ m = re.search(r"step +(\d+)/", ls[-1])
+ if not m:
+ continue
+ step = int(m.group(1))
+ if step not in seen[s] and os.path.exists(ck) and os.path.getsize(ck) > 1e6:
+ try:
+ shutil.copy2(ck, f"runs/dipfarm_s{s}_traj/s{step}.pt")
+ seen[s].add(step)
+ print(f"froze s{s}/{step}", flush=True)
+ except Exception:
+ pass
+ if step >= 4000 or "DONE" in ls[-1]:
+ done += 1
+ if done == len(SEEDS):
+ break
+print("dipfarm freezer done:", {s: len(v) for s, v in seen.items()}, flush=True)