From 8a9db1232a1bca01900f4113bb68666229384512 Mon Sep 17 00:00:00 2001 From: Yuren Hao Date: Tue, 14 Jul 2026 12:53:22 -0500 Subject: wandb -> team entity eqprop-llm-training (trainers + full-history replay script) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014FAPDWQ49M5Ye3NpTndTpn --- ep_run/casc_bp_train.py | 2 +- ep_run/casc_eq_train.py | 2 +- ep_run/sync_wandb.py | 76 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 2 deletions(-) create mode 100644 ep_run/sync_wandb.py (limited to 'ep_run') diff --git a/ep_run/casc_bp_train.py b/ep_run/casc_bp_train.py index db8ab52..0b155d4 100644 --- a/ep_run/casc_bp_train.py +++ b/ep_run/casc_bp_train.py @@ -186,7 +186,7 @@ wb = None if args.wandb: try: import wandb as _w - wb = _w.init(project=args.wandb, name=args.wandb_run or args.tag, id=args.wandb_run or args.tag, + wb = _w.init(entity='eqprop-llm-training', project=args.wandb, name=args.wandb_run or args.tag, id=args.wandb_run or args.tag, resume='allow', config=vars(args)) except Exception as e: print(f'[wandb] disabled ({e})', flush=True) diff --git a/ep_run/casc_eq_train.py b/ep_run/casc_eq_train.py index faf827e..2e221f6 100644 --- a/ep_run/casc_eq_train.py +++ b/ep_run/casc_eq_train.py @@ -529,7 +529,7 @@ wb = None if args.wandb and RANK == 0: try: import wandb as _w - wb = _w.init(project=args.wandb, name=args.wandb_run or args.tag, id=args.wandb_run or args.tag, + wb = _w.init(entity='eqprop-llm-training', project=args.wandb, name=args.wandb_run or args.tag, id=args.wandb_run or args.tag, resume='allow', config=vars(args)) except Exception as e: print(f'[wandb] disabled ({e})', flush=True) diff --git a/ep_run/sync_wandb.py b/ep_run/sync_wandb.py new file mode 100644 index 0000000..2bfd031 --- /dev/null +++ b/ep_run/sync_wandb.py @@ -0,0 +1,76 @@ +"""Retro + live wandb sync from trainer logs (runs launched before wandb default-on). +History logs replayed once; fw72m.log followed until DONE. Multi-run files (nsize_ladder) +switch tag on '[tag] cascade-EP' headers. Project: ept-cascade.""" +import re, time +from pathlib import Path +import wandb +ENTITY = 'eqprop-llm-training' + +R = Path('/home/yurenh2/ept/ep_run/runs') +EP_RE = re.compile( + r'^step\s+(\d+)/\d+ \| train ([\d.]+) val ([\d.]+) \(best ([\d.]+)\) ' + r'\| beta=([\d.e+-]+) K=(\d+) skips=(\d+)\(d(\d+)/g(\d+)/r(\d+)\)(?: cos=([\d.]+))? ' + r'drift=([\d.]+) gn=([\d.e+-]+) sig=([\d.]+) \| ([\d.]+) it/s') +BP_RE = re.compile(r'^step\s+(\d+)/\d+ \| train ([\d.]+) val ([\d.]+) \(best ([\d.]+)\) \| ([\d.]+) it/s') +HDR = re.compile(r'^\[(\w+)\] cascade-') + +def parse(line): + m = EP_RE.match(line) + if m: + g = m.groups() + d = {'train_ce': float(g[1]), 'val_ce': float(g[2]), 'best': float(g[3]), + 'beta_t': float(g[4]), 'K': int(g[5]), 'skips': int(g[6]), + 'drift': float(g[11]), 'gn': float(g[12]), 'sig': float(g[13]), 'it_s': float(g[14])} + if g[10]: d['gate_cos'] = float(g[10]) + return int(g[0]), d + m = BP_RE.match(line) + if m: + g = m.groups() + return int(g[0]), {'train_ce': float(g[1]), 'val_ce': float(g[2]), 'best': float(g[3]), 'it_s': float(g[4])} + return None + +def sync_file(path, default_tag, follow=False): + tag, run, last = default_tag, None, -1 + def ensure(t): + nonlocal run, last + if run is not None: run.finish() + run = wandb.init(entity=ENTITY, project='ept-cascade', name=t, id=t, resume='allow', reinit=True) + last = -1 + f = open(path, errors='replace') + ensure(tag) + while True: + line = f.readline() + if not line: + if not follow: break + if 'DONE best' in open(path, errors='replace').read()[-2000:]: break + time.sleep(60); continue + h = HDR.match(line) + if h and h.group(1) != tag: + tag = h.group(1); ensure(tag) + p = parse(line) + if p and p[0] > last: + run.log(p[1], step=p[0]); last = p[0] + if run is not None: run.finish() + print(f'[sync] {path} done', flush=True) + +HISTORY = [ + ('stage1b_ep_muon.log', 'stage1b_ep_muon'), ('stage1b_amp.log', 'stage1b_amp'), + ('fw_smoke.log', 'fw_smoke'), ('amp_s1.log', 'amp_s1'), ('amp_s2.log', 'amp_s2'), + ('amp_s3.log', 'amp_s3'), ('nsize_ladder.log', 'nsize_c256'), + ('bsign_s1.log', 'bsign_s1'), ('bsign_s2.log', 'bsign_s2'), ('bsign_s3.log', 'bsign_s3'), + ('f3e3cont.log', 'f3e3cont'), + ('fw72m.log', 'fw72m'), ('fw72m_bp.log', 'fw72m_bp'), ('fw72m_f3e3.log', 'fw72m_f3e3'), + ('fw72m_c.log', 'fw72m_c'), ('fw72m_c2.log', 'fw72m_c2'), + ('w2_ctl.log', 'w2_ctl'), ('w2_blow.log', 'w2_blow'), ('w2_geta07.log', 'w2_geta07'), + ('w2_geta05_k5.log', 'w2_geta05_k5'), ('w2_bcap.log', 'w2_bcap'), ('w2_adapt.log', 'w2_adapt'), + ('arm_ctl.log', 'arm_ctl'), ('arm_mom99_f3e4.log', 'arm_mom99_f3e4'), + ('arm_mom995_f1e4.log', 'arm_mom995_f1e4'), ('arm_cent_f3e3.log', 'arm_cent_f3e3'), + ('arm_rich_f1e3.log', 'arm_rich_f1e3'), ('arm_mom99_f1e3.log', 'arm_mom99_f1e3'), + ('arm_adamw_f1e3.log', 'arm_adamw_f1e3'), +] +for fn, tag in HISTORY: + p = R / fn + if p.exists(): + try: sync_file(p, tag) + except Exception as e: print(f'[sync] {fn} FAILED: {e}', flush=True) +print('[sync] ALL DONE', flush=True) -- cgit v1.2.3