From 836e72a707de4f8bd7679400cb40fe59464daca9 Mon Sep 17 00:00:00 2001 From: Yuren Hao Date: Tue, 14 Jul 2026 13:04:38 -0500 Subject: wandb reorganized: per-regime team projects (ept-tinystories-42m / ept-fineweb-72m), auto project routing in trainers, split replay + reports Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014FAPDWQ49M5Ye3NpTndTpn --- ep_run/casc_bp_train.py | 4 +++- ep_run/casc_eq_train.py | 4 +++- ep_run/sync_wandb.py | 36 +++++++++++++++++++----------------- 3 files changed, 25 insertions(+), 19 deletions(-) (limited to 'ep_run') diff --git a/ep_run/casc_bp_train.py b/ep_run/casc_bp_train.py index 0b155d4..bed8d1c 100644 --- a/ep_run/casc_bp_train.py +++ b/ep_run/casc_bp_train.py @@ -13,7 +13,7 @@ ap.add_argument('--B', type=int, default=24); ap.add_argument('--steps', type=in ap.add_argument('--lr', type=float, default=3e-4); ap.add_argument('--warmup', type=int, default=200) ap.add_argument('--seed', type=int, default=0) ap.add_argument('--save_every', type=int, default=500); ap.add_argument('--log', type=int, default=200) -ap.add_argument('--wandb', default='ept-cascade') # ON BY DEFAULT (user directive 07-13); --wandb '' to disable +ap.add_argument('--wandb', default='auto') # ON BY DEFAULT; 'auto' = per-regime project (ept-fineweb-72m / ept-tinystories-42m); --wandb '' to disable ap.add_argument('--wandb_run', default='') ap.add_argument('--amp', action='store_true') # bf16 autocast fwd/loss (no scaler needed for bf16) ap.add_argument('--opt', choices=['adamw', 'muon'], default='adamw') @@ -183,6 +183,8 @@ def evaluate(nb=6): return tot / nb wb = None +if args.wandb == 'auto': + args.wandb = 'ept-fineweb-72m' if 'fineweb' in args.data else 'ept-tinystories-42m' if args.wandb: try: import wandb as _w diff --git a/ep_run/casc_eq_train.py b/ep_run/casc_eq_train.py index 2e221f6..f360df1 100644 --- a/ep_run/casc_eq_train.py +++ b/ep_run/casc_eq_train.py @@ -17,7 +17,7 @@ ap.add_argument('--beta', type=float, default=0.003); ap.add_argument('--seed', ap.add_argument('--K', type=int, default=3) # fb (message-passing) rounds ap.add_argument('--geta', type=float, default=1.0) # fb mixing (1.0 = undamped) ap.add_argument('--save_every', type=int, default=1000); ap.add_argument('--log', type=int, default=100) -ap.add_argument('--wandb', default='ept-cascade') # ON BY DEFAULT (user directive 07-13); --wandb '' to disable +ap.add_argument('--wandb', default='auto') # ON BY DEFAULT; 'auto' = per-regime project (ept-fineweb-72m / ept-tinystories-42m); --wandb '' to disable ap.add_argument('--wandb_run', default='') ap.add_argument('--kmax', type=int, default=8) # adaptive fb rounds cap ap.add_argument('--noguard', action='store_true') # diagnosis: skip only non-finite grads @@ -526,6 +526,8 @@ if DDP: # belt & suspenders on top of identical init seeds: rank0's params are if RANK == 0: print(f'[ddp] world={WORLD} backend={args.ddp_backend} params broadcast; eff batch {args.B}x{WORLD}={args.B*WORLD}', flush=True) wb = None +if args.wandb == 'auto': + args.wandb = 'ept-fineweb-72m' if 'fineweb' in args.data else 'ept-tinystories-42m' if args.wandb and RANK == 0: try: import wandb as _w diff --git a/ep_run/sync_wandb.py b/ep_run/sync_wandb.py index 2bfd031..74fff35 100644 --- a/ep_run/sync_wandb.py +++ b/ep_run/sync_wandb.py @@ -29,12 +29,12 @@ def parse(line): return int(g[0]), {'train_ce': float(g[1]), 'val_ce': float(g[2]), 'best': float(g[3]), 'it_s': float(g[4])} return None -def sync_file(path, default_tag, follow=False): +def sync_file(path, default_tag, project, follow=False): tag, run, last = default_tag, None, -1 def ensure(t): nonlocal run, last if run is not None: run.finish() - run = wandb.init(entity=ENTITY, project='ept-cascade', name=t, id=t, resume='allow', reinit=True) + run = wandb.init(entity=ENTITY, project=project, name=t, id=t, resume='allow', reinit=True) last = -1 f = open(path, errors='replace') ensure(tag) @@ -53,24 +53,26 @@ def sync_file(path, default_tag, follow=False): if run is not None: run.finish() print(f'[sync] {path} done', flush=True) +TS, FW = 'ept-tinystories-42m', 'ept-fineweb-72m' HISTORY = [ - ('stage1b_ep_muon.log', 'stage1b_ep_muon'), ('stage1b_amp.log', 'stage1b_amp'), - ('fw_smoke.log', 'fw_smoke'), ('amp_s1.log', 'amp_s1'), ('amp_s2.log', 'amp_s2'), - ('amp_s3.log', 'amp_s3'), ('nsize_ladder.log', 'nsize_c256'), - ('bsign_s1.log', 'bsign_s1'), ('bsign_s2.log', 'bsign_s2'), ('bsign_s3.log', 'bsign_s3'), - ('f3e3cont.log', 'f3e3cont'), - ('fw72m.log', 'fw72m'), ('fw72m_bp.log', 'fw72m_bp'), ('fw72m_f3e3.log', 'fw72m_f3e3'), - ('fw72m_c.log', 'fw72m_c'), ('fw72m_c2.log', 'fw72m_c2'), - ('w2_ctl.log', 'w2_ctl'), ('w2_blow.log', 'w2_blow'), ('w2_geta07.log', 'w2_geta07'), - ('w2_geta05_k5.log', 'w2_geta05_k5'), ('w2_bcap.log', 'w2_bcap'), ('w2_adapt.log', 'w2_adapt'), - ('arm_ctl.log', 'arm_ctl'), ('arm_mom99_f3e4.log', 'arm_mom99_f3e4'), - ('arm_mom995_f1e4.log', 'arm_mom995_f1e4'), ('arm_cent_f3e3.log', 'arm_cent_f3e3'), - ('arm_rich_f1e3.log', 'arm_rich_f1e3'), ('arm_mom99_f1e3.log', 'arm_mom99_f1e3'), - ('arm_adamw_f1e3.log', 'arm_adamw_f1e3'), + ('stage1b_ep_muon.log', 'stage1b_ep_muon', TS), ('stage1b_amp.log', 'stage1b_amp', TS), + ('amp_s1.log', 'amp_s1', TS), ('amp_s2.log', 'amp_s2', TS), ('amp_s3.log', 'amp_s3', TS), + ('nsize_ladder.log', 'nsize_c256', TS), + ('bsign_s1.log', 'bsign_s1', TS), ('bsign_s2.log', 'bsign_s2', TS), ('bsign_s3.log', 'bsign_s3', TS), + ('f3e3cont.log', 'f3e3cont', TS), + ('arm_ctl.log', 'arm_ctl', TS), ('arm_mom99_f3e4.log', 'arm_mom99_f3e4', TS), + ('arm_mom995_f1e4.log', 'arm_mom995_f1e4', TS), ('arm_cent_f3e3.log', 'arm_cent_f3e3', TS), + ('arm_rich_f1e3.log', 'arm_rich_f1e3', TS), ('arm_mom99_f1e3.log', 'arm_mom99_f1e3', TS), + ('arm_adamw_f1e3.log', 'arm_adamw_f1e3', TS), + ('fw_smoke.log', 'fw_smoke', FW), + ('fw72m.log', 'fw72m', FW), ('fw72m_bp.log', 'fw72m_bp', FW), ('fw72m_f3e3.log', 'fw72m_f3e3', FW), + ('fw72m_c.log', 'fw72m_c', FW), ('fw72m_c2.log', 'fw72m_c2', FW), + ('w2_ctl.log', 'w2_ctl', FW), ('w2_blow.log', 'w2_blow', FW), ('w2_geta07.log', 'w2_geta07', FW), + ('w2_geta05_k5.log', 'w2_geta05_k5', FW), ('w2_bcap.log', 'w2_bcap', FW), ('w2_adapt.log', 'w2_adapt', FW), ] -for fn, tag in HISTORY: +for fn, tag, prj in HISTORY: p = R / fn if p.exists(): - try: sync_file(p, tag) + try: sync_file(p, tag, prj) except Exception as e: print(f'[sync] {fn} FAILED: {e}', flush=True) print('[sync] ALL DONE', flush=True) -- cgit v1.2.3