diff options
Diffstat (limited to 'ep_run/sync_wandb.py')
| -rw-r--r-- | ep_run/sync_wandb.py | 36 |
1 files changed, 19 insertions, 17 deletions
diff --git a/ep_run/sync_wandb.py b/ep_run/sync_wandb.py index 2bfd031..74fff35 100644 --- a/ep_run/sync_wandb.py +++ b/ep_run/sync_wandb.py @@ -29,12 +29,12 @@ def parse(line): return int(g[0]), {'train_ce': float(g[1]), 'val_ce': float(g[2]), 'best': float(g[3]), 'it_s': float(g[4])} return None -def sync_file(path, default_tag, follow=False): +def sync_file(path, default_tag, project, follow=False): tag, run, last = default_tag, None, -1 def ensure(t): nonlocal run, last if run is not None: run.finish() - run = wandb.init(entity=ENTITY, project='ept-cascade', name=t, id=t, resume='allow', reinit=True) + run = wandb.init(entity=ENTITY, project=project, name=t, id=t, resume='allow', reinit=True) last = -1 f = open(path, errors='replace') ensure(tag) @@ -53,24 +53,26 @@ def sync_file(path, default_tag, follow=False): if run is not None: run.finish() print(f'[sync] {path} done', flush=True) +TS, FW = 'ept-tinystories-42m', 'ept-fineweb-72m' HISTORY = [ - ('stage1b_ep_muon.log', 'stage1b_ep_muon'), ('stage1b_amp.log', 'stage1b_amp'), - ('fw_smoke.log', 'fw_smoke'), ('amp_s1.log', 'amp_s1'), ('amp_s2.log', 'amp_s2'), - ('amp_s3.log', 'amp_s3'), ('nsize_ladder.log', 'nsize_c256'), - ('bsign_s1.log', 'bsign_s1'), ('bsign_s2.log', 'bsign_s2'), ('bsign_s3.log', 'bsign_s3'), - ('f3e3cont.log', 'f3e3cont'), - ('fw72m.log', 'fw72m'), ('fw72m_bp.log', 'fw72m_bp'), ('fw72m_f3e3.log', 'fw72m_f3e3'), - ('fw72m_c.log', 'fw72m_c'), ('fw72m_c2.log', 'fw72m_c2'), - ('w2_ctl.log', 'w2_ctl'), ('w2_blow.log', 'w2_blow'), ('w2_geta07.log', 'w2_geta07'), - ('w2_geta05_k5.log', 'w2_geta05_k5'), ('w2_bcap.log', 'w2_bcap'), ('w2_adapt.log', 'w2_adapt'), - ('arm_ctl.log', 'arm_ctl'), ('arm_mom99_f3e4.log', 'arm_mom99_f3e4'), - ('arm_mom995_f1e4.log', 'arm_mom995_f1e4'), ('arm_cent_f3e3.log', 'arm_cent_f3e3'), - ('arm_rich_f1e3.log', 'arm_rich_f1e3'), ('arm_mom99_f1e3.log', 'arm_mom99_f1e3'), - ('arm_adamw_f1e3.log', 'arm_adamw_f1e3'), + ('stage1b_ep_muon.log', 'stage1b_ep_muon', TS), ('stage1b_amp.log', 'stage1b_amp', TS), + ('amp_s1.log', 'amp_s1', TS), ('amp_s2.log', 'amp_s2', TS), ('amp_s3.log', 'amp_s3', TS), + ('nsize_ladder.log', 'nsize_c256', TS), + ('bsign_s1.log', 'bsign_s1', TS), ('bsign_s2.log', 'bsign_s2', TS), ('bsign_s3.log', 'bsign_s3', TS), + ('f3e3cont.log', 'f3e3cont', TS), + ('arm_ctl.log', 'arm_ctl', TS), ('arm_mom99_f3e4.log', 'arm_mom99_f3e4', TS), + ('arm_mom995_f1e4.log', 'arm_mom995_f1e4', TS), ('arm_cent_f3e3.log', 'arm_cent_f3e3', TS), + ('arm_rich_f1e3.log', 'arm_rich_f1e3', TS), ('arm_mom99_f1e3.log', 'arm_mom99_f1e3', TS), + ('arm_adamw_f1e3.log', 'arm_adamw_f1e3', TS), + ('fw_smoke.log', 'fw_smoke', FW), + ('fw72m.log', 'fw72m', FW), ('fw72m_bp.log', 'fw72m_bp', FW), ('fw72m_f3e3.log', 'fw72m_f3e3', FW), + ('fw72m_c.log', 'fw72m_c', FW), ('fw72m_c2.log', 'fw72m_c2', FW), + ('w2_ctl.log', 'w2_ctl', FW), ('w2_blow.log', 'w2_blow', FW), ('w2_geta07.log', 'w2_geta07', FW), + ('w2_geta05_k5.log', 'w2_geta05_k5', FW), ('w2_bcap.log', 'w2_bcap', FW), ('w2_adapt.log', 'w2_adapt', FW), ] -for fn, tag in HISTORY: +for fn, tag, prj in HISTORY: p = R / fn if p.exists(): - try: sync_file(p, tag) + try: sync_file(p, tag, prj) except Exception as e: print(f'[sync] {fn} FAILED: {e}', flush=True) print('[sync] ALL DONE', flush=True) |