diff options
| author | Oscar Wan <oscarwan@oriong12.stanford.edu> | 2026-07-21 15:28:06 -0700 |
|---|---|---|
| committer | Oscar Wan <oscarwan@oriong12.stanford.edu> | 2026-07-21 15:28:06 -0700 |
| commit | ad4622e42abf2a55a37c5e522a6f8302e2e95cfd (patch) | |
| tree | ce668b916f9f16c4153c8b658bd772cd338ff90b /ep_run/prepare_fineweb.py | |
| parent | 09c52fa97a946cf6dbdc500aef88da8522c2ce7d (diff) | |
Add portable 135M BP sweep handoff
Match the active EP configuration and support four-GPU BP candidates from collaborators' data locations.
Co-authored-by: Cursor <cursoragent@cursor.com>
Diffstat (limited to 'ep_run/prepare_fineweb.py')
| -rw-r--r-- | ep_run/prepare_fineweb.py | 5 |
1 files changed, 3 insertions, 2 deletions
diff --git a/ep_run/prepare_fineweb.py b/ep_run/prepare_fineweb.py index 33fce59..afd2e45 100644 --- a/ep_run/prepare_fineweb.py +++ b/ep_run/prepare_fineweb.py @@ -11,7 +11,7 @@ Phases (all resumable-ish, markers for the watcher): Docs are joined with a <|eot|> separator (id 0). vocab 32768 fits uint16. NFS note: peak disk = raw parquet ~28GB + bins ~20GB; keep raw/ for tokenizer reruns. """ -import pickle, time +import os, pickle, time from pathlib import Path import numpy as np import pyarrow.parquet as pq @@ -22,7 +22,8 @@ from tokenizers.trainers import BpeTrainer from tokenizers.pre_tokenizers import ByteLevel from tokenizers.decoders import ByteLevel as ByteLevelDec -D = Path('/home/yurenh2/ept/ep_run/data/fineweb_edu') +DATA_ROOT = Path(os.environ.get('EPT_DATA_ROOT', Path(__file__).resolve().parent / 'data')) +D = DATA_ROOT / 'fineweb_edu' RAW = D / 'raw' D.mkdir(parents=True, exist_ok=True) VOCAB = 32768 |