From ad4622e42abf2a55a37c5e522a6f8302e2e95cfd Mon Sep 17 00:00:00 2001 From: Oscar Wan Date: Tue, 21 Jul 2026 15:28:06 -0700 Subject: Add portable 135M BP sweep handoff Match the active EP configuration and support four-GPU BP candidates from collaborators' data locations. Co-authored-by: Cursor --- ep_run/prepare_fineweb.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'ep_run/prepare_fineweb.py') diff --git a/ep_run/prepare_fineweb.py b/ep_run/prepare_fineweb.py index 33fce59..afd2e45 100644 --- a/ep_run/prepare_fineweb.py +++ b/ep_run/prepare_fineweb.py @@ -11,7 +11,7 @@ Phases (all resumable-ish, markers for the watcher): Docs are joined with a <|eot|> separator (id 0). vocab 32768 fits uint16. NFS note: peak disk = raw parquet ~28GB + bins ~20GB; keep raw/ for tokenizer reruns. """ -import pickle, time +import os, pickle, time from pathlib import Path import numpy as np import pyarrow.parquet as pq @@ -22,7 +22,8 @@ from tokenizers.trainers import BpeTrainer from tokenizers.pre_tokenizers import ByteLevel from tokenizers.decoders import ByteLevel as ByteLevelDec -D = Path('/home/yurenh2/ept/ep_run/data/fineweb_edu') +DATA_ROOT = Path(os.environ.get('EPT_DATA_ROOT', Path(__file__).resolve().parent / 'data')) +D = DATA_ROOT / 'fineweb_edu' RAW = D / 'raw' D.mkdir(parents=True, exist_ok=True) VOCAB = 32768 -- cgit v1.2.3