diff options
Diffstat (limited to 'ep_run/prepare_fineweb.py')
| -rw-r--r-- | ep_run/prepare_fineweb.py | 5 |
1 files changed, 3 insertions, 2 deletions
diff --git a/ep_run/prepare_fineweb.py b/ep_run/prepare_fineweb.py index 33fce59..afd2e45 100644 --- a/ep_run/prepare_fineweb.py +++ b/ep_run/prepare_fineweb.py @@ -11,7 +11,7 @@ Phases (all resumable-ish, markers for the watcher): Docs are joined with a <|eot|> separator (id 0). vocab 32768 fits uint16. NFS note: peak disk = raw parquet ~28GB + bins ~20GB; keep raw/ for tokenizer reruns. """ -import pickle, time +import os, pickle, time from pathlib import Path import numpy as np import pyarrow.parquet as pq @@ -22,7 +22,8 @@ from tokenizers.trainers import BpeTrainer from tokenizers.pre_tokenizers import ByteLevel from tokenizers.decoders import ByteLevel as ByteLevelDec -D = Path('/home/yurenh2/ept/ep_run/data/fineweb_edu') +DATA_ROOT = Path(os.environ.get('EPT_DATA_ROOT', Path(__file__).resolve().parent / 'data')) +D = DATA_ROOT / 'fineweb_edu' RAW = D / 'raw' D.mkdir(parents=True, exist_ok=True) VOCAB = 32768 |