1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
|
"""Upload collected results (and optionally checkpoints) to a private HF repo.
AUTH: read STRICTLY from the environment -- the HF_TOKEN env var or a standard `hf auth login`.
Never passed as a CLI argument (argv is world-readable on shared nodes), never written to any file.
On shared nodes use a FINE-GRAINED token scoped to this single repo with write-only permission,
exported per session: export HF_TOKEN=hf_... (see README "Security").
python scripts/upload_hf.py --results results/h200node1 [--repo user/zbp-scaling-runs] [--with-ckpt runs]
"""
import os, glob, argparse
from huggingface_hub import HfApi
p = argparse.ArgumentParser()
p.add_argument("--results", required=True)
p.add_argument("--repo", default=None, help="default: <whoami>/zbp-scaling-runs")
p.add_argument("--with-ckpt", default=None, help="runs dir: also upload runs/*/ckpt.pt (large!)")
a = p.parse_args()
api = HfApi() # token from env / login cache only
repo = a.repo or f"{api.whoami()['name']}/zbp-scaling-runs"
api.create_repo(repo, private=True, exist_ok=True)
tag = os.path.basename(os.path.normpath(a.results))
api.upload_folder(folder_path=a.results, path_in_repo=f"results/{tag}", repo_id=repo,
commit_message=f"results: {tag}")
print(f"uploaded results/{tag} -> https://huggingface.co/{repo}")
if a.with_ckpt:
for c in sorted(glob.glob(os.path.join(a.with_ckpt, "*", "ckpt.pt"))):
name = os.path.basename(os.path.dirname(c))
api.upload_file(path_or_fileobj=c, path_in_repo=f"ckpts/{tag}/{name}.pt", repo_id=repo,
commit_message=f"ckpt: {tag}/{name}")
print(f"uploaded ckpts/{tag}/{name}.pt")
|
