From d75a36d29aabbc15031a292caa52e565fdd7ea44 Mon Sep 17 00:00:00 2001 From: yurenh Date: Mon, 31 Aug 2026 18:38:42 -0500 Subject: HF upload tooling: env-only auth, scoped-token guidance, optional ladder auto-upload Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01GkgLsACEF6CCP7EUfA5fZe --- scripts/upload_hf.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 scripts/upload_hf.py (limited to 'scripts/upload_hf.py') diff --git a/scripts/upload_hf.py b/scripts/upload_hf.py new file mode 100644 index 0000000..284b9a9 --- /dev/null +++ b/scripts/upload_hf.py @@ -0,0 +1,30 @@ +"""Upload collected results (and optionally checkpoints) to a private HF repo. + +AUTH: read STRICTLY from the environment -- the HF_TOKEN env var or a standard `hf auth login`. +Never passed as a CLI argument (argv is world-readable on shared nodes), never written to any file. +On shared nodes use a FINE-GRAINED token scoped to this single repo with write-only permission, +exported per session: export HF_TOKEN=hf_... (see README "Security"). + + python scripts/upload_hf.py --results results/h200node1 [--repo user/zbp-scaling-runs] [--with-ckpt runs] +""" +import os, glob, argparse +from huggingface_hub import HfApi + +p = argparse.ArgumentParser() +p.add_argument("--results", required=True) +p.add_argument("--repo", default=None, help="default: /zbp-scaling-runs") +p.add_argument("--with-ckpt", default=None, help="runs dir: also upload runs/*/ckpt.pt (large!)") +a = p.parse_args() +api = HfApi() # token from env / login cache only +repo = a.repo or f"{api.whoami()['name']}/zbp-scaling-runs" +api.create_repo(repo, private=True, exist_ok=True) +tag = os.path.basename(os.path.normpath(a.results)) +api.upload_folder(folder_path=a.results, path_in_repo=f"results/{tag}", repo_id=repo, + commit_message=f"results: {tag}") +print(f"uploaded results/{tag} -> https://huggingface.co/{repo}") +if a.with_ckpt: + for c in sorted(glob.glob(os.path.join(a.with_ckpt, "*", "ckpt.pt"))): + name = os.path.basename(os.path.dirname(c)) + api.upload_file(path_or_fileobj=c, path_in_repo=f"ckpts/{tag}/{name}.pt", repo_id=repo, + commit_message=f"ckpt: {tag}/{name}") + print(f"uploaded ckpts/{tag}/{name}.pt") -- cgit v1.2.3