From 3022330910f8ada2106b5e0803b032f7b8fc6cfd Mon Sep 17 00:00:00 2001 From: Yuren Hao Date: Tue, 4 Aug 2026 05:52:16 -0500 Subject: =?UTF-8?q?HF=E4=B8=8A=E4=BC=A0:=20135M=20EP/BP=E6=9D=83=E9=87=8D?= =?UTF-8?q?=E5=AF=B9(=E5=89=A5=E4=BC=98=E5=8C=96=E5=99=A8=E7=8A=B6?= =?UTF-8?q?=E6=80=81542MB=E5=90=84)+tokenizer+notebook+=E6=9B=B2=E7=BA=BF?= =?UTF-8?q?=E5=9B=BE+model=20card=E5=88=B0=E7=A7=81=E6=9C=89ept-assets/mod?= =?UTF-8?q?els/fw135m;=20=E5=85=AC=E5=BC=80=E5=8F=91=E5=B8=83=E7=95=99?= =?UTF-8?q?=E4=BD=9C=E5=8D=95=E7=8B=AC=E5=86=B3=E5=AE=9A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014FAPDWQ49M5Ye3NpTndTpn --- ep_run/hf_upload_135m.py | 70 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 ep_run/hf_upload_135m.py (limited to 'ep_run') diff --git a/ep_run/hf_upload_135m.py b/ep_run/hf_upload_135m.py new file mode 100644 index 0000000..220e7fc --- /dev/null +++ b/ep_run/hf_upload_135m.py @@ -0,0 +1,70 @@ +"""Upload the 135M checkpoints and their demo material to the private assets repo. + +Uploads weight-only checkpoints (the optimizer state is stripped, halving the EP file), the +tokenizer, the sample notebook, the training-curve figure, and a card describing how the pair was +produced. The repo stays private; making a public model release is a separate decision. +""" +from pathlib import Path + +from huggingface_hub import HfApi + +REPO = 'blackhao0426/ept-assets' +RUN = Path('/home/yurenh2/ept/ep_run') +ASSETS = Path('/home/yurenh2/ept/assets') +PREFIX = 'models/fw135m' + +CARD = """# 135M transformer language models: Equilibrium Propagation and its backprop twin + +Two checkpoints of the same architecture, trained from scratch on FineWeb-Edu, differing only in the +training rule. + +| file | training rule | val CE (tail mean, last 10%) | best val CE | +|---|---|---|---| +| `ep_fw135m_s440000.pt` | Equilibrium Propagation, no backward pass anywhere | 3.2087 | 3.0423 | +| `bp_fw135m_s440000.pt` | backpropagation (matched twin) | 3.2071 | 3.0902 | + +A second backprop seed reached 3.2076, so the two backprop seeds differ by 0.0005 and EP sits 0.0013 +above their mean, which is 0.1% in perplexity. With one EP seed and two backprop seeds this design +cannot resolve a difference of that size; more seeds are running. + +Architecture: 12 layers, width 768, 12 heads, context 256, OLMo2-style blocks, untied 32k-vocabulary +output. 440k steps at effective batch 24, Muon hybrid optimizer, cosine schedule, mixed precision. +Trained on 2.7B tokens. + +Contents are weight-only: `tok`, `pos`, `blocks`, `wout`, `lnf`, plus `step`, `val`, and the full +training `config`. Optimizer state is stripped. Both files load with the same code, since the two +training scripts save identical keys. + +Sampling is an ordinary forward pass; Equilibrium Propagation appears only during training. See +`EPT_135M_samples.ipynb` for generations from both models on identical prompts and seeds, and +`fig_135m_curves.png` for the validation curves, including a third run that plateaus 45% higher in +perplexity because its contrast readout recovered the nudge by subtracting two large states, which +destroys the part of it that falls below single-precision resolution. +""" + + +def main(): + api = HfApi() + items = [ + (RUN / 'runs/share_fw135m_ep.pt', f'{PREFIX}/ep_fw135m_s440000.pt'), + (RUN / 'runs/share_fw135m_bp.pt', f'{PREFIX}/bp_fw135m_s440000.pt'), + (RUN / 'data/fineweb_edu/tokenizer.json', f'{PREFIX}/tokenizer.json'), + (ASSETS / 'EPT_135M_samples.ipynb', f'{PREFIX}/EPT_135M_samples.ipynb'), + (ASSETS / 'figs/fig_135m_curves.png', f'{PREFIX}/fig_135m_curves.png'), + ] + card = RUN / 'runs/_card_135m.md' + card.write_text(CARD) + items.append((card, f'{PREFIX}/README.md')) + + for src, dst in items: + if not src.exists(): + print(f'MISSING {src}') + continue + api.upload_file(path_or_fileobj=str(src), path_in_repo=dst, repo_id=REPO, + repo_type='dataset') + print(f'uploaded {dst} ({src.stat().st_size/1e6:.1f} MB)') + print(f'\nhttps://huggingface.co/datasets/{REPO}/tree/main/{PREFIX}') + + +if __name__ == '__main__': + main() -- cgit v1.2.3