From 6a544fabfc2af22e4d5823410dd2387b5af89ea9 Mon Sep 17 00:00:00 2001 From: yurenh Date: Mon, 31 Aug 2026 18:14:09 -0500 Subject: scaffold: model (OLMo2-ish + ZBP partition), trainer (DDP/config), data shards, bench Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01GkgLsACEF6CCP7EUfA5fZe --- configs/model/l350.yaml | 7 +++++++ configs/model/m124.yaml | 7 +++++++ configs/model/s60.yaml | 7 +++++++ configs/train/bp.yaml | 11 +++++++++++ configs/train/zbp_n16.yaml | 16 ++++++++++++++++ configs/train/zbp_n4.yaml | 16 ++++++++++++++++ 6 files changed, 64 insertions(+) create mode 100644 configs/model/l350.yaml create mode 100644 configs/model/m124.yaml create mode 100644 configs/model/s60.yaml create mode 100644 configs/train/bp.yaml create mode 100644 configs/train/zbp_n16.yaml create mode 100644 configs/train/zbp_n4.yaml (limited to 'configs') diff --git a/configs/model/l350.yaml b/configs/model/l350.yaml new file mode 100644 index 0000000..cd36f07 --- /dev/null +++ b/configs/model/l350.yaml @@ -0,0 +1,7 @@ +d: 1024 +layers: 24 +heads: 16 +vocab: 50257 +seq_len: 1024 +ffn_mult: 8/3 +pos: learned diff --git a/configs/model/m124.yaml b/configs/model/m124.yaml new file mode 100644 index 0000000..ad3b5fa --- /dev/null +++ b/configs/model/m124.yaml @@ -0,0 +1,7 @@ +d: 768 +layers: 12 +heads: 12 +vocab: 50257 +seq_len: 1024 +ffn_mult: 8/3 +pos: learned diff --git a/configs/model/s60.yaml b/configs/model/s60.yaml new file mode 100644 index 0000000..ebccc0a --- /dev/null +++ b/configs/model/s60.yaml @@ -0,0 +1,7 @@ +d: 512 +layers: 8 +heads: 8 +vocab: 50257 +seq_len: 1024 +ffn_mult: 8/3 # SwiGLU: hidden = mult*d rounded to 64 +pos: learned diff --git a/configs/train/bp.yaml b/configs/train/bp.yaml new file mode 100644 index 0000000..4573a30 --- /dev/null +++ b/configs/train/bp.yaml @@ -0,0 +1,11 @@ +mode: bp +tokens: 2e9 +global_batch_tokens: 524288 # 512 seqs x 1024 +lr: 6e-4 +warmup: 2000 +optimizer: adamw +wd: 0.1 +betas: [0.9, 0.95] +grad_clip: 1.0 +eval_every: 500 +seed: 0 diff --git a/configs/train/zbp_n16.yaml b/configs/train/zbp_n16.yaml new file mode 100644 index 0000000..6f97a80 --- /dev/null +++ b/configs/train/zbp_n16.yaml @@ -0,0 +1,16 @@ +mode: cd +n_probes: 16 +probe: rademacher +eps: 0.1 +probe_chunk: 8 +measure_fp32: true +tokens: 2e9 +global_batch_tokens: 524288 +lr: 6e-4 +warmup: 2000 +optimizer: adamw +wd: 0.1 +betas: [0.9, 0.95] +grad_clip: 1.0 +eval_every: 500 +seed: 0 diff --git a/configs/train/zbp_n4.yaml b/configs/train/zbp_n4.yaml new file mode 100644 index 0000000..cb26322 --- /dev/null +++ b/configs/train/zbp_n4.yaml @@ -0,0 +1,16 @@ +mode: cd +n_probes: 4 +probe: rademacher +eps: 0.1 +probe_chunk: 8 +measure_fp32: true +tokens: 2e9 +global_batch_tokens: 524288 +lr: 6e-4 +warmup: 2000 +optimizer: adamw +wd: 0.1 +betas: [0.9, 0.95] +grad_clip: 1.0 +eval_every: 500 +seed: 0 -- cgit v1.2.3