6 files changed, 64 insertions, 0 deletions

diff --git a/configs/model/l350.yaml b/configs/model/l350.yaml
new file mode 100644
index 0000000..cd36f07
--- /dev/null
+++ b/configs/model/l350.yaml
@@ -0,0 +1,7 @@
+d: 1024
+layers: 24
+heads: 16
+vocab: 50257
+seq_len: 1024
+ffn_mult: 8/3
+pos: learned
diff --git a/configs/model/m124.yaml b/configs/model/m124.yaml
new file mode 100644
index 0000000..ad3b5fa
--- /dev/null
+++ b/configs/model/m124.yaml
@@ -0,0 +1,7 @@
+d: 768
+layers: 12
+heads: 12
+vocab: 50257
+seq_len: 1024
+ffn_mult: 8/3
+pos: learned
diff --git a/configs/model/s60.yaml b/configs/model/s60.yaml
new file mode 100644
index 0000000..ebccc0a
--- /dev/null
+++ b/configs/model/s60.yaml
@@ -0,0 +1,7 @@
+d: 512
+layers: 8
+heads: 8
+vocab: 50257
+seq_len: 1024
+ffn_mult: 8/3        # SwiGLU: hidden = mult*d rounded to 64
+pos: learned
diff --git a/configs/train/bp.yaml b/configs/train/bp.yaml
new file mode 100644
index 0000000..4573a30
--- /dev/null
+++ b/configs/train/bp.yaml
@@ -0,0 +1,11 @@
+mode: bp
+tokens: 2e9
+global_batch_tokens: 524288      # 512 seqs x 1024
+lr: 6e-4
+warmup: 2000
+optimizer: adamw
+wd: 0.1
+betas: [0.9, 0.95]
+grad_clip: 1.0
+eval_every: 500
+seed: 0
diff --git a/configs/train/zbp_n16.yaml b/configs/train/zbp_n16.yaml
new file mode 100644
index 0000000..6f97a80
--- /dev/null
+++ b/configs/train/zbp_n16.yaml
@@ -0,0 +1,16 @@
+mode: cd
+n_probes: 16
+probe: rademacher
+eps: 0.1
+probe_chunk: 8
+measure_fp32: true
+tokens: 2e9
+global_batch_tokens: 524288
+lr: 6e-4
+warmup: 2000
+optimizer: adamw
+wd: 0.1
+betas: [0.9, 0.95]
+grad_clip: 1.0
+eval_every: 500
+seed: 0
diff --git a/configs/train/zbp_n4.yaml b/configs/train/zbp_n4.yaml
new file mode 100644
index 0000000..cb26322
--- /dev/null
+++ b/configs/train/zbp_n4.yaml
@@ -0,0 +1,16 @@
+mode: cd
+n_probes: 4
+probe: rademacher
+eps: 0.1
+probe_chunk: 8
+measure_fp32: true
+tokens: 2e9
+global_batch_tokens: 524288
+lr: 6e-4
+warmup: 2000
+optimizer: adamw
+wd: 0.1
+betas: [0.9, 0.95]
+grad_clip: 1.0
+eval_every: 500
+seed: 0