configs/train/zbp_n64.yaml


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16

mode: cd
n_probes: 64
probe: rademacher
eps: 0.1
probe_chunk: 8
measure_fp32: true
tokens: 2e9
global_batch_tokens: 524288
lr: 6e-4
warmup: 2000
optimizer: adamw
wd: 0.1
betas: [0.9, 0.95]
grad_clip: 1.0
eval_every: 500
seed: 0