mode: bp
tokens: 2e9
global_batch_tokens: 524288      # 512 seqs x 1024
lr: 6e-4
warmup: 2000
optimizer: adamw
wd: 0.1
betas: [0.9, 0.95]
grad_clip: 1.0
eval_every: 500
seed: 0