diff options
Diffstat (limited to 'configs/train/bp.yaml')
| -rw-r--r-- | configs/train/bp.yaml | 11 |
1 files changed, 11 insertions, 0 deletions
diff --git a/configs/train/bp.yaml b/configs/train/bp.yaml new file mode 100644 index 0000000..4573a30 --- /dev/null +++ b/configs/train/bp.yaml @@ -0,0 +1,11 @@ +mode: bp +tokens: 2e9 +global_batch_tokens: 524288 # 512 seqs x 1024 +lr: 6e-4 +warmup: 2000 +optimizer: adamw +wd: 0.1 +betas: [0.9, 0.95] +grad_clip: 1.0 +eval_every: 500 +seed: 0 |