diff options
| author | One <imone@tuta.io> | 2025-07-09 10:13:51 +0800 |
|---|---|---|
| committer | One <imone@tuta.io> | 2025-07-09 10:13:51 +0800 |
| commit | bd6222774edcec1608a6842d0b06a637a4acef59 (patch) | |
| tree | 3b95517044286d82a9166bcce3134bbea099fcfe /config/cfg_pretrain.yaml | |
| parent | caa00bb77fbfce0cc14a45d5ec1c754394f96c1a (diff) | |
Release
Diffstat (limited to 'config/cfg_pretrain.yaml')
| -rw-r--r-- | config/cfg_pretrain.yaml | 31 |
1 files changed, 31 insertions, 0 deletions
diff --git a/config/cfg_pretrain.yaml b/config/cfg_pretrain.yaml new file mode 100644 index 0000000..51c55a0 --- /dev/null +++ b/config/cfg_pretrain.yaml @@ -0,0 +1,31 @@ +# ARC training config + +defaults: + - arch: hrm_v1 + - _self_ + +hydra: + output_subdir: null + +# Data path +data_path: data/arc-aug-1000 + +# Hyperparams - Training +global_batch_size: 768 + +epochs: 100000 +eval_interval: 10000 +checkpoint_every_eval: True + +lr: 1e-4 +lr_min_ratio: 1.0 +lr_warmup_steps: 2000 + +# Standard hyperparameter settings for LM, as used in Llama +beta1: 0.9 +beta2: 0.95 +weight_decay: 0.1 +puzzle_emb_weight_decay: 0.1 + +# Hyperparams - Puzzle embeddings training +puzzle_emb_lr: 1e-2 |