diff options
| author | YurenHao0426 <blackhao0426@gmail.com> | 2025-12-17 04:29:37 -0600 |
|---|---|---|
| committer | YurenHao0426 <blackhao0426@gmail.com> | 2025-12-17 04:29:37 -0600 |
| commit | e43b3f8aa36c198b95c1e46bea2eaf3893b13dc3 (patch) | |
| tree | 6ce8a00d2f8b9ebd83c894a27ea01ac50cfb2ff5 /configs/qwen2.5_0.5b_full_sft.yaml | |
Diffstat (limited to 'configs/qwen2.5_0.5b_full_sft.yaml')
| -rw-r--r-- | configs/qwen2.5_0.5b_full_sft.yaml | 34 |
1 files changed, 34 insertions, 0 deletions
diff --git a/configs/qwen2.5_0.5b_full_sft.yaml b/configs/qwen2.5_0.5b_full_sft.yaml new file mode 100644 index 0000000..ca1cca2 --- /dev/null +++ b/configs/qwen2.5_0.5b_full_sft.yaml @@ -0,0 +1,34 @@ +### Qwen2.5-0.5B Full SFT Config +model_name_or_path: Qwen/Qwen2.5-0.5B-Instruct +stage: sft +do_train: true +finetuning_type: full +freeze_trainable_layers: 0 + +dataset: preference_extractor_train +template: qwen +cutoff_len: 1024 +overwrite_cache: true +preprocessing_num_workers: 16 + +output_dir: saves/qwen2.5-0.5b-full-sft +logging_steps: 10 +save_strategy: steps +save_steps: 500 +plot_loss: true +overwrite_output_dir: true + +per_device_train_batch_size: 16 +gradient_accumulation_steps: 8 +learning_rate: 2.0e-5 +num_train_epochs: 1.0 +lr_scheduler_type: cosine +warmup_ratio: 0.05 +bf16: true +flash_attn: fa2 + +val_size: 0.01 +per_device_eval_batch_size: 16 +eval_strategy: steps +eval_steps: 500 + |