From f1c2cc22d46a6976df3555391e667c7e61592fad Mon Sep 17 00:00:00 2001 From: YurenHao0426 Date: Wed, 4 Feb 2026 18:59:35 -0600 Subject: Initial commit: RL floating-point noise project --- configs/deepspeed_zero2.json | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 configs/deepspeed_zero2.json (limited to 'configs/deepspeed_zero2.json') diff --git a/configs/deepspeed_zero2.json b/configs/deepspeed_zero2.json new file mode 100644 index 0000000..bb7f7aa --- /dev/null +++ b/configs/deepspeed_zero2.json @@ -0,0 +1,31 @@ +{ + "train_batch_size": "auto", + "train_micro_batch_size_per_gpu": "auto", + "gradient_accumulation_steps": "auto", + + "zero_optimization": { + "stage": 2, + "offload_optimizer": { + "device": "none" + }, + "contiguous_gradients": true, + "overlap_comm": true, + "reduce_scatter": true, + "reduce_bucket_size": 5e8, + "allgather_bucket_size": 5e8 + }, + + "bf16": { + "enabled": false + }, + + "fp16": { + "enabled": false + }, + + "gradient_clipping": 1.0, + + "zero_allow_untested_optimizer": true, + + "wall_clock_breakdown": false +} -- cgit v1.2.3