From f1c2cc22d46a6976df3555391e667c7e61592fad Mon Sep 17 00:00:00 2001 From: YurenHao0426 Date: Wed, 4 Feb 2026 18:59:35 -0600 Subject: Initial commit: RL floating-point noise project --- configs/deepspeed_zero3.json | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 configs/deepspeed_zero3.json (limited to 'configs/deepspeed_zero3.json') diff --git a/configs/deepspeed_zero3.json b/configs/deepspeed_zero3.json new file mode 100644 index 0000000..6e68c8f --- /dev/null +++ b/configs/deepspeed_zero3.json @@ -0,0 +1,38 @@ +{ + "train_batch_size": "auto", + "train_micro_batch_size_per_gpu": "auto", + "gradient_accumulation_steps": "auto", + + "zero_optimization": { + "stage": 3, + "offload_optimizer": { + "device": "none" + }, + "offload_param": { + "device": "none" + }, + "overlap_comm": true, + "contiguous_gradients": true, + "sub_group_size": 1e9, + "reduce_bucket_size": "auto", + "stage3_prefetch_bucket_size": "auto", + "stage3_param_persistence_threshold": "auto", + "stage3_max_live_parameters": 1e9, + "stage3_max_reuse_distance": 1e9, + "stage3_gather_16bit_weights_on_model_save": true + }, + + "bf16": { + "enabled": false + }, + + "fp16": { + "enabled": false + }, + + "gradient_clipping": 1.0, + + "zero_allow_untested_optimizer": true, + + "wall_clock_breakdown": false +} -- cgit v1.2.3