diff options
Diffstat (limited to 'ep_run/baseline_configs')
| -rw-r--r-- | ep_run/baseline_configs/fw135m_matched_bp.json | 10 |
1 files changed, 5 insertions, 5 deletions
diff --git a/ep_run/baseline_configs/fw135m_matched_bp.json b/ep_run/baseline_configs/fw135m_matched_bp.json index 0e22686..98fa032 100644 --- a/ep_run/baseline_configs/fw135m_matched_bp.json +++ b/ep_run/baseline_configs/fw135m_matched_bp.json @@ -5,7 +5,7 @@ "width": "512 -> 768", "heads": "8 -> 12", "parameters": "72,114,688 -> 135,303,936", - "training_updates": "440,443 updates to preserve approximately 20 tokens/parameter" + "training_updates": "440,001 updates to match the fw135m_bsign EP trainer argument" }, "held_fixed_from_72m": { "layers": 12, @@ -19,7 +19,7 @@ "muon_lr": 0.02, "adam_side_lr_center": 0.001, "weight_decay": 0.1, - "warmup_steps": 500, + "warmup_steps": 1000, "schedule": "cosine to 0.1 of peak", "precision": "bf16 autocast with fp32 parameters/states" }, @@ -33,9 +33,9 @@ "batch": 24, "parameters": 135303936, "target_tokens_20N": 2706078720, - "trainer_steps_argument": 440442, - "actual_updates": 440443, - "actual_tokens": 2706081792 + "trainer_steps_argument": 440000, + "actual_updates": 440001, + "actual_tokens": 2703366144 }, "bp_lr_sweep_smallest_rung_only": [0.0007, 0.001, 0.0014], "minimum_seeds_before_reporting": 2 |