diff options
| author | Oscar Wan <oscarwan@oriong12.stanford.edu> | 2026-07-21 15:28:06 -0700 |
|---|---|---|
| committer | Oscar Wan <oscarwan@oriong12.stanford.edu> | 2026-07-21 15:28:06 -0700 |
| commit | ad4622e42abf2a55a37c5e522a6f8302e2e95cfd (patch) | |
| tree | ce668b916f9f16c4153c8b658bd772cd338ff90b /ep_run/baseline_configs | |
| parent | 09c52fa97a946cf6dbdc500aef88da8522c2ce7d (diff) | |
Add portable 135M BP sweep handoff
Match the active EP configuration and support four-GPU BP candidates from collaborators' data locations.
Co-authored-by: Cursor <cursoragent@cursor.com>
Diffstat (limited to 'ep_run/baseline_configs')
| -rw-r--r-- | ep_run/baseline_configs/fw135m_matched_bp.json | 10 |
1 files changed, 5 insertions, 5 deletions
diff --git a/ep_run/baseline_configs/fw135m_matched_bp.json b/ep_run/baseline_configs/fw135m_matched_bp.json index 0e22686..98fa032 100644 --- a/ep_run/baseline_configs/fw135m_matched_bp.json +++ b/ep_run/baseline_configs/fw135m_matched_bp.json @@ -5,7 +5,7 @@ "width": "512 -> 768", "heads": "8 -> 12", "parameters": "72,114,688 -> 135,303,936", - "training_updates": "440,443 updates to preserve approximately 20 tokens/parameter" + "training_updates": "440,001 updates to match the fw135m_bsign EP trainer argument" }, "held_fixed_from_72m": { "layers": 12, @@ -19,7 +19,7 @@ "muon_lr": 0.02, "adam_side_lr_center": 0.001, "weight_decay": 0.1, - "warmup_steps": 500, + "warmup_steps": 1000, "schedule": "cosine to 0.1 of peak", "precision": "bf16 autocast with fp32 parameters/states" }, @@ -33,9 +33,9 @@ "batch": 24, "parameters": 135303936, "target_tokens_20N": 2706078720, - "trainer_steps_argument": 440442, - "actual_updates": 440443, - "actual_tokens": 2706081792 + "trainer_steps_argument": 440000, + "actual_updates": 440001, + "actual_tokens": 2703366144 }, "bp_lr_sweep_smallest_rung_only": [0.0007, 0.001, 0.0014], "minimum_seeds_before_reporting": 2 |