1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
|
{
"source_of_truth": "docs/BASELINE_SPEC.md",
"purpose": "First approximately 2x BP twin rung after the completed 72M FineWeb model",
"changed_from_72m": {
"width": "512 -> 768",
"heads": "8 -> 12",
"parameters": "72,114,688 -> 135,303,936",
"training_updates": "440,443 updates to preserve approximately 20 tokens/parameter"
},
"held_fixed_from_72m": {
"layers": 12,
"head_dim": 64,
"context": 256,
"global_sequence_batch": 24,
"data": "fineweb_edu",
"vocab": 32768,
"architecture": "local OLMo2-style",
"optimizer": "Muon on hidden matrices plus AdamW on remaining parameters",
"muon_lr": 0.02,
"adam_side_lr_center": 0.001,
"weight_decay": 0.1,
"warmup_steps": 500,
"schedule": "cosine to 0.1 of peak",
"precision": "bf16 autocast with fp32 parameters/states"
},
"locked_run": {
"tag": "fw135m_bp",
"layers": 12,
"width": 768,
"heads": 12,
"head_dim": 64,
"context": 256,
"batch": 24,
"parameters": 135303936,
"target_tokens_20N": 2706078720,
"trainer_steps_argument": 440442,
"actual_updates": 440443,
"actual_tokens": 2706081792
},
"bp_lr_sweep_smallest_rung_only": [0.0007, 0.001, 0.0014],
"minimum_seeds_before_reporting": 2
}
|
