1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
|
{
"status": "development_only",
"claim_policy": "This short same-rate pilot checks mechanics and direction only. It is excluded from T1 selection, the primary crossover, figures, and reviewer-score changes.",
"source_commit": "08554657946e54d8d908a2866e82504e9b194617",
"configuration": {
"task": "tiny Shakespeare character language modeling",
"train_sha256": "6ec305602a99ac2802745a134e1f5e33e2231b4855525b00b9aebb730ac2626f",
"validation_sha256": "d37d30cc0c8327c270d493299c3dca54135f6d5f1c9ef60cda78076e311204b1",
"depths": [
4,
8,
12
],
"methods": [
"bp",
"fa",
"dfa",
"clean_kp",
"sdil"
],
"width": 128,
"heads": 4,
"context_length": 64,
"batch_size": 8,
"train_steps": 100,
"learning_rate": 0.001,
"schedule": "constant",
"validation_batches": 10,
"validation_tokens": 5120,
"test_evaluations": 0
},
"records": [
{
"depth": 4,
"method": "bp",
"validation_nll": 2.5736704111099242,
"wall_seconds": 3.6037685871124268,
"first_nonfinite_step": null
},
{
"depth": 4,
"method": "fa",
"validation_nll": 2.7691116333007812,
"wall_seconds": 4.258021593093872,
"first_nonfinite_step": null
},
{
"depth": 4,
"method": "dfa",
"validation_nll": 3.252720332145691,
"wall_seconds": 4.3479225635528564,
"first_nonfinite_step": null
},
{
"depth": 4,
"method": "clean_kp",
"validation_nll": 2.642165517807007,
"wall_seconds": 5.114134073257446,
"first_nonfinite_step": null
},
{
"depth": 4,
"method": "sdil",
"validation_nll": 2.6421636819839476,
"wall_seconds": 5.935818672180176,
"first_nonfinite_step": null
},
{
"depth": 8,
"method": "bp",
"validation_nll": 2.8022268295288084,
"wall_seconds": 5.785389184951782,
"first_nonfinite_step": null
},
{
"depth": 8,
"method": "fa",
"validation_nll": 2.8849047660827636,
"wall_seconds": 7.843722105026245,
"first_nonfinite_step": null
},
{
"depth": 8,
"method": "dfa",
"validation_nll": 3.3771399736404417,
"wall_seconds": 7.509201526641846,
"first_nonfinite_step": null
},
{
"depth": 8,
"method": "clean_kp",
"validation_nll": 2.691094470024109,
"wall_seconds": 9.399603366851807,
"first_nonfinite_step": null
},
{
"depth": 8,
"method": "sdil",
"validation_nll": 2.691092610359192,
"wall_seconds": 10.85186505317688,
"first_nonfinite_step": null
},
{
"depth": 12,
"method": "bp",
"validation_nll": 2.771989607810974,
"wall_seconds": 9.70148754119873,
"first_nonfinite_step": null
},
{
"depth": 12,
"method": "fa",
"validation_nll": 2.8445181608200074,
"wall_seconds": 11.526866912841797,
"first_nonfinite_step": null
},
{
"depth": 12,
"method": "dfa",
"validation_nll": 3.3635837554931642,
"wall_seconds": 12.529596090316772,
"first_nonfinite_step": null
},
{
"depth": 12,
"method": "clean_kp",
"validation_nll": 2.6933349132537843,
"wall_seconds": 13.416383028030396,
"first_nonfinite_step": null
},
{
"depth": 12,
"method": "sdil",
"validation_nll": 2.6933418989181517,
"wall_seconds": 15.886815071105957,
"first_nonfinite_step": null
}
],
"depth12_minus_depth4_nll": {
"bp": 0.1983191967010498,
"fa": 0.07540652751922616,
"dfa": 0.11086342334747323,
"clean_kp": 0.051169395446777344,
"sdil": 0.0511782169342041
},
"summary": "All 15 trajectories are finite. Reciprocal KP/SDIL show the smallest short-budget depth degradation, but the clean-task KP/SDIL identity is not innovation evidence."
}
|
