1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
|
{
"config": {
"capacity_q": 0.01,
"data_seed": 7,
"device": "cpu",
"feedback_scale": "relu",
"feedback_seed_offset": 0,
"feedback_seeds": 8,
"init_seed_offset": 0,
"init_seeds": 3,
"input_dim": 16,
"jacobian_lambda_rel": 0.001,
"lr": 0.01,
"noise_std": 0.0,
"normalize_targets": true,
"optimizer": "sgd",
"outdir": "outputs/teacher_test_gap_sgd_T8000",
"output_dim": 4,
"plot": true,
"probe_samples": 32,
"skip_jacobian": true,
"steps": 8000,
"task": "mlp",
"teacher_hidden_layers": 2,
"teacher_rank": 4,
"teacher_width": 64,
"test_samples": 2048,
"torch_threads": 8,
"train_samples": 256,
"widths": [
8,
12,
16,
24,
32,
48,
64,
96
]
},
"width_summary": [
{
"bp_capacity_margin": -800,
"bp_test_mean": 0.8547195125219496,
"bp_test_std": 0.09416632052520177,
"bp_train_mean": 0.48654225381273647,
"bp_train_std": 0.009430396910714468,
"d_eff_mean": 0.0,
"fa_burden_nats": 1.3936644807238818,
"fa_capacity_margin": -894,
"fa_constraint_rank": 94,
"fa_test_mean": 1.03173780938202,
"fa_test_std": 0.07195063786750613,
"fa_train_mean": 0.7360187817128564,
"fa_train_std": 0.08065577661619448,
"gap_mean": 0.1770182968600705,
"gap_std": 0.09915053419946865,
"hard_rank_mean": 0.0,
"parameter_count": 224,
"redundancy_score_mean": 222.6063355192761,
"runs_bp": 3,
"runs_fa": 24,
"task_dimension": 1024,
"train_gap_mean": 0.24947652790011995,
"train_gap_std": 0.07918404890030231,
"width": 8
},
{
"bp_capacity_margin": -640,
"bp_test_mean": 0.8912211187592197,
"bp_test_std": 0.06296622353182581,
"bp_train_mean": 0.35139158815529714,
"bp_train_std": 0.004244751408287705,
"d_eff_mean": 0.0,
"fa_burden_nats": 2.1683902917154665,
"fa_capacity_margin": -830,
"fa_constraint_rank": 190,
"fa_test_mean": 0.9292951764049313,
"fa_test_std": 0.07525411172556047,
"fa_train_mean": 0.5094820739240674,
"fa_train_std": 0.0707313348013235,
"gap_mean": 0.03807405764571161,
"gap_std": 0.07715664669655001,
"hard_rank_mean": 0.0,
"parameter_count": 384,
"redundancy_score_mean": 381.83160970828453,
"runs_bp": 3,
"runs_fa": 24,
"task_dimension": 1024,
"train_gap_mean": 0.15809048576877027,
"train_gap_std": 0.07029272672610735,
"width": 12
},
{
"bp_capacity_margin": -448,
"bp_test_mean": 0.9192548641208319,
"bp_test_std": 0.033740239256328644,
"bp_train_mean": 0.2520286029851933,
"bp_train_std": 0.007647223072501642,
"d_eff_mean": 0.0,
"fa_burden_nats": 3.058119959290285,
"fa_capacity_margin": -766,
"fa_constraint_rank": 318,
"fa_test_mean": 0.8812679121253054,
"fa_test_std": 0.07027921512298203,
"fa_train_mean": 0.3635437201887566,
"fa_train_std": 0.047364665684149765,
"gap_mean": -0.037986951995526305,
"gap_std": 0.07361982827194134,
"hard_rank_mean": 0.0,
"parameter_count": 576,
"redundancy_score_mean": 572.9418800407097,
"runs_bp": 3,
"runs_fa": 24,
"task_dimension": 1024,
"train_gap_mean": 0.11151511720356323,
"train_gap_std": 0.04520548886437488,
"width": 16
},
{
"bp_capacity_margin": 32,
"bp_test_mean": 0.9293582800572858,
"bp_test_std": 0.07241895652383532,
"bp_train_mean": 0.1396784759530465,
"bp_train_std": 0.01665120505504232,
"d_eff_mean": 0.0,
"fa_burden_nats": 5.228037692481937,
"fa_capacity_margin": -638,
"fa_constraint_rank": 670,
"fa_test_mean": 0.8851293442398639,
"fa_test_std": 0.06255425075132401,
"fa_train_mean": 0.219954653180603,
"fa_train_std": 0.033287750843013345,
"gap_mean": -0.044228935817421745,
"gap_std": 0.09040086678358636,
"hard_rank_mean": 0.0,
"parameter_count": 1056,
"redundancy_score_mean": 1050.7719623075182,
"runs_bp": 3,
"runs_fa": 24,
"task_dimension": 1024,
"train_gap_mean": 0.08027617722755646,
"train_gap_std": 0.034751538455168735,
"width": 24
},
{
"bp_capacity_margin": 640,
"bp_test_mean": 1.0541338661341875,
"bp_test_std": 0.018275526958484174,
"bp_train_mean": 0.0674924657026168,
"bp_train_std": 0.010299416214868514,
"d_eff_mean": 0.0,
"fa_burden_nats": 7.958568352977123,
"fa_capacity_margin": -510,
"fa_constraint_rank": 1150,
"fa_test_mean": 0.9339004211933818,
"fa_test_std": 0.06403389508198398,
"fa_train_mean": 0.1521629814467578,
"fa_train_std": 0.01750960740931603,
"gap_mean": -0.12023344494080564,
"gap_std": 0.06569708866945986,
"hard_rank_mean": 0.0,
"parameter_count": 1664,
"redundancy_score_mean": 1656.0414316470226,
"runs_bp": 3,
"runs_fa": 24,
"task_dimension": 1024,
"train_gap_mean": 0.08467051574414101,
"train_gap_std": 0.015298364700276968,
"width": 32
},
{
"bp_capacity_margin": 2240,
"bp_test_mean": 0.9791622527841524,
"bp_test_std": 0.047110478662005864,
"bp_train_mean": 0.022625612139090635,
"bp_train_std": 0.004106878454242747,
"d_eff_mean": 0.0,
"fa_burden_nats": 15.199401552235027,
"fa_capacity_margin": -254,
"fa_constraint_rank": 2494,
"fa_test_mean": 0.9653186953239867,
"fa_test_std": 0.05187486017852229,
"fa_train_mean": 0.07109851792006452,
"fa_train_std": 0.007557098583336046,
"gap_mean": -0.013843557460165468,
"gap_std": 0.06974990528996139,
"hard_rank_mean": 0.0,
"parameter_count": 3264,
"redundancy_score_mean": 3248.8005984477654,
"runs_bp": 3,
"runs_fa": 24,
"task_dimension": 1024,
"train_gap_mean": 0.048472905780973886,
"train_gap_std": 0.008244124666444821,
"width": 48
},
{
"bp_capacity_margin": 4352,
"bp_test_mean": 0.9653310456956622,
"bp_test_std": 0.03177320206769566,
"bp_train_mean": 0.006939575153079867,
"bp_train_std": 0.0012012935377682527,
"d_eff_mean": 0.0,
"fa_burden_nats": 24.89249668727425,
"fa_capacity_margin": 2,
"fa_constraint_rank": 4350,
"fa_test_mean": 0.9750088396534556,
"fa_test_std": 0.04179311994033258,
"fa_train_mean": 0.033249367604793946,
"fa_train_std": 0.004644998390740636,
"gap_mean": 0.009677793957793427,
"gap_std": 0.040482838262068055,
"hard_rank_mean": 0.0,
"parameter_count": 5376,
"redundancy_score_mean": 5351.107503312725,
"runs_bp": 3,
"runs_fa": 24,
"task_dimension": 1024,
"train_gap_mean": 0.02630979245171408,
"train_gap_std": 0.004635709869038379,
"width": 64
},
{
"bp_capacity_margin": 10112,
"bp_test_mean": 0.9064751809116146,
"bp_test_std": 0.05494609929229648,
"bp_train_mean": 0.0010748642908555096,
"bp_train_std": 2.7736586821309024e-05,
"d_eff_mean": 0.0,
"fa_burden_nats": 51.802320004200304,
"fa_capacity_margin": 514,
"fa_constraint_rank": 9598,
"fa_test_mean": 0.9513155139710398,
"fa_test_std": 0.04816011708858036,
"fa_train_mean": 0.007945258532614088,
"fa_train_std": 0.001452885056156375,
"gap_mean": 0.044840333059425225,
"gap_std": 0.05220001570680324,
"hard_rank_mean": 0.0,
"parameter_count": 11136,
"redundancy_score_mean": 11084.1976799958,
"runs_bp": 3,
"runs_fa": 24,
"task_dimension": 1024,
"train_gap_mean": 0.006870394241758578,
"train_gap_std": 0.0014660396836900331,
"width": 96
}
]
}
|
