diff options
Diffstat (limited to 'results/fa_dfa_d256_L12_seed0/results_cifar10.json')
| -rw-r--r-- | results/fa_dfa_d256_L12_seed0/results_cifar10.json | 969 |
1 files changed, 969 insertions, 0 deletions
diff --git a/results/fa_dfa_d256_L12_seed0/results_cifar10.json b/results/fa_dfa_d256_L12_seed0/results_cifar10.json new file mode 100644 index 0000000..a31aa8b --- /dev/null +++ b/results/fa_dfa_d256_L12_seed0/results_cifar10.json @@ -0,0 +1,969 @@ +{ + "0": { + "dfa": { + "log": { + "train_loss": [ + 2.078884480743408, + 2.059846487350464, + 2.0553709620666503, + 2.0546888257598876, + 2.047630134963989, + 2.045008226928711, + 2.0420518775177, + 2.041442600250244, + 2.0381681137847902, + 2.0336293191146853, + 2.0332381079864503, + 2.0334933743286134, + 2.035796566848755, + 2.0294106813049315, + 2.0285469360351565, + 2.026694899978638, + 2.0260820417785643, + 2.0254271863555906, + 2.0230323413848876, + 2.0229154515075685, + 2.0223220541763305, + 2.0228745832824706, + 2.021729295196533, + 2.017538852996826, + 2.017971972579956, + 2.0173734220886232, + 2.017427021217346, + 2.016292642364502, + 2.0162722787475587, + 2.0143618490600588, + 2.013466402053833, + 2.013067155303955, + 2.0135757694244383, + 2.0145153722381592, + 2.0109932864379885, + 2.013635636329651, + 2.0102212376403807, + 2.0110452947616575, + 2.011999717025757, + 2.0092155500030517, + 2.0120949505615235, + 2.010387943572998, + 2.010760363922119, + 2.0082851540374755, + 2.010158937225342, + 2.0103255702209473, + 2.0088272121429442, + 2.0083791939544677, + 2.009366819381714, + 2.0072296501922606, + 2.0084847410583495, + 2.0094225827789307, + 2.00697819190979, + 2.0068142838287355, + 2.0079344313049314, + 2.0090717527008057, + 2.0060496669769288, + 2.008173867645264, + 2.0086808282089232, + 2.0082889751434325, + 2.004431292037964, + 2.006781820449829, + 2.0053993869018556, + 2.0058481903457643, + 2.006508606338501, + 2.0069684927368163, + 2.0063695631408693, + 2.008684331817627, + 2.0065671909332274, + 2.005968345184326, + 2.0088549881744386, + 2.0075259527969362, + 2.0050431396484374, + 2.0057015911102294, + 2.0043167867279053, + 2.005799201889038, + 2.0063393185043337, + 2.0052655740737917, + 2.0044282928848265, + 2.003640436248779, + 2.005821014328003, + 2.003054510650635, + 2.0039722763824463, + 2.004264993286133, + 2.004719021987915, + 2.004308237876892, + 2.004206368179321, + 2.003580096473694, + 2.0051046691131593, + 2.0024837198638914, + 2.004619668197632, + 2.0042623318481447, + 2.0048826543426514, + 2.003167839813232, + 2.0047615603637694, + 2.005793152923584, + 2.004860108909607, + 2.0029871907043457, + 2.0043380725097655, + 2.00175498626709 + ], + "train_acc": [ + 0.23242, + 0.23972, + 0.2412, + 0.24256, + 0.24318, + 0.2452, + 0.24798, + 0.24904, + 0.25184, + 0.25274, + 0.25106, + 0.25332, + 0.25176, + 0.25698, + 0.2543, + 0.25926, + 0.25692, + 0.25934, + 0.25892, + 0.26004, + 0.26144, + 0.26174, + 0.26074, + 0.26036, + 0.26248, + 0.2653, + 0.26038, + 0.2655, + 0.26348, + 0.26562, + 0.26624, + 0.2666, + 0.2666, + 0.26524, + 0.2674, + 0.2642, + 0.266, + 0.26908, + 0.26762, + 0.26734, + 0.26894, + 0.26826, + 0.26892, + 0.27234, + 0.26856, + 0.26836, + 0.2683, + 0.26972, + 0.2684, + 0.27156, + 0.26852, + 0.27026, + 0.27, + 0.27166, + 0.26992, + 0.27072, + 0.27186, + 0.27046, + 0.26968, + 0.27074, + 0.27308, + 0.27064, + 0.27174, + 0.26952, + 0.27196, + 0.27196, + 0.27176, + 0.27202, + 0.26996, + 0.27208, + 0.26994, + 0.27192, + 0.27492, + 0.27146, + 0.272, + 0.2721, + 0.27436, + 0.27366, + 0.27262, + 0.2733, + 0.27358, + 0.27316, + 0.27382, + 0.27264, + 0.274, + 0.27248, + 0.2722, + 0.2718, + 0.27186, + 0.27288, + 0.27412, + 0.27164, + 0.27014, + 0.27418, + 0.2732, + 0.27242, + 0.2715, + 0.27234, + 0.27324, + 0.27442 + ], + "test_acc": [ + 0.2663, + 0.2538, + 0.2393, + 0.2585, + 0.2479, + 0.2622, + 0.2687, + 0.2675, + 0.2687, + 0.2625, + 0.2707, + 0.2776, + 0.2781, + 0.2768, + 0.2674, + 0.2767, + 0.2932, + 0.2846, + 0.2656, + 0.277, + 0.2843, + 0.2856, + 0.2648, + 0.2894, + 0.2777, + 0.2864, + 0.2809, + 0.2937, + 0.2826, + 0.2832, + 0.2769, + 0.2874, + 0.2864, + 0.2842, + 0.295, + 0.2794, + 0.2872, + 0.2969, + 0.2732, + 0.29, + 0.2922, + 0.2994, + 0.2883, + 0.2934, + 0.2842, + 0.29, + 0.299, + 0.2933, + 0.2918, + 0.288, + 0.2901, + 0.2966, + 0.2922, + 0.2894, + 0.2872, + 0.292, + 0.2925, + 0.2937, + 0.2959, + 0.2909, + 0.2932, + 0.2954, + 0.2919, + 0.2989, + 0.2933, + 0.2987, + 0.2977, + 0.2861, + 0.2912, + 0.2878, + 0.2991, + 0.2888, + 0.2907, + 0.2875, + 0.2937, + 0.2968, + 0.2893, + 0.2961, + 0.2933, + 0.2928, + 0.2911, + 0.2945, + 0.2983, + 0.2959, + 0.2963, + 0.296, + 0.293, + 0.295, + 0.2947, + 0.2959, + 0.2972, + 0.2934, + 0.296, + 0.2957, + 0.2949, + 0.2955, + 0.2957, + 0.2956, + 0.2956, + 0.2956 + ] + }, + "diagnostics": { + "bp_cosine": [ + 0.38264644145965576, + 0.0008421496022492647, + -0.000935245247092098, + 0.0012684656539931893, + -7.955901673994958e-05, + 4.1166742448695004e-05, + -0.0003361787530593574, + -0.000117168077849783, + 0.00040733604691922665, + 0.0004055476747453213, + 0.00027774946647696197, + -0.00022715271916240454 + ], + "perturbation_rho": [ + 0.02277328446507454, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "nudging": { + "0.001": [ + -2.998858690261841e-07, + 0.0, + -1.862645149230957e-09, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "0.003": [ + -9.373761713504791e-07, + 0.0, + -1.862645149230957e-09, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "0.01": [ + -2.9550865292549133e-06, + -3.725290298461914e-09, + -1.862645149230957e-09, + 9.313225746154785e-10, + 0.0, + 0.0, + -5.587935447692871e-09, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ] + }, + "hidden_norms_per_layer": [ + 40328.7421875, + 215651328.0, + 1357730560.0, + 1379884160.0, + 1907440512.0, + 1914356608.0, + 2525564416.0, + 3004119296.0, + 3396676352.0, + 3411036160.0, + 3436140800.0, + 3741719808.0, + 3836453888.0 + ], + "bp_grad_norms_per_layer": [ + 3.007613997851877e-07, + 3.797166470143054e-10, + 3.725074693150532e-10, + 3.724560382334374e-10, + 3.726816910631925e-10, + 3.727783637330617e-10, + 3.7286917997647606e-10, + 3.728354847076787e-10, + 3.7310107781074464e-10, + 3.731098485726392e-10, + 3.7311384537552783e-10, + 3.7324987545162003e-10, + 3.732594233696318e-10 + ] + }, + "drift": { + "embed.weight": 356.12528180806567, + "embed.bias": 271.7696079359235, + "blocks.0.ln.weight": 12.160896301269531, + "blocks.0.w1.weight": 213.2156736743355, + "blocks.0.w1.bias": 188.46956990883064, + "blocks.0.w2.weight": 511.5389504411083, + "blocks.1.ln.weight": 11.089088439941406, + "blocks.1.w1.weight": 313.55891993641393, + "blocks.1.w1.bias": 313.7326373058554, + "blocks.1.w2.weight": 494.64000104284804, + "blocks.2.ln.weight": 7.991178512573242, + "blocks.2.w1.weight": 192.53270642077123, + "blocks.2.w1.bias": 179.27909905821068, + "blocks.2.w2.weight": 281.61858273592986, + "blocks.3.ln.weight": 11.628951072692871, + "blocks.3.w1.weight": 320.27019212444, + "blocks.3.w1.bias": 308.4511088686199, + "blocks.3.w2.weight": 439.5098655875687, + "blocks.4.ln.weight": 7.837525844573975, + "blocks.4.w1.weight": 198.99888492277404, + "blocks.4.w1.bias": 188.93588246764426, + "blocks.4.w2.weight": 279.38242336071187, + "blocks.5.ln.weight": 11.894773483276367, + "blocks.5.w1.weight": 335.3636162468368, + "blocks.5.w1.bias": 316.6947924070723, + "blocks.5.w2.weight": 491.62782621790865, + "blocks.6.ln.weight": 13.031082153320312, + "blocks.6.w1.weight": 372.496461957569, + "blocks.6.w1.bias": 361.37412651682865, + "blocks.6.w2.weight": 534.5811734194972, + "blocks.7.ln.weight": 11.872922897338867, + "blocks.7.w1.weight": 332.30066015200464, + "blocks.7.w1.bias": 318.8149983218427, + "blocks.7.w2.weight": 467.51464843487275, + "blocks.8.ln.weight": 8.978517532348633, + "blocks.8.w1.weight": 249.40733753718652, + "blocks.8.w1.bias": 227.90248095685143, + "blocks.8.w2.weight": 340.61733231825633, + "blocks.9.ln.weight": 8.258359909057617, + "blocks.9.w1.weight": 227.6803182887842, + "blocks.9.w1.bias": 215.0496950812678, + "blocks.9.w2.weight": 304.7891443514947, + "blocks.10.ln.weight": 11.31588363647461, + "blocks.10.w1.weight": 320.63785926119624, + "blocks.10.w1.bias": 285.28965414218027, + "blocks.10.w2.weight": 449.4781041979696, + "blocks.11.ln.weight": 8.833503723144531, + "blocks.11.w1.weight": 246.89372906458416, + "blocks.11.w1.bias": 234.18032268874182, + "blocks.11.w2.weight": 346.3219491734876, + "out_ln.weight": 0.7750099301338196, + "out_head.weight": 8.051922392430235, + "out_head.bias": 0.5259713382162419 + } + }, + "fa": { + "log": { + "train_loss": [ + 2.090004820022583, + 2.0399895847320555, + 2.016654765472412, + 1.993640735435486, + 1.970404408721924, + 1.9607704742050172, + 1.9497378304672242, + 1.945601725769043, + 1.9354119133758545, + 1.926653540725708, + 1.919797864074707, + 1.9190785704040527, + 1.921803589859009, + 1.9146928586196899, + 1.9141384088134765, + 1.911525807800293, + 1.9086668190765381, + 1.9076120470809936, + 1.9019774869155883, + 1.8993370356750487, + 1.8970520935440063, + 1.8962269541931152, + 1.8934082318496703, + 1.8856604388809204, + 1.8881552615356445, + 1.8857498220062257, + 1.8820669253158568, + 1.882652628211975, + 1.8782193230819701, + 1.8737722314453125, + 1.870555029067993, + 1.870324335861206, + 1.8657306994247436, + 1.866270491065979, + 1.8603372681427002, + 1.8619992153930665, + 1.8532835510253907, + 1.8544231454086304, + 1.8503647320175172, + 1.848538630065918, + 1.8463982132720946, + 1.844799047012329, + 1.843893060836792, + 1.8374801321411134, + 1.8375832195281983, + 1.836339090499878, + 1.8296538625335694, + 1.8266508029174804, + 1.8317974203109741, + 1.8261817306518555, + 1.8269930626678468, + 1.8243385958480836, + 1.8216499743270873, + 1.8209158150482179, + 1.8218781385040284, + 1.8225933773040772, + 1.8176012789916993, + 1.8158467154312135, + 1.814079772338867, + 1.8132556122207641, + 1.8133179102325439, + 1.8107317385101318, + 1.8110995058822632, + 1.809731153831482, + 1.8064914291381835, + 1.8078764828109741, + 1.8058526267242432, + 1.8063254193496705, + 1.808208419113159, + 1.801958323020935, + 1.805595525779724, + 1.8042979398345946, + 1.7999617199325562, + 1.8028947134017945, + 1.800519053878784, + 1.7981389894866944, + 1.7996525979614257, + 1.7991148620224, + 1.7987511922836303, + 1.7935400792694092, + 1.7988948291397096, + 1.7971973416900635, + 1.7980579327392578, + 1.798070336303711, + 1.7963832228088379, + 1.7970877619171142, + 1.795718716392517, + 1.794479237098694, + 1.7980125144195556, + 1.7947333910369874, + 1.798018039779663, + 1.7939606005096436, + 1.7963129815292358, + 1.7946422113418579, + 1.79368143699646, + 1.7950917078399657, + 1.794961125793457, + 1.7920723850250244, + 1.793914785118103, + 1.7935118531036376 + ], + "train_acc": [ + 0.22204, + 0.23944, + 0.25174, + 0.26568, + 0.2792, + 0.28158, + 0.29006, + 0.29422, + 0.29792, + 0.3005, + 0.30396, + 0.3033, + 0.30092, + 0.30788, + 0.30678, + 0.30974, + 0.31076, + 0.3128, + 0.31236, + 0.31404, + 0.31956, + 0.31576, + 0.31692, + 0.32034, + 0.31844, + 0.32288, + 0.32082, + 0.32326, + 0.3258, + 0.32692, + 0.32782, + 0.32868, + 0.32982, + 0.32992, + 0.33264, + 0.33086, + 0.33356, + 0.33424, + 0.33596, + 0.33532, + 0.33676, + 0.33762, + 0.33838, + 0.34134, + 0.34054, + 0.34094, + 0.34518, + 0.34504, + 0.34382, + 0.34594, + 0.34606, + 0.3472, + 0.34726, + 0.3491, + 0.34792, + 0.34796, + 0.35154, + 0.34984, + 0.34972, + 0.34986, + 0.35286, + 0.35258, + 0.3507, + 0.35242, + 0.35492, + 0.35352, + 0.35304, + 0.35276, + 0.35448, + 0.35806, + 0.3536, + 0.35518, + 0.35724, + 0.3548, + 0.35754, + 0.3575, + 0.3584, + 0.35666, + 0.358, + 0.35686, + 0.35654, + 0.35748, + 0.3573, + 0.3578, + 0.3589, + 0.35928, + 0.35822, + 0.35924, + 0.3585, + 0.3598, + 0.35814, + 0.36028, + 0.3576, + 0.3603, + 0.35812, + 0.36048, + 0.3598, + 0.35942, + 0.361, + 0.35982 + ], + "test_acc": [ + 0.2317, + 0.2482, + 0.2524, + 0.2817, + 0.2906, + 0.3053, + 0.3168, + 0.3229, + 0.3333, + 0.3223, + 0.3265, + 0.3361, + 0.3348, + 0.3282, + 0.3316, + 0.3402, + 0.3496, + 0.3424, + 0.3301, + 0.3303, + 0.3445, + 0.3441, + 0.3309, + 0.3506, + 0.3379, + 0.3507, + 0.351, + 0.3592, + 0.3551, + 0.3504, + 0.3446, + 0.3557, + 0.3632, + 0.3475, + 0.3565, + 0.3584, + 0.3592, + 0.3606, + 0.3493, + 0.3691, + 0.3623, + 0.3675, + 0.371, + 0.3667, + 0.3641, + 0.3695, + 0.3741, + 0.3705, + 0.3696, + 0.3682, + 0.3803, + 0.3754, + 0.3701, + 0.3767, + 0.3715, + 0.3816, + 0.3792, + 0.3848, + 0.3825, + 0.3768, + 0.3823, + 0.3775, + 0.3814, + 0.3805, + 0.3795, + 0.3823, + 0.3807, + 0.3802, + 0.3818, + 0.3826, + 0.3834, + 0.3871, + 0.3823, + 0.379, + 0.3835, + 0.3823, + 0.3841, + 0.3867, + 0.3848, + 0.3844, + 0.3849, + 0.3854, + 0.385, + 0.3827, + 0.3843, + 0.3859, + 0.3865, + 0.385, + 0.3868, + 0.3874, + 0.3882, + 0.3865, + 0.3878, + 0.3893, + 0.3869, + 0.3872, + 0.3872, + 0.387, + 0.3868, + 0.3867 + ] + }, + "diagnostics": { + "bp_cosine": [ + 0.005462596658617258, + 0.04850056767463684, + 0.013480322435498238, + 0.017950695008039474, + -0.06765332818031311, + -0.0009795809164643288, + -0.01368715800344944, + 0.008839694783091545, + -0.02521103248000145, + 0.02215195819735527, + 0.024440081790089607, + 0.9901714324951172 + ], + "perturbation_rho": [ + -0.018134091049432755, + -0.015751726925373077, + 0.00351056270301342, + -0.00837995670735836, + -0.011277807876467705, + 0.028536146506667137, + -0.04167690500617027, + -0.03265586867928505, + 0.04896393418312073, + 0.020356643944978714, + 0.027929214760661125, + -0.0027437973767518997 + ], + "nudging": { + "0.001": [ + 1.5739351511001587e-07, + -1.4295801520347595e-07, + 2.0023435354232788e-08, + -3.4924596548080444e-08, + 8.055940270423889e-08, + -1.6298145055770874e-08, + -3.585591912269592e-08, + -1.5832483768463135e-08, + -3.725290298461914e-08, + -3.073364496231079e-08, + -8.521601557731628e-08, + -9.82079654932022e-07 + ], + "0.003": [ + 4.880130290985107e-07, + -3.8463622331619263e-07, + -6.05359673500061e-08, + -1.1082738637924194e-07, + 2.5797635316848755e-07, + -7.776543498039246e-08, + 5.8673322200775146e-08, + -3.67872416973114e-08, + 3.771856427192688e-08, + -1.1362135410308838e-07, + -1.0477378964424133e-07, + -3.4580007195472717e-06 + ], + "0.01": [ + 1.5916302800178528e-06, + -1.2102536857128143e-06, + -2.3888424038887024e-07, + -3.0547380447387695e-07, + 9.206123650074005e-07, + -1.8766149878501892e-07, + 1.1455267667770386e-07, + -8.242204785346985e-08, + 2.086162567138672e-07, + -3.003515303134918e-07, + -3.511086106300354e-07, + -1.198984682559967e-05 + ] + }, + "hidden_norms_per_layer": [ + 8903.6103515625, + 53416.37109375, + 323086.75, + 543887.1875, + 720660.5625, + 852344.25, + 912922.875, + 915610.5, + 918604.6875, + 933013.8125, + 945300.25, + 944588.0625, + 786758.5625 + ], + "bp_grad_norms_per_layer": [ + 1.4760601516172756e-05, + 1.4644468819824397e-06, + 7.143049174374028e-07, + 6.862116492811765e-07, + 6.922383022356371e-07, + 6.88848786012386e-07, + 6.914569894433953e-07, + 6.923283422111126e-07, + 6.937441980880976e-07, + 6.82936615703511e-07, + 6.813309028075309e-07, + 6.779434329473588e-07, + 6.447000373555056e-07 + ] + }, + "drift": { + "embed.weight": 84.35971307415942, + "embed.bias": 32.19502954792936, + "blocks.0.ln.weight": 1.6290949583053589, + "blocks.0.w1.weight": 18.83059829156805, + "blocks.0.w1.bias": 11.893972638627677, + "blocks.0.w2.weight": 85.79388547124405, + "blocks.1.ln.weight": 1.249456763267517, + "blocks.1.w1.weight": 22.541099264872408, + "blocks.1.w1.bias": 19.900474950549036, + "blocks.1.w2.weight": 60.379082703434506, + "blocks.2.ln.weight": 0.8102703094482422, + "blocks.2.w1.weight": 21.89597288552171, + "blocks.2.w1.bias": 23.382511495918294, + "blocks.2.w2.weight": 50.297376870531075, + "blocks.3.ln.weight": 0.8057949542999268, + "blocks.3.w1.weight": 22.204088550957646, + "blocks.3.w1.bias": 23.964618513708253, + "blocks.3.w2.weight": 54.618428441768906, + "blocks.4.ln.weight": 0.739398181438446, + "blocks.4.w1.weight": 21.57539189625426, + "blocks.4.w1.bias": 22.672842875453526, + "blocks.4.w2.weight": 42.39506645504441, + "blocks.5.ln.weight": 0.7785037159919739, + "blocks.5.w1.weight": 21.199615780366805, + "blocks.5.w1.bias": 21.79400267554791, + "blocks.5.w2.weight": 45.035435661106206, + "blocks.6.ln.weight": 0.6276997923851013, + "blocks.6.w1.weight": 14.78204467257611, + "blocks.6.w1.bias": 13.651015110446988, + "blocks.6.w2.weight": 47.906552978017096, + "blocks.7.ln.weight": 0.6718869209289551, + "blocks.7.w1.weight": 15.301077297758928, + "blocks.7.w1.bias": 11.14864058748224, + "blocks.7.w2.weight": 81.22443944293431, + "blocks.8.ln.weight": 0.6778666377067566, + "blocks.8.w1.weight": 15.225997184691419, + "blocks.8.w1.bias": 12.413017961913502, + "blocks.8.w2.weight": 70.26476379892472, + "blocks.9.ln.weight": 0.689741849899292, + "blocks.9.w1.weight": 14.934806218542306, + "blocks.9.w1.bias": 13.186621451322553, + "blocks.9.w2.weight": 66.88878955298375, + "blocks.10.ln.weight": 0.6384391188621521, + "blocks.10.w1.weight": 14.869272775568358, + "blocks.10.w1.bias": 11.801783819091257, + "blocks.10.w2.weight": 74.67053466138057, + "blocks.11.ln.weight": 0.7496767044067383, + "blocks.11.w1.weight": 17.612627240302658, + "blocks.11.w1.bias": 15.909240129776858, + "blocks.11.w2.weight": 90.63484324360367, + "out_ln.weight": 0.389094740152359, + "out_head.weight": 5.407193127809483, + "out_head.bias": 0.7008348223133053 + } + } + }, + "config": { + "dataset": "cifar10", + "d_hidden": 256, + "num_blocks": 12, + "batch_size": 128, + "epochs": 100, + "lr": 0.001, + "lr_fb": 0.001, + "wd": 0.01, + "lam": 0.1, + "K": 4, + "sigma_bridge": 0.05, + "ema_momentum": 0.995, + "term_grad_weight": 1.0, + "seeds": [ + 0 + ], + "gpu": 0, + "output_dir": "results/fa_dfa_d256_L12_seed0", + "methods": [ + "fa", + "dfa" + ], + "random_targets": false, + "penalty_lam": 0.0, + "num_classes": 10 + } +}
\ No newline at end of file |