diff options
Diffstat (limited to 'results/fa_dfa_d512_L12_seed1/results_cifar10.json')
| -rw-r--r-- | results/fa_dfa_d512_L12_seed1/results_cifar10.json | 969 |
1 files changed, 969 insertions, 0 deletions
diff --git a/results/fa_dfa_d512_L12_seed1/results_cifar10.json b/results/fa_dfa_d512_L12_seed1/results_cifar10.json new file mode 100644 index 0000000..e317465 --- /dev/null +++ b/results/fa_dfa_d512_L12_seed1/results_cifar10.json @@ -0,0 +1,969 @@ +{ + "1": { + "dfa": { + "log": { + "train_loss": [ + 2.08784130317688, + 2.052014544372559, + 2.045669429321289, + 2.0424993614196776, + 2.0406430601501464, + 2.038676989364624, + 2.0339627771759035, + 2.0320363275527953, + 2.0293006126403808, + 2.027781917953491, + 2.0250831903839113, + 2.0254856860351564, + 2.025074804534912, + 2.020526984100342, + 2.017961084213257, + 2.0189661613464356, + 2.0154323001098633, + 2.012715454673767, + 2.0142161911010743, + 2.0169401965713503, + 2.015820147628784, + 2.0139186489105225, + 2.0170457207489014, + 2.010938446311951, + 2.0119680180358888, + 2.011925876312256, + 2.0143484900665283, + 2.011023857269287, + 2.011966758041382, + 2.011277076072693, + 2.0121180039978026, + 2.011552553100586, + 2.0112545052337647, + 2.0096049158477785, + 2.0116290228271483, + 2.012332228317261, + 2.0100498531341553, + 2.013163764266968, + 2.0090835204696655, + 2.0087667613983156, + 2.008153441734314, + 2.008515095252991, + 2.0076544715118407, + 2.006217385635376, + 2.0079916049957274, + 2.0093279348754884, + 2.0085637417602538, + 2.0080473442077635, + 2.005413489227295, + 2.006327581214905, + 2.0087718325042725, + 2.005806240501404, + 2.0063090213012695, + 2.005670380554199, + 2.008137595367432, + 2.0058074869537355, + 2.00653946685791, + 2.008738963394165, + 2.005000798873901, + 2.005233299484253, + 2.0051020904159547, + 2.0054547385406494, + 2.006043081893921, + 2.0066007862091064, + 2.0075411237335206, + 2.004597886505127, + 2.004815984649658, + 2.0049288347625733, + 2.004783443069458, + 2.006772222366333, + 2.0067188888549805, + 2.005615498046875, + 2.0044576259613036, + 2.006375897064209, + 2.0036719734954835, + 2.0061926458740236, + 2.004457794265747, + 2.0043624324035645, + 2.00480925743103, + 2.0047112017822264, + 2.0034663817977907, + 2.002827211380005, + 2.003827619934082, + 2.0031036682891847, + 2.0055768743133546, + 2.002000348739624, + 2.0022059215545656, + 2.0022215761566162, + 2.0029819064331056, + 2.0014546299743654, + 2.00309593082428, + 2.0011430258178713, + 2.004017787246704, + 2.001931449203491, + 2.000619831314087, + 2.0033977560043335, + 2.0014762798690797, + 2.0025222287368774, + 2.003771315956116, + 2.0038832190704348 + ], + "train_acc": [ + 0.22474, + 0.23978, + 0.242, + 0.24386, + 0.2415, + 0.24352, + 0.24826, + 0.2474, + 0.25112, + 0.24912, + 0.25434, + 0.25094, + 0.25204, + 0.25484, + 0.2583, + 0.25824, + 0.2601, + 0.26048, + 0.261, + 0.25998, + 0.2624, + 0.2619, + 0.25918, + 0.26472, + 0.2644, + 0.26346, + 0.2648, + 0.26354, + 0.26368, + 0.26704, + 0.26128, + 0.26468, + 0.26524, + 0.26586, + 0.26558, + 0.26324, + 0.26776, + 0.2652, + 0.26808, + 0.26752, + 0.26614, + 0.26558, + 0.2692, + 0.27084, + 0.27106, + 0.26932, + 0.2685, + 0.26908, + 0.26978, + 0.27274, + 0.26924, + 0.27318, + 0.27128, + 0.27216, + 0.27004, + 0.27288, + 0.26872, + 0.27234, + 0.27316, + 0.27282, + 0.27218, + 0.2717, + 0.27198, + 0.27498, + 0.27174, + 0.27282, + 0.27204, + 0.27394, + 0.27538, + 0.26966, + 0.27104, + 0.27454, + 0.27422, + 0.27358, + 0.27468, + 0.27304, + 0.27244, + 0.27386, + 0.27408, + 0.27234, + 0.27666, + 0.2753, + 0.2739, + 0.27624, + 0.27434, + 0.27818, + 0.27466, + 0.27586, + 0.27504, + 0.27554, + 0.27526, + 0.27624, + 0.27486, + 0.27606, + 0.27568, + 0.27474, + 0.27578, + 0.27652, + 0.27508, + 0.27436 + ], + "test_acc": [ + 0.2444, + 0.2437, + 0.2469, + 0.2484, + 0.2807, + 0.266, + 0.2731, + 0.2735, + 0.2783, + 0.2609, + 0.2648, + 0.2716, + 0.2541, + 0.2846, + 0.2739, + 0.2748, + 0.28, + 0.2759, + 0.2345, + 0.2676, + 0.271, + 0.2659, + 0.2785, + 0.2728, + 0.2906, + 0.269, + 0.2846, + 0.2651, + 0.2937, + 0.2971, + 0.2925, + 0.2807, + 0.2778, + 0.2837, + 0.2793, + 0.292, + 0.2866, + 0.2911, + 0.2915, + 0.2833, + 0.2907, + 0.2971, + 0.2829, + 0.2891, + 0.2955, + 0.2991, + 0.295, + 0.2912, + 0.2697, + 0.2874, + 0.2741, + 0.2949, + 0.2836, + 0.2944, + 0.2934, + 0.2975, + 0.2873, + 0.2882, + 0.2889, + 0.2901, + 0.2789, + 0.2908, + 0.2953, + 0.2911, + 0.2987, + 0.2861, + 0.291, + 0.2942, + 0.2938, + 0.301, + 0.2972, + 0.2922, + 0.2872, + 0.2859, + 0.298, + 0.2942, + 0.2918, + 0.2964, + 0.2973, + 0.2944, + 0.2903, + 0.2885, + 0.2969, + 0.2897, + 0.291, + 0.294, + 0.2903, + 0.2931, + 0.2894, + 0.293, + 0.2936, + 0.2923, + 0.291, + 0.291, + 0.2933, + 0.293, + 0.2935, + 0.2933, + 0.2932, + 0.2932 + ] + }, + "diagnostics": { + "bp_cosine": [ + 0.3784182071685791, + 0.00024469412164762616, + -6.5529711719136685e-06, + -2.4012413632590324e-05, + -2.2034288122085854e-05, + 0.0002968982153106481, + -0.0007156325737014413, + 0.0002482909185346216, + -0.00023413923918269575, + -0.000343983992934227, + -0.0004118037468288094, + -0.0001832617272157222 + ], + "perturbation_rho": [ + 0.02268090471625328, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "nudging": { + "0.001": [ + -2.682209014892578e-07, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "0.003": [ + -8.391216397285461e-07, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 9.313225746154785e-10, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "0.01": [ + -3.262888640165329e-06, + -2.7939677238464355e-09, + 0.0, + 0.0, + 9.313225746154785e-10, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ] + }, + "hidden_norms_per_layer": [ + 58511.79296875, + 1607470848.0, + 3539406336.0, + 4483452928.0, + 5316571136.0, + 6553798144.0, + 7388260352.0, + 9242703872.0, + 9554661376.0, + 11009835008.0, + 11375313920.0, + 12110455808.0, + 12125570048.0 + ], + "bp_grad_norms_per_layer": [ + 2.394743603417737e-07, + 1.8629858766772145e-10, + 1.8620747999076315e-10, + 1.8607929086478237e-10, + 1.8619832065081e-10, + 1.8618356856237028e-10, + 1.862058007784384e-10, + 1.8604620621864854e-10, + 1.860521597896181e-10, + 1.860489817762101e-10, + 1.860557541366603e-10, + 1.860564341482629e-10, + 1.8607769491918447e-10 + ] + }, + "drift": { + "embed.weight": 354.85094069636483, + "embed.bias": 257.5504888266541, + "blocks.0.ln.weight": 10.020600874142511, + "blocks.0.w1.weight": 311.43953558716794, + "blocks.0.w1.bias": 277.95760174306895, + "blocks.0.w2.weight": 495.0366657957393, + "blocks.1.ln.weight": 9.233680086701126, + "blocks.1.w1.weight": 352.291334675127, + "blocks.1.w1.bias": 326.78546394466474, + "blocks.1.w2.weight": 328.5686664739385, + "blocks.2.ln.weight": 8.349584849320378, + "blocks.2.w1.weight": 347.4686920651646, + "blocks.2.w1.bias": 314.54485739427827, + "blocks.2.w2.weight": 322.05654919163993, + "blocks.3.ln.weight": 8.748114666066721, + "blocks.3.w1.weight": 341.6411014867908, + "blocks.3.w1.bias": 316.14847884372415, + "blocks.3.w2.weight": 332.52492325797556, + "blocks.4.ln.weight": 9.48080425966404, + "blocks.4.w1.weight": 385.71563399378635, + "blocks.4.w1.bias": 355.1208489434028, + "blocks.4.w2.weight": 353.2677244575998, + "blocks.5.ln.weight": 8.852085211151078, + "blocks.5.w1.weight": 359.24445324084536, + "blocks.5.w1.bias": 327.2053984662166, + "blocks.5.w2.weight": 334.59987322510256, + "blocks.6.ln.weight": 11.348583355888922, + "blocks.6.w1.weight": 445.597521308562, + "blocks.6.w1.bias": 417.10230766347513, + "blocks.6.w2.weight": 410.28435436855875, + "blocks.7.ln.weight": 8.64827114045918, + "blocks.7.w1.weight": 341.9936697385893, + "blocks.7.w1.bias": 328.17910031845435, + "blocks.7.w2.weight": 322.2651678366388, + "blocks.8.ln.weight": 10.71699832162817, + "blocks.8.w1.weight": 430.40137342022086, + "blocks.8.w1.bias": 395.45655885652957, + "blocks.8.w2.weight": 418.09527663040274, + "blocks.9.ln.weight": 7.689168672786446, + "blocks.9.w1.weight": 307.285121525094, + "blocks.9.w1.bias": 286.6467639719644, + "blocks.9.w2.weight": 284.90185966096766, + "blocks.10.ln.weight": 9.362189538932856, + "blocks.10.w1.weight": 374.1142646081974, + "blocks.10.w1.bias": 340.73445175290055, + "blocks.10.w2.weight": 334.3762368608616, + "blocks.11.ln.weight": 6.971496170157957, + "blocks.11.w1.weight": 273.28051237448034, + "blocks.11.w1.bias": 249.7041416371339, + "blocks.11.w2.weight": 250.47506193148064, + "out_ln.weight": 0.6766551046620063, + "out_head.weight": 9.660594953000444, + "out_head.bias": 0.6081606213236193 + } + }, + "fa": { + "log": { + "train_loss": [ + 2.039813977584839, + 1.9617359282684326, + 1.9338554121398925, + 1.920875029220581, + 1.9130360520172118, + 1.9065295135116578, + 1.8998144156646728, + 1.893558436279297, + 1.8832812244033814, + 1.8782934536361695, + 1.8706854000091553, + 1.8697611987304688, + 1.8661894456481933, + 1.8597797629165649, + 1.8567528272247313, + 1.8511868425750733, + 1.8451919631195068, + 1.8397227505493163, + 1.834096513671875, + 1.8383717123413086, + 1.8340178047943114, + 1.828574577331543, + 1.8276408141326905, + 1.816076180152893, + 1.8154300131225587, + 1.8111229518890382, + 1.8086885763549805, + 1.8008410723114014, + 1.797977260093689, + 1.791234312095642, + 1.790951509628296, + 1.783079252281189, + 1.779609817199707, + 1.7747913509750366, + 1.7767339920043945, + 1.7752550736236572, + 1.7656426385879516, + 1.766551993637085, + 1.763684893760681, + 1.7585593047332764, + 1.7559137664413451, + 1.7532807390975953, + 1.75251891330719, + 1.749091358680725, + 1.7431825846099853, + 1.745732957458496, + 1.744267059288025, + 1.7419162277603149, + 1.738112122116089, + 1.7364409768676758, + 1.7408631985473633, + 1.7369310165405274, + 1.7362467727279662, + 1.7374004998016357, + 1.736051294517517, + 1.734863005027771, + 1.7319254892730713, + 1.7279368935775756, + 1.728024648475647, + 1.72793106716156, + 1.726857622718811, + 1.7304030743408203, + 1.725222360267639, + 1.7216177057647706, + 1.7253669234466553, + 1.7201292371368408, + 1.7183555184173585, + 1.7207783557891845, + 1.7150025174713135, + 1.7195079833221436, + 1.718960595779419, + 1.7189737926483155, + 1.7145323584365846, + 1.7151868480682373, + 1.7150000100708007, + 1.7164085680389405, + 1.7131129906463622, + 1.7118962685775756, + 1.7095365018463136, + 1.7063884579086304, + 1.7068846228790284, + 1.7066734680175781, + 1.7093646648406982, + 1.7075168813323975, + 1.7101067148590088, + 1.7065433209228515, + 1.70673194480896, + 1.706905789451599, + 1.7064489783096313, + 1.7049209537124634, + 1.702039204940796, + 1.7093150380706787, + 1.7065809494018556, + 1.7064716823959352, + 1.7058918637466431, + 1.7017672801971435, + 1.7050170278549195, + 1.7040833249664307, + 1.7052951337432862, + 1.7028977381134034 + ], + "train_acc": [ + 0.2476, + 0.28516, + 0.30182, + 0.30518, + 0.30668, + 0.3081, + 0.30996, + 0.31082, + 0.31996, + 0.31962, + 0.32402, + 0.32084, + 0.32474, + 0.3262, + 0.33392, + 0.33128, + 0.33602, + 0.33814, + 0.34208, + 0.33954, + 0.34316, + 0.34226, + 0.34322, + 0.34824, + 0.35108, + 0.35332, + 0.35372, + 0.35524, + 0.35692, + 0.3583, + 0.3558, + 0.36152, + 0.36236, + 0.36616, + 0.36662, + 0.3641, + 0.36936, + 0.36966, + 0.36902, + 0.37004, + 0.3724, + 0.37158, + 0.37278, + 0.3725, + 0.375, + 0.37618, + 0.3764, + 0.3774, + 0.37752, + 0.37724, + 0.37826, + 0.37848, + 0.38024, + 0.3791, + 0.37902, + 0.38188, + 0.38038, + 0.38478, + 0.3799, + 0.38192, + 0.38352, + 0.38058, + 0.38324, + 0.38586, + 0.38004, + 0.38654, + 0.3842, + 0.38632, + 0.38732, + 0.38408, + 0.38422, + 0.38632, + 0.38902, + 0.38782, + 0.38674, + 0.38694, + 0.38626, + 0.38678, + 0.38948, + 0.39038, + 0.39018, + 0.38814, + 0.38618, + 0.3908, + 0.38982, + 0.39066, + 0.38904, + 0.38854, + 0.38872, + 0.39008, + 0.39052, + 0.38934, + 0.38852, + 0.39056, + 0.39032, + 0.3915, + 0.39092, + 0.39138, + 0.3895, + 0.39112 + ], + "test_acc": [ + 0.2816, + 0.3167, + 0.3194, + 0.3093, + 0.3417, + 0.3344, + 0.3297, + 0.3418, + 0.3479, + 0.3372, + 0.3432, + 0.3446, + 0.3458, + 0.3646, + 0.3616, + 0.3529, + 0.3617, + 0.3576, + 0.329, + 0.3542, + 0.3701, + 0.3647, + 0.3697, + 0.3697, + 0.3808, + 0.3735, + 0.3695, + 0.3735, + 0.3711, + 0.379, + 0.3877, + 0.3805, + 0.3751, + 0.3909, + 0.381, + 0.3909, + 0.3902, + 0.3862, + 0.3899, + 0.3879, + 0.3994, + 0.3963, + 0.3954, + 0.3882, + 0.3978, + 0.3919, + 0.3928, + 0.3909, + 0.3966, + 0.3986, + 0.3939, + 0.4016, + 0.3973, + 0.397, + 0.4019, + 0.4004, + 0.4018, + 0.4052, + 0.3951, + 0.4016, + 0.396, + 0.4024, + 0.4075, + 0.4051, + 0.4025, + 0.4037, + 0.4041, + 0.4061, + 0.4056, + 0.4041, + 0.4009, + 0.405, + 0.399, + 0.4012, + 0.4076, + 0.4079, + 0.4096, + 0.407, + 0.4049, + 0.4059, + 0.4026, + 0.406, + 0.4048, + 0.4027, + 0.407, + 0.4022, + 0.4067, + 0.4063, + 0.4062, + 0.4052, + 0.4059, + 0.4068, + 0.4063, + 0.4076, + 0.4063, + 0.4065, + 0.4062, + 0.4069, + 0.4069, + 0.4067 + ] + }, + "diagnostics": { + "bp_cosine": [ + 0.027296170592308044, + 0.04133676737546921, + 0.08515161275863647, + 0.026729261502623558, + 0.01583785191178322, + -0.025018498301506042, + -0.044713616371154785, + -0.05714074894785881, + -0.040867358446121216, + 0.012529742904007435, + -0.027012888342142105, + 0.9971734881401062 + ], + "perturbation_rho": [ + 0.053132910281419754, + -0.014070656150579453, + 0.02090507186949253, + 0.005164717324078083, + -0.031424831598997116, + -0.010921415872871876, + -0.005948010832071304, + 0.01678430661559105, + -0.00991929043084383, + -0.006591915152966976, + -0.010621514171361923, + -0.019635088741779327 + ], + "nudging": { + "0.001": [ + -2.4188775569200516e-06, + -2.6740599423646927e-07, + -1.7171259969472885e-07, + -1.0791700333356857e-07, + -3.4458935260772705e-08, + -4.493631422519684e-08, + 5.0407834351062775e-08, + 1.2456439435482025e-08, + 9.19681042432785e-09, + -1.6065314412117004e-08, + 2.444721758365631e-08, + -6.272457540035248e-07 + ], + "0.003": [ + -7.179100066423416e-06, + -6.825430318713188e-07, + -5.516340024769306e-07, + -1.1979136615991592e-07, + -2.8172507882118225e-08, + -1.9907020032405853e-08, + 1.2316741049289703e-07, + 7.264316082000732e-08, + 9.010545909404755e-08, + -2.7706846594810486e-08, + 9.778887033462524e-09, + -2.26777046918869e-06 + ], + "0.01": [ + -2.3640692234039307e-05, + -2.3529864847660065e-06, + -1.5455298125743866e-06, + -3.109453245997429e-07, + -1.5425030142068863e-07, + 1.0291114449501038e-07, + 3.3457763493061066e-07, + 3.76836396753788e-07, + 3.371387720108032e-07, + -1.2211967259645462e-07, + 1.5588011592626572e-07, + -8.008093573153019e-06 + ] + }, + "hidden_norms_per_layer": [ + 6886.94775390625, + 87584.2109375, + 126785.4921875, + 355667.78125, + 552057.5625, + 850422.75, + 1293051.0, + 1541609.25, + 1868753.125, + 2007669.375, + 2071472.125, + 2260452.75, + 1769309.25 + ], + "bp_grad_norms_per_layer": [ + 3.3799024095060304e-05, + 2.5621802706154995e-06, + 8.161263167494326e-07, + 4.5329542786021193e-07, + 4.0827075054039597e-07, + 4.0655893940311216e-07, + 4.057951343838795e-07, + 4.061944878230861e-07, + 4.055657427670667e-07, + 4.0522803601561463e-07, + 4.0508828647034534e-07, + 4.0517943489248864e-07, + 3.946254025777307e-07 + ] + }, + "drift": { + "embed.weight": 46.66262768106957, + "embed.bias": 12.349060830406593, + "blocks.0.ln.weight": 1.186887811746546, + "blocks.0.w1.weight": 16.691636363051998, + "blocks.0.w1.bias": 10.396716096626863, + "blocks.0.w2.weight": 53.63063338700071, + "blocks.1.ln.weight": 0.9868002811700606, + "blocks.1.w1.weight": 18.44033798724087, + "blocks.1.w1.bias": 4.605202402802353, + "blocks.1.w2.weight": 52.226844329919246, + "blocks.2.ln.weight": 1.0614278526441627, + "blocks.2.w1.weight": 19.34190693735112, + "blocks.2.w1.bias": 11.075717016343752, + "blocks.2.w2.weight": 42.37755333929534, + "blocks.3.ln.weight": 0.8798892347084698, + "blocks.3.w1.weight": 19.160031225604996, + "blocks.3.w1.bias": 13.88384021345869, + "blocks.3.w2.weight": 33.30996859649038, + "blocks.4.ln.weight": 0.7104088303381113, + "blocks.4.w1.weight": 18.209430962072435, + "blocks.4.w1.bias": 16.92336837763847, + "blocks.4.w2.weight": 28.747777526920554, + "blocks.5.ln.weight": 0.7207815914026496, + "blocks.5.w1.weight": 20.407344689317952, + "blocks.5.w1.bias": 19.928474035997958, + "blocks.5.w2.weight": 30.668259780333205, + "blocks.6.ln.weight": 0.6859569564344036, + "blocks.6.w1.weight": 20.174584059207987, + "blocks.6.w1.bias": 20.229323693427677, + "blocks.6.w2.weight": 30.149329891021953, + "blocks.7.ln.weight": 0.7233680592463317, + "blocks.7.w1.weight": 21.7410787215414, + "blocks.7.w1.bias": 22.40005573271207, + "blocks.7.w2.weight": 28.827371299185597, + "blocks.8.ln.weight": 0.6511450076677763, + "blocks.8.w1.weight": 17.427880699003644, + "blocks.8.w1.bias": 16.729590061303803, + "blocks.8.w2.weight": 28.2153738883236, + "blocks.9.ln.weight": 0.6038210923569808, + "blocks.9.w1.weight": 16.214819848855626, + "blocks.9.w1.bias": 15.839234916532192, + "blocks.9.w2.weight": 28.312285773435068, + "blocks.10.ln.weight": 0.6503320370929363, + "blocks.10.w1.weight": 19.037981824154166, + "blocks.10.w1.bias": 18.568708970692136, + "blocks.10.w2.weight": 20.586050451276414, + "blocks.11.ln.weight": 0.5703826672599819, + "blocks.11.w1.weight": 16.05034192522954, + "blocks.11.w1.bias": 14.75267875282651, + "blocks.11.w2.weight": 45.75154443737838, + "out_ln.weight": 0.36468260935864044, + "out_head.weight": 6.277188782444725, + "out_head.bias": 0.6248347589534139 + } + } + }, + "config": { + "dataset": "cifar10", + "d_hidden": 512, + "num_blocks": 12, + "batch_size": 128, + "epochs": 100, + "lr": 0.001, + "lr_fb": 0.001, + "wd": 0.01, + "lam": 0.1, + "K": 4, + "sigma_bridge": 0.05, + "ema_momentum": 0.995, + "term_grad_weight": 1.0, + "seeds": [ + 1 + ], + "gpu": 0, + "output_dir": "results/fa_dfa_d512_L12_seed1", + "methods": [ + "fa", + "dfa" + ], + "random_targets": false, + "penalty_lam": 0.0, + "num_classes": 10 + } +}
\ No newline at end of file |