diff options
Diffstat (limited to 'results/fa_dfa_d512_L12_seed4/results_cifar10.json')
| -rw-r--r-- | results/fa_dfa_d512_L12_seed4/results_cifar10.json | 969 |
1 files changed, 969 insertions, 0 deletions
diff --git a/results/fa_dfa_d512_L12_seed4/results_cifar10.json b/results/fa_dfa_d512_L12_seed4/results_cifar10.json new file mode 100644 index 0000000..7eb8f8a --- /dev/null +++ b/results/fa_dfa_d512_L12_seed4/results_cifar10.json @@ -0,0 +1,969 @@ +{ + "4": { + "dfa": { + "log": { + "train_loss": [ + 2.0759089875793455, + 2.049546967468262, + 2.050352345352173, + 2.0449130432891844, + 2.0451894146728518, + 2.0407823961639404, + 2.031551725692749, + 2.0308942835998534, + 2.027821557159424, + 2.0279336640930175, + 2.030788267745972, + 2.027367984313965, + 2.023764753379822, + 2.0222284102630614, + 2.021787865447998, + 2.0210360749053957, + 2.020778656768799, + 2.0178001490020754, + 2.015317197265625, + 2.0181816483306885, + 2.0157879919433594, + 2.0184039527893067, + 2.0140462054443358, + 2.0133035342025756, + 2.0117346116638184, + 2.0124337507629395, + 2.015013382720947, + 2.0131209099578857, + 2.0119356175994874, + 2.0105034595489504, + 2.0114617578125, + 2.010921164550781, + 2.0091999086761474, + 2.0087246141052244, + 2.008987555999756, + 2.007724672088623, + 2.0079352700424193, + 2.0069879000854494, + 2.0099592947769165, + 2.0069592917633057, + 2.0085809383773805, + 2.007678118972778, + 2.006203598976135, + 2.0054306941986084, + 2.004828847351074, + 2.0060369828796385, + 2.0055570992279055, + 2.0053103733825686, + 2.004637515335083, + 2.0055320530700684, + 2.0035772978973387, + 2.0045177731323243, + 2.0053057807922365, + 2.004057818374634, + 2.0017144575500487, + 2.0030729360580444, + 2.002487395324707, + 2.0019396072387696, + 1.9998790828704833, + 2.003716873703003, + 2.001451046066284, + 1.9996818887329102, + 2.0031208934783935, + 2.0009771130371092, + 2.0052372956848146, + 2.000408243637085, + 2.0017312159729004, + 2.00103297542572, + 2.0006809278106688, + 1.999061644744873, + 1.9998556817626953, + 2.000588871498108, + 2.0007700952148437, + 2.001544753036499, + 2.000170311355591, + 2.0008300536346435, + 2.00009318069458, + 2.0018883587646483, + 1.9953095329284667, + 1.9990402968597412, + 1.9993244941711426, + 1.9997524154281616, + 1.9993522864151, + 1.999408567199707, + 2.000375003089905, + 2.0002292515563966, + 1.99836263092041, + 1.999543589744568, + 1.9990913265609742, + 1.9996358081436156, + 1.9965649541854857, + 1.9988786743164062, + 2.000610784988403, + 1.9997782401275634, + 1.9977047104644776, + 1.998138542137146, + 1.9980609845733643, + 1.9994125312805175, + 1.9971481538391114, + 1.999670766143799 + ], + "train_acc": [ + 0.2298, + 0.24208, + 0.24002, + 0.24094, + 0.24586, + 0.24526, + 0.2481, + 0.24888, + 0.25388, + 0.25086, + 0.24904, + 0.25404, + 0.25272, + 0.2537, + 0.2557, + 0.25446, + 0.25566, + 0.25962, + 0.25888, + 0.25898, + 0.25988, + 0.25998, + 0.26148, + 0.2599, + 0.26144, + 0.2642, + 0.2616, + 0.2624, + 0.26138, + 0.26398, + 0.2628, + 0.26448, + 0.26602, + 0.2662, + 0.26392, + 0.26558, + 0.26618, + 0.26806, + 0.26572, + 0.26504, + 0.26606, + 0.26642, + 0.26868, + 0.26806, + 0.2681, + 0.26742, + 0.26938, + 0.26904, + 0.26912, + 0.26822, + 0.26928, + 0.26642, + 0.26932, + 0.26744, + 0.2692, + 0.27166, + 0.27004, + 0.27224, + 0.26922, + 0.26924, + 0.2704, + 0.2713, + 0.27188, + 0.27038, + 0.2696, + 0.27172, + 0.26974, + 0.27234, + 0.27158, + 0.27154, + 0.27218, + 0.27162, + 0.271, + 0.27146, + 0.27242, + 0.27194, + 0.27084, + 0.27166, + 0.2737, + 0.27286, + 0.27226, + 0.27186, + 0.27026, + 0.27378, + 0.27114, + 0.27044, + 0.27224, + 0.27336, + 0.27488, + 0.27168, + 0.27362, + 0.27224, + 0.27022, + 0.2718, + 0.27058, + 0.27258, + 0.27366, + 0.2716, + 0.2725, + 0.27256 + ], + "test_acc": [ + 0.26, + 0.261, + 0.2579, + 0.2572, + 0.2579, + 0.2482, + 0.2715, + 0.2746, + 0.2616, + 0.2747, + 0.2443, + 0.2845, + 0.2756, + 0.2732, + 0.274, + 0.2875, + 0.2614, + 0.2732, + 0.2699, + 0.2708, + 0.2874, + 0.277, + 0.2787, + 0.2872, + 0.2775, + 0.2813, + 0.2711, + 0.2819, + 0.2716, + 0.2875, + 0.2749, + 0.279, + 0.2977, + 0.282, + 0.2738, + 0.2876, + 0.2788, + 0.2943, + 0.2736, + 0.2936, + 0.2773, + 0.2777, + 0.2891, + 0.2962, + 0.2734, + 0.2955, + 0.2867, + 0.2864, + 0.2834, + 0.2845, + 0.2853, + 0.2937, + 0.2705, + 0.292, + 0.2952, + 0.2875, + 0.2771, + 0.2837, + 0.2843, + 0.2887, + 0.291, + 0.2917, + 0.2763, + 0.2862, + 0.2897, + 0.2876, + 0.2865, + 0.2799, + 0.2772, + 0.2898, + 0.2823, + 0.2833, + 0.2875, + 0.2866, + 0.2771, + 0.28, + 0.2827, + 0.2895, + 0.2925, + 0.2896, + 0.2889, + 0.2882, + 0.2886, + 0.2864, + 0.2873, + 0.2909, + 0.2878, + 0.2848, + 0.286, + 0.2834, + 0.287, + 0.2888, + 0.2892, + 0.2871, + 0.2873, + 0.2878, + 0.2872, + 0.2875, + 0.2877, + 0.2878 + ] + }, + "diagnostics": { + "bp_cosine": [ + 0.36940303444862366, + 0.00047992257168516517, + 0.00055807048920542, + -0.000561900029424578, + -0.00015003856969997287, + 0.0003543531056493521, + -0.000369079178199172, + -7.983684918144718e-05, + -0.0001259066048078239, + 0.0002819746732711792, + -2.340562059544027e-05, + -5.241552571533248e-05 + ], + "perturbation_rho": [ + -0.0012638717889785767, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "nudging": { + "0.001": [ + -2.9383227229118347e-07, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.1175870895385742e-08, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "0.003": [ + -1.0235235095024109e-06, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.1175870895385742e-08, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "0.01": [ + -3.4552067518234253e-06, + -8.381903171539307e-09, + 4.656612873077393e-10, + 9.313225746154785e-10, + -3.725290298461914e-09, + 0.0, + 1.1175870895385742e-08, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ] + }, + "hidden_norms_per_layer": [ + 56077.015625, + 1373153024.0, + 3371879936.0, + 5206262272.0, + 7341849088.0, + 8992748544.0, + 9065292800.0, + 9464700928.0, + 9539394560.0, + 10031855616.0, + 10931398656.0, + 11201608704.0, + 11850394624.0 + ], + "bp_grad_norms_per_layer": [ + 2.49191884904576e-07, + 1.9557305774853262e-10, + 1.9563668740563145e-10, + 1.9543285045831027e-10, + 1.9541313012183537e-10, + 1.9539742046603692e-10, + 1.9536555706523018e-10, + 1.953735784265831e-10, + 1.9537529927227126e-10, + 1.9540973006382245e-10, + 1.9539764251064184e-10, + 1.9554310948244336e-10, + 1.9556442576451616e-10 + ] + }, + "drift": { + "embed.weight": 341.4389143853609, + "embed.bias": 269.18455752628876, + "blocks.0.ln.weight": 10.301521425392954, + "blocks.0.w1.weight": 298.4302439253081, + "blocks.0.w1.bias": 278.6169359900818, + "blocks.0.w2.weight": 499.7366737832077, + "blocks.1.ln.weight": 8.816575984910754, + "blocks.1.w1.weight": 341.9389165053106, + "blocks.1.w1.bias": 332.3162371142644, + "blocks.1.w2.weight": 347.85214832372776, + "blocks.2.ln.weight": 9.211748215059862, + "blocks.2.w1.weight": 389.3615048816918, + "blocks.2.w1.bias": 365.40848517889776, + "blocks.2.w2.weight": 370.8759731237476, + "blocks.3.ln.weight": 10.03198861563026, + "blocks.3.w1.weight": 406.5535727387553, + "blocks.3.w1.bias": 385.3233146773144, + "blocks.3.w2.weight": 394.381764481299, + "blocks.4.ln.weight": 10.474602013624727, + "blocks.4.w1.weight": 429.8395174539667, + "blocks.4.w1.bias": 400.4483643636673, + "blocks.4.w2.weight": 397.09630840545793, + "blocks.5.ln.weight": 7.5593503080049596, + "blocks.5.w1.weight": 296.69313861733195, + "blocks.5.w1.bias": 284.6870161884467, + "blocks.5.w2.weight": 262.57737559088133, + "blocks.6.ln.weight": 9.004407296024212, + "blocks.6.w1.weight": 356.77068113879443, + "blocks.6.w1.bias": 339.0443824568545, + "blocks.6.w2.weight": 315.61513347531445, + "blocks.7.ln.weight": 7.295378859292153, + "blocks.7.w1.weight": 264.4208821925752, + "blocks.7.w1.bias": 244.22970176166734, + "blocks.7.w2.weight": 243.628762418729, + "blocks.8.ln.weight": 8.932564788347374, + "blocks.8.w1.weight": 351.408355367191, + "blocks.8.w1.bias": 334.01069244258156, + "blocks.8.w2.weight": 326.1916980797317, + "blocks.9.ln.weight": 10.358840465999167, + "blocks.9.w1.weight": 415.0022329456538, + "blocks.9.w1.bias": 381.22980098380185, + "blocks.9.w2.weight": 375.2902593077766, + "blocks.10.ln.weight": 9.099883733701708, + "blocks.10.w1.weight": 357.2725104478134, + "blocks.10.w1.bias": 319.59512329402446, + "blocks.10.w2.weight": 347.7682207115722, + "blocks.11.ln.weight": 9.671063229293274, + "blocks.11.w1.weight": 383.4569677056522, + "blocks.11.w1.bias": 366.9661288886149, + "blocks.11.w2.weight": 358.31090602111163, + "out_ln.weight": 0.6468211624050442, + "out_head.weight": 9.099909010510617, + "out_head.bias": 0.5580580979411405 + } + }, + "fa": { + "log": { + "train_loss": [ + 2.0478644479370116, + 1.9653604916763305, + 1.9457170885849, + 1.9292129779052734, + 1.911267426147461, + 1.8966463186264038, + 1.8790605630874633, + 1.8708782619476318, + 1.8626127404403687, + 1.8581517023468017, + 1.8520171240234375, + 1.8454141430282593, + 1.8327462087249755, + 1.8297313425064088, + 1.8218167012786866, + 1.8236521685791016, + 1.810682963180542, + 1.8065698401641845, + 1.7991282674407958, + 1.794059274559021, + 1.7904878783416749, + 1.7887911895370483, + 1.7848259868621825, + 1.7783562173461913, + 1.7769769561767579, + 1.7767581796264649, + 1.7756539735412598, + 1.776255124130249, + 1.7723581911849975, + 1.7729071472549438, + 1.7715669748687743, + 1.771372833633423, + 1.767975373764038, + 1.7652683026123046, + 1.766329913673401, + 1.7672014434051513, + 1.758714213027954, + 1.7554267383193969, + 1.756759292869568, + 1.7521009868621826, + 1.7497902758789063, + 1.750489810180664, + 1.7499482775497437, + 1.747922197303772, + 1.744107078781128, + 1.7441058347320557, + 1.7407303936004639, + 1.7397420559310912, + 1.7378707089996337, + 1.7362638066864013, + 1.739966582069397, + 1.7335311608505248, + 1.7373196209716797, + 1.7352568884658814, + 1.7298812759399413, + 1.732886531715393, + 1.7291758307647704, + 1.724468434715271, + 1.7238299334716798, + 1.7226758923339844, + 1.7246140502548217, + 1.7225423165130616, + 1.7256555751800537, + 1.7229330892944337, + 1.7288363692855835, + 1.7249921246337891, + 1.7212819675445556, + 1.7200141415023804, + 1.7191423548126221, + 1.7222559392929078, + 1.7194412873077392, + 1.7188739904022217, + 1.7199222569274903, + 1.7203983585357665, + 1.7149837328338624, + 1.7162825980377197, + 1.7136023723602294, + 1.7168209014129638, + 1.7104442378997802, + 1.7158753946685792, + 1.7140782101821899, + 1.7154304480743408, + 1.7131923955535888, + 1.711485964012146, + 1.7109355539703368, + 1.716393593826294, + 1.7096369751358031, + 1.7133884111785889, + 1.7087384057998658, + 1.7118981928253174, + 1.7084463762664794, + 1.7090828707504273, + 1.7140569379425048, + 1.7086596160507201, + 1.706718920211792, + 1.7098103982925414, + 1.7067520601654054, + 1.7073699936676026, + 1.7088190727996826, + 1.7079839348983765 + ], + "train_acc": [ + 0.2395, + 0.28052, + 0.2902, + 0.297, + 0.30538, + 0.31214, + 0.32012, + 0.3235, + 0.32874, + 0.3302, + 0.33236, + 0.336, + 0.3379, + 0.33908, + 0.34476, + 0.34302, + 0.34752, + 0.35002, + 0.3536, + 0.35486, + 0.35646, + 0.35768, + 0.3591, + 0.36082, + 0.36046, + 0.36418, + 0.36422, + 0.3634, + 0.36582, + 0.36498, + 0.3655, + 0.3643, + 0.36678, + 0.36726, + 0.36698, + 0.3653, + 0.37052, + 0.37094, + 0.36996, + 0.37132, + 0.37166, + 0.3736, + 0.37276, + 0.37378, + 0.37664, + 0.37462, + 0.37638, + 0.37634, + 0.3792, + 0.37858, + 0.3749, + 0.37682, + 0.37684, + 0.37932, + 0.3824, + 0.38052, + 0.38102, + 0.38218, + 0.3842, + 0.3825, + 0.38362, + 0.38164, + 0.38222, + 0.38416, + 0.3834, + 0.38208, + 0.38782, + 0.38568, + 0.3865, + 0.38586, + 0.3853, + 0.3871, + 0.3852, + 0.38532, + 0.38646, + 0.38714, + 0.3898, + 0.38778, + 0.38802, + 0.38644, + 0.38714, + 0.39032, + 0.39064, + 0.39112, + 0.39104, + 0.39074, + 0.38882, + 0.38916, + 0.38948, + 0.38788, + 0.38876, + 0.39034, + 0.3877, + 0.38964, + 0.39044, + 0.38936, + 0.39126, + 0.3913, + 0.3887, + 0.3921 + ], + "test_acc": [ + 0.2937, + 0.318, + 0.3284, + 0.3218, + 0.3414, + 0.3309, + 0.3539, + 0.3542, + 0.3588, + 0.3564, + 0.3555, + 0.3677, + 0.3666, + 0.3693, + 0.3683, + 0.3771, + 0.3637, + 0.3767, + 0.3747, + 0.3788, + 0.3798, + 0.3875, + 0.3865, + 0.3832, + 0.3831, + 0.3875, + 0.3819, + 0.3915, + 0.3952, + 0.392, + 0.3842, + 0.3883, + 0.4019, + 0.3939, + 0.3936, + 0.3968, + 0.3946, + 0.3997, + 0.3989, + 0.3971, + 0.4026, + 0.3991, + 0.4026, + 0.4039, + 0.4021, + 0.4032, + 0.4022, + 0.4026, + 0.407, + 0.4115, + 0.4082, + 0.4054, + 0.4011, + 0.4083, + 0.4078, + 0.4068, + 0.4023, + 0.4005, + 0.4032, + 0.4055, + 0.4038, + 0.4083, + 0.4038, + 0.408, + 0.4047, + 0.4058, + 0.4078, + 0.4108, + 0.4074, + 0.41, + 0.406, + 0.4068, + 0.408, + 0.4086, + 0.4098, + 0.4093, + 0.411, + 0.4114, + 0.4128, + 0.4126, + 0.4111, + 0.4122, + 0.4107, + 0.4102, + 0.4096, + 0.4108, + 0.4108, + 0.4128, + 0.4113, + 0.4082, + 0.411, + 0.4112, + 0.4109, + 0.4108, + 0.4107, + 0.4111, + 0.4116, + 0.411, + 0.4108, + 0.4108 + ] + }, + "diagnostics": { + "bp_cosine": [ + 0.03154352679848671, + 0.05562606453895569, + 0.031104888767004013, + -0.07920745015144348, + -0.07174454629421234, + -0.02221393957734108, + -0.0608971044421196, + -0.043849505484104156, + -0.07298076152801514, + -0.004631989635527134, + 0.026790393516421318, + 0.9985308051109314 + ], + "perturbation_rho": [ + 0.027997372671961784, + 0.017702028155326843, + -0.01458565704524517, + 0.0021418300457298756, + 0.01191677525639534, + 0.033614952117204666, + 0.024964284151792526, + 0.01625584065914154, + 0.024897336959838867, + 0.035478636622428894, + 0.016699712723493576, + 0.03735386207699776 + ], + "nudging": { + "0.001": [ + -3.6178389564156532e-06, + -1.6938429325819016e-07, + -6.28642737865448e-09, + 5.6694261729717255e-08, + 5.681067705154419e-08, + 1.909211277961731e-08, + 2.3748725652694702e-08, + 3.166496753692627e-08, + 3.3993273973464966e-08, + 4.9243681132793427e-08, + -2.3283064365386963e-08, + -8.114147931337357e-07 + ], + "0.003": [ + -1.0870513506233692e-05, + -7.745111361145973e-07, + -1.3748649507761002e-07, + 2.558808773756027e-07, + 1.755543053150177e-07, + 4.9709342420101166e-08, + 4.563480615615845e-08, + 9.592622518539429e-08, + 2.1245796233415604e-07, + 2.9569491744041443e-08, + -1.1490192264318466e-07, + -2.9135262593626976e-06 + ], + "0.01": [ + -3.632775042206049e-05, + -2.4959444999694824e-06, + -4.919711500406265e-07, + 7.352791726589203e-07, + 7.244525477290154e-07, + 1.8265563994646072e-07, + 6.683403626084328e-07, + 4.987232387065887e-07, + 7.337657734751701e-07, + 2.223532646894455e-08, + -3.0745286494493484e-07, + -1.0411371476948261e-05 + ] + }, + "hidden_norms_per_layer": [ + 5638.3818359375, + 81270.953125, + 444372.78125, + 1134060.625, + 1891502.0, + 2104572.25, + 2255040.75, + 2375941.75, + 2391241.75, + 2426907.25, + 2452151.0, + 2467291.0, + 1570105.625 + ], + "bp_grad_norms_per_layer": [ + 3.355086664669216e-05, + 1.9466469893814065e-06, + 5.659199473484477e-07, + 4.918285867461236e-07, + 4.922185325995088e-07, + 4.924264089822827e-07, + 4.924931431560253e-07, + 4.925626626572921e-07, + 4.927623535877501e-07, + 4.928857038066781e-07, + 4.913110842608148e-07, + 4.904794081994623e-07, + 4.797909127773892e-07 + ] + }, + "drift": { + "embed.weight": 41.710097786001306, + "embed.bias": 15.464283033590101, + "blocks.0.ln.weight": 1.200268762028184, + "blocks.0.w1.weight": 16.308146249299604, + "blocks.0.w1.bias": 12.841405630221256, + "blocks.0.w2.weight": 57.47900576989446, + "blocks.1.ln.weight": 1.058833461192825, + "blocks.1.w1.weight": 20.255632368172027, + "blocks.1.w1.bias": 14.323810463836294, + "blocks.1.w2.weight": 51.70227234444696, + "blocks.2.ln.weight": 1.0067914589915794, + "blocks.2.w1.weight": 23.883193038734614, + "blocks.2.w1.bias": 21.70680288297347, + "blocks.2.w2.weight": 32.15341601636593, + "blocks.3.ln.weight": 0.6991583611383122, + "blocks.3.w1.weight": 26.213152107729808, + "blocks.3.w1.bias": 27.835553865312857, + "blocks.3.w2.weight": 22.33686846223663, + "blocks.4.ln.weight": 0.49537939877182396, + "blocks.4.w1.weight": 20.812006160361022, + "blocks.4.w1.bias": 22.653788785176967, + "blocks.4.w2.weight": 18.947997726554537, + "blocks.5.ln.weight": 0.5246639568490442, + "blocks.5.w1.weight": 21.161102756452053, + "blocks.5.w1.bias": 23.494613405758418, + "blocks.5.w2.weight": 18.27047815490039, + "blocks.6.ln.weight": 0.5056438845806848, + "blocks.6.w1.weight": 21.366617590276153, + "blocks.6.w1.bias": 23.57646482535375, + "blocks.6.w2.weight": 17.743866768773838, + "blocks.7.ln.weight": 0.37623071960771737, + "blocks.7.w1.weight": 15.484194372550624, + "blocks.7.w1.bias": 16.374712028510555, + "blocks.7.w2.weight": 18.270059859207745, + "blocks.8.ln.weight": 0.4246821474291729, + "blocks.8.w1.weight": 15.052279864294677, + "blocks.8.w1.bias": 15.735595602596712, + "blocks.8.w2.weight": 20.64493053094595, + "blocks.9.ln.weight": 0.40094293459913244, + "blocks.9.w1.weight": 13.315401582838119, + "blocks.9.w1.bias": 11.448042864028588, + "blocks.9.w2.weight": 39.460291915843406, + "blocks.10.ln.weight": 0.3986931357871173, + "blocks.10.w1.weight": 12.861490846141741, + "blocks.10.w1.bias": 10.433659114876445, + "blocks.10.w2.weight": 37.16747442552621, + "blocks.11.ln.weight": 0.5241503553443005, + "blocks.11.w1.weight": 18.756889297523845, + "blocks.11.w1.bias": 18.863150794441257, + "blocks.11.w2.weight": 46.497694834593375, + "out_ln.weight": 0.3799708731554326, + "out_head.weight": 6.730072302834575, + "out_head.bias": 0.6968175874942788 + } + } + }, + "config": { + "dataset": "cifar10", + "d_hidden": 512, + "num_blocks": 12, + "batch_size": 128, + "epochs": 100, + "lr": 0.001, + "lr_fb": 0.001, + "wd": 0.01, + "lam": 0.1, + "K": 4, + "sigma_bridge": 0.05, + "ema_momentum": 0.995, + "term_grad_weight": 1.0, + "seeds": [ + 4 + ], + "gpu": 0, + "output_dir": "results/fa_dfa_d512_L12_seed4", + "methods": [ + "fa", + "dfa" + ], + "random_targets": false, + "penalty_lam": 0.0, + "num_classes": 10 + } +}
\ No newline at end of file |