diff options
Diffstat (limited to 'results/fa_dfa_d512_L12_seed5/results_cifar10.json')
| -rw-r--r-- | results/fa_dfa_d512_L12_seed5/results_cifar10.json | 969 |
1 files changed, 969 insertions, 0 deletions
diff --git a/results/fa_dfa_d512_L12_seed5/results_cifar10.json b/results/fa_dfa_d512_L12_seed5/results_cifar10.json new file mode 100644 index 0000000..1e9a983 --- /dev/null +++ b/results/fa_dfa_d512_L12_seed5/results_cifar10.json @@ -0,0 +1,969 @@ +{ + "5": { + "dfa": { + "log": { + "train_loss": [ + 2.0684647270965577, + 2.04593505569458, + 2.0332052132797243, + 2.0338553718566894, + 2.026373544998169, + 2.0213139679336547, + 2.0205811246109007, + 2.018696953125, + 2.014947237319946, + 2.0151999055480956, + 2.0129982511138915, + 2.011509538650513, + 2.0100906581115723, + 2.0089198484802244, + 2.0063697845458983, + 2.006922910385132, + 2.0044513999176026, + 2.0016603435897826, + 2.000636997833252, + 2.002304845352173, + 2.0021789043426512, + 2.001000444030762, + 2.00005261428833, + 2.002928783569336, + 2.0017156188201906, + 1.9988520227050781, + 1.9994586238861083, + 1.9980162719726562, + 1.9998335108184815, + 1.9996961016845702, + 1.9991928104400636, + 1.9968728270721436, + 1.9977278369903564, + 1.9969172173309326, + 1.997944600830078, + 1.9979880140686035, + 1.994605475616455, + 1.9973402403259277, + 1.997146150894165, + 1.99599303855896, + 1.9974721060943603, + 1.9941783197402954, + 1.9956128702545166, + 1.9942147409057618, + 1.994319501800537, + 1.994643907546997, + 1.9942114786529541, + 1.994346463241577, + 1.9929966555023193, + 1.9937490250396728, + 1.992175287437439, + 1.9944900625610351, + 1.9938098908233644, + 1.9900078340911864, + 1.994505626373291, + 1.9926387735366822, + 1.990931240310669, + 1.9906283693695068, + 1.9929009646606446, + 1.9926953937530518, + 1.9925627197647096, + 1.990387219581604, + 1.9893561473846435, + 1.9920542861557007, + 1.9893005572509765, + 1.9888617028808593, + 1.991559531326294, + 1.9911002109527587, + 1.9900375119781495, + 1.9892236702728272, + 1.9937436754608153, + 1.991619889755249, + 1.9916846366882324, + 1.9896835166931153, + 1.9901411660003663, + 1.9924274538421631, + 1.9892655798721313, + 1.9914114519500732, + 1.990216528968811, + 1.9892477695465087, + 1.9900167115783691, + 1.98849309715271, + 1.9889385766983032, + 1.988801861000061, + 1.9898810375976563, + 1.9873209413909911, + 1.9886231661224365, + 1.988806999168396, + 1.9894798669052125, + 1.988367915725708, + 1.9881281168365479, + 1.9901167456054687, + 1.9896381842041015, + 1.9841947576522827, + 1.98667507106781, + 1.9886395093154907, + 1.9892304638290406, + 1.9895199391174316, + 1.9884832098388672, + 1.9883703183746337 + ], + "train_acc": [ + 0.23802, + 0.24804, + 0.25088, + 0.25142, + 0.25734, + 0.2584, + 0.26202, + 0.26208, + 0.265, + 0.26464, + 0.26442, + 0.26658, + 0.26774, + 0.26682, + 0.27064, + 0.26844, + 0.26844, + 0.27148, + 0.2714, + 0.26686, + 0.27288, + 0.26878, + 0.27184, + 0.27342, + 0.2741, + 0.27264, + 0.27204, + 0.27416, + 0.27192, + 0.27138, + 0.27496, + 0.27304, + 0.27252, + 0.2761, + 0.2752, + 0.27484, + 0.27692, + 0.2757, + 0.27436, + 0.27528, + 0.27534, + 0.276, + 0.27886, + 0.27704, + 0.27484, + 0.27556, + 0.27668, + 0.27734, + 0.27718, + 0.27692, + 0.27726, + 0.27706, + 0.2785, + 0.27902, + 0.2764, + 0.27802, + 0.27758, + 0.2812, + 0.27958, + 0.2804, + 0.27626, + 0.27992, + 0.2817, + 0.27822, + 0.2799, + 0.28178, + 0.2783, + 0.27782, + 0.28144, + 0.28136, + 0.2789, + 0.2784, + 0.27892, + 0.28034, + 0.28046, + 0.2792, + 0.28034, + 0.28004, + 0.28032, + 0.28042, + 0.28062, + 0.28206, + 0.28064, + 0.28308, + 0.2811, + 0.28224, + 0.28148, + 0.2803, + 0.2814, + 0.27944, + 0.28004, + 0.27996, + 0.28, + 0.28104, + 0.28072, + 0.28176, + 0.28028, + 0.27976, + 0.28074, + 0.27914 + ], + "test_acc": [ + 0.2445, + 0.2726, + 0.2811, + 0.2819, + 0.2748, + 0.2867, + 0.2884, + 0.2833, + 0.2868, + 0.2909, + 0.2961, + 0.2829, + 0.2919, + 0.3048, + 0.2947, + 0.2998, + 0.2984, + 0.2733, + 0.2898, + 0.2992, + 0.2974, + 0.289, + 0.2923, + 0.2983, + 0.2849, + 0.3036, + 0.2861, + 0.2991, + 0.2961, + 0.3025, + 0.2979, + 0.2897, + 0.2946, + 0.2973, + 0.2984, + 0.3042, + 0.2986, + 0.2994, + 0.3083, + 0.2802, + 0.3018, + 0.2976, + 0.301, + 0.301, + 0.2973, + 0.3045, + 0.297, + 0.2933, + 0.306, + 0.3031, + 0.2937, + 0.3027, + 0.2977, + 0.2974, + 0.3053, + 0.301, + 0.2965, + 0.3001, + 0.2994, + 0.2956, + 0.3025, + 0.286, + 0.2916, + 0.2956, + 0.3058, + 0.2961, + 0.2973, + 0.2996, + 0.2925, + 0.3068, + 0.3085, + 0.3058, + 0.2965, + 0.3035, + 0.3075, + 0.301, + 0.3066, + 0.3077, + 0.3019, + 0.3036, + 0.2997, + 0.3005, + 0.297, + 0.3008, + 0.3006, + 0.3103, + 0.3054, + 0.3048, + 0.2988, + 0.3009, + 0.3011, + 0.3039, + 0.3005, + 0.3037, + 0.3031, + 0.3022, + 0.302, + 0.3018, + 0.3018, + 0.3018 + ] + }, + "diagnostics": { + "bp_cosine": [ + 0.41004806756973267, + 0.0008555407403036952, + 0.0005728952819481492, + -0.0003633289597928524, + 0.00043969464604742825, + 0.0006190181011334062, + -0.0006453525274991989, + 0.0001250960340257734, + -0.0005502170533873141, + -6.381357525242493e-05, + -0.00011379925126675516, + -0.00034056592267006636 + ], + "perturbation_rho": [ + -0.0048940712586045265, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "nudging": { + "0.001": [ + -3.5297125577926636e-07, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "0.003": [ + -1.1865049600601196e-06, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "0.01": [ + -3.998167812824249e-06, + 0.0, + 0.0, + 9.313225746154785e-10, + -8.381903171539307e-09, + 0.0, + 0.0, + 9.313225746154785e-10, + 0.0, + 0.0, + 0.0, + 0.0 + ] + }, + "hidden_norms_per_layer": [ + 55109.36328125, + 840561728.0, + 1444462336.0, + 2124835456.0, + 3210963200.0, + 6536487936.0, + 7593719808.0, + 8378088960.0, + 8459614208.0, + 9916916736.0, + 10216921088.0, + 11792236544.0, + 12205541376.0 + ], + "bp_grad_norms_per_layer": [ + 2.8213216296535393e-07, + 2.0590699978395577e-10, + 2.0399454347952428e-10, + 2.03904518270015e-10, + 2.0418219892626155e-10, + 2.0476981221762003e-10, + 2.0479024032127313e-10, + 2.0459070548817238e-10, + 2.0458530702871514e-10, + 2.045856123400469e-10, + 2.04587194407857e-10, + 2.044076713447751e-10, + 2.045200814260184e-10 + ] + }, + "drift": { + "embed.weight": 331.27602666259105, + "embed.bias": 239.81762105942474, + "blocks.0.ln.weight": 10.927184995923978, + "blocks.0.w1.weight": 251.71912543831303, + "blocks.0.w1.bias": 221.3073991198696, + "blocks.0.w2.weight": 446.18400062248946, + "blocks.1.ln.weight": 7.981967217674905, + "blocks.1.w1.weight": 253.74455824953162, + "blocks.1.w1.bias": 216.76732500414306, + "blocks.1.w2.weight": 295.48008203993817, + "blocks.2.ln.weight": 8.227632098130004, + "blocks.2.w1.weight": 269.5007877113237, + "blocks.2.w1.bias": 241.96437719605447, + "blocks.2.w2.weight": 310.98985782960006, + "blocks.3.ln.weight": 8.112799129886726, + "blocks.3.w1.weight": 308.3323380552167, + "blocks.3.w1.bias": 309.27230622114104, + "blocks.3.w2.weight": 318.17414451568425, + "blocks.4.ln.weight": 10.16216573439241, + "blocks.4.w1.weight": 424.58007413849407, + "blocks.4.w1.bias": 404.55597818760236, + "blocks.4.w2.weight": 422.3987925739278, + "blocks.5.ln.weight": 9.551306834710088, + "blocks.5.w1.weight": 374.36129372347426, + "blocks.5.w1.bias": 352.23829526395434, + "blocks.5.w2.weight": 368.0730794757206, + "blocks.6.ln.weight": 8.854285613855794, + "blocks.6.w1.weight": 345.52287116736915, + "blocks.6.w1.bias": 319.68248259727653, + "blocks.6.w2.weight": 327.22678020367175, + "blocks.7.ln.weight": 7.811231342959617, + "blocks.7.w1.weight": 277.64478372749727, + "blocks.7.w1.bias": 254.112362538096, + "blocks.7.w2.weight": 265.1743796968286, + "blocks.8.ln.weight": 10.228506898132666, + "blocks.8.w1.weight": 404.4219524430621, + "blocks.8.w1.bias": 393.5696990939405, + "blocks.8.w2.weight": 394.0798993515089, + "blocks.9.ln.weight": 8.0107262045422, + "blocks.9.w1.weight": 316.5928270106835, + "blocks.9.w1.bias": 286.78238448653866, + "blocks.9.w2.weight": 310.9127649917408, + "blocks.10.ln.weight": 11.136451857382143, + "blocks.10.w1.weight": 438.2046003550788, + "blocks.10.w1.bias": 405.1181282489877, + "blocks.10.w2.weight": 413.8473423192705, + "blocks.11.ln.weight": 9.410383955267694, + "blocks.11.w1.weight": 372.4463938503806, + "blocks.11.w1.bias": 347.63407553366176, + "blocks.11.w2.weight": 344.49269974034115, + "out_ln.weight": 0.7270416629191093, + "out_head.weight": 10.238250621105578, + "out_head.bias": 0.4975266289585721 + } + }, + "fa": { + "log": { + "train_loss": [ + 2.0707826377868654, + 1.9932800539779663, + 1.959168056678772, + 1.9474205780029297, + 1.9326355227279663, + 1.9199382480239868, + 1.9140991415405273, + 1.9034375612640382, + 1.8959864074707031, + 1.8931433800506592, + 1.8902097482681275, + 1.8859386821746826, + 1.882199794769287, + 1.8849088175201416, + 1.8818455212020875, + 1.8821384326171875, + 1.8799380892944335, + 1.875362028427124, + 1.870530955467224, + 1.8683246834564209, + 1.8638563344573975, + 1.8618846607208253, + 1.8603020819854736, + 1.8594753393936156, + 1.8566845929718017, + 1.8526717345428467, + 1.849912834777832, + 1.8465362030029298, + 1.848408186569214, + 1.845232035217285, + 1.843059235229492, + 1.839803340072632, + 1.8417775817871094, + 1.8390681932067872, + 1.8388364002227784, + 1.8360300784683228, + 1.8323893267059326, + 1.834402112121582, + 1.8309936769866944, + 1.8322431957244874, + 1.8288187261581421, + 1.8238647890472413, + 1.824506708908081, + 1.8221080670166017, + 1.8222445755767822, + 1.8188702960205079, + 1.817204086036682, + 1.8124210216522216, + 1.8137152634429932, + 1.8142603408050537, + 1.8083323223876953, + 1.8079692990493774, + 1.8071039364624024, + 1.8000855495834351, + 1.8052594284057617, + 1.802986600036621, + 1.7997608633422852, + 1.7976583826065065, + 1.798347628555298, + 1.7966360546875, + 1.7926319261932373, + 1.7933891333007812, + 1.7893145151138306, + 1.7920790769195556, + 1.7883893395614625, + 1.785349788131714, + 1.7879791070938111, + 1.7846136389541627, + 1.783509234313965, + 1.7843902270507812, + 1.7877061660385132, + 1.7827738708877563, + 1.7823649167251587, + 1.7814125487899781, + 1.7809069869995118, + 1.7828298908233642, + 1.7791348586273192, + 1.7774614221191407, + 1.7751106949615478, + 1.7760692443466186, + 1.7797742696380616, + 1.7786880410003663, + 1.7746904688262939, + 1.776022590560913, + 1.7778167572402954, + 1.775718899459839, + 1.7761091399383544, + 1.7744762685394286, + 1.7767131290435791, + 1.7732363131713866, + 1.7731415704345703, + 1.7759572997665405, + 1.7733897400283813, + 1.7683659911346437, + 1.7711752317047118, + 1.772973984146118, + 1.772844222984314, + 1.774391251449585, + 1.7711415176010132, + 1.7753938333511352 + ], + "train_acc": [ + 0.22922, + 0.26562, + 0.28014, + 0.286, + 0.29238, + 0.29874, + 0.30312, + 0.30664, + 0.30876, + 0.31168, + 0.3128, + 0.31518, + 0.31798, + 0.317, + 0.3238, + 0.32118, + 0.32352, + 0.32418, + 0.329, + 0.32642, + 0.33084, + 0.3318, + 0.3332, + 0.33264, + 0.33466, + 0.33646, + 0.33754, + 0.34266, + 0.3406, + 0.33896, + 0.34342, + 0.34406, + 0.3431, + 0.34494, + 0.34444, + 0.346, + 0.34776, + 0.34624, + 0.34814, + 0.34848, + 0.34938, + 0.35124, + 0.35154, + 0.3523, + 0.35154, + 0.35416, + 0.3531, + 0.3559, + 0.35634, + 0.35504, + 0.35722, + 0.35778, + 0.35632, + 0.36152, + 0.36064, + 0.35976, + 0.35954, + 0.36118, + 0.36272, + 0.36392, + 0.36202, + 0.36176, + 0.3667, + 0.36318, + 0.36542, + 0.36726, + 0.36706, + 0.36818, + 0.3674, + 0.36874, + 0.36462, + 0.36766, + 0.36724, + 0.3694, + 0.36714, + 0.36818, + 0.36918, + 0.37, + 0.36942, + 0.37088, + 0.36958, + 0.3705, + 0.3704, + 0.37006, + 0.37176, + 0.37004, + 0.37138, + 0.37146, + 0.37036, + 0.36844, + 0.37048, + 0.37066, + 0.37288, + 0.37296, + 0.37426, + 0.37316, + 0.37076, + 0.37046, + 0.37382, + 0.37288 + ], + "test_acc": [ + 0.2587, + 0.2962, + 0.3064, + 0.3196, + 0.323, + 0.3355, + 0.3259, + 0.3316, + 0.345, + 0.3454, + 0.347, + 0.3503, + 0.3501, + 0.3606, + 0.35, + 0.366, + 0.3606, + 0.345, + 0.3598, + 0.3644, + 0.365, + 0.3731, + 0.3727, + 0.3741, + 0.3592, + 0.3701, + 0.3692, + 0.375, + 0.3687, + 0.3686, + 0.3813, + 0.3762, + 0.3719, + 0.3836, + 0.3831, + 0.3884, + 0.3927, + 0.3866, + 0.3775, + 0.3781, + 0.3852, + 0.3899, + 0.3881, + 0.3895, + 0.3869, + 0.3874, + 0.3932, + 0.3817, + 0.3908, + 0.3893, + 0.3806, + 0.3918, + 0.3899, + 0.39, + 0.3949, + 0.3925, + 0.391, + 0.3933, + 0.3968, + 0.3904, + 0.3935, + 0.3933, + 0.3933, + 0.3992, + 0.4014, + 0.3955, + 0.3973, + 0.396, + 0.3963, + 0.3933, + 0.4005, + 0.4025, + 0.3997, + 0.4007, + 0.4027, + 0.4043, + 0.4044, + 0.4033, + 0.4033, + 0.4026, + 0.3994, + 0.4039, + 0.3988, + 0.4037, + 0.4041, + 0.4071, + 0.4054, + 0.4045, + 0.4049, + 0.4049, + 0.404, + 0.4069, + 0.4028, + 0.4041, + 0.4052, + 0.4053, + 0.4038, + 0.4034, + 0.4041, + 0.404 + ] + }, + "diagnostics": { + "bp_cosine": [ + 0.02494833618402481, + 0.06461166590452194, + 0.007773830089718103, + -0.053252629935741425, + -0.03213752806186676, + -0.0018765359418466687, + -0.020534943789243698, + -0.05978960543870926, + -0.019162429496645927, + 0.007563438266515732, + 0.033061787486076355, + 0.989224910736084 + ], + "perturbation_rho": [ + -0.0004792527761310339, + -0.012973977252840996, + 0.01789543777704239, + 0.0009404178708791733, + -0.03582464158535004, + -0.024608338251709938, + 0.03623630106449127, + -0.0018152520060539246, + 0.017969228327274323, + 0.0033451307099312544, + 0.0, + 0.029760660603642464 + ], + "nudging": { + "0.001": [ + -9.243376553058624e-07, + -5.587935447692871e-08, + 2.0721927285194397e-08, + -3.119930624961853e-08, + -2.1420419216156006e-08, + -2.9336661100387573e-08, + -2.0023435354232788e-08, + -5.122274160385132e-09, + -6.05359673500061e-09, + 3.725290298461914e-09, + -2.3748725652694702e-08, + -2.4866312742233276e-07 + ], + "0.003": [ + -2.8724316507577896e-06, + -2.635642886161804e-07, + -5.4249539971351624e-08, + -1.5599653124809265e-08, + -4.44706529378891e-08, + -1.6996636986732483e-08, + 1.1641532182693481e-09, + 5.704350769519806e-08, + -1.3504177331924438e-08, + -1.3737007975578308e-08, + -3.259629011154175e-08, + -1.0190997272729874e-06 + ], + "0.01": [ + -9.626368409954011e-06, + -7.129274308681488e-07, + 5.145557224750519e-08, + 1.8137507140636444e-07, + 1.0221265256404877e-07, + 1.3969838619232178e-08, + 3.608874976634979e-08, + 2.377200871706009e-07, + -1.6298145055770874e-08, + -8.731149137020111e-08, + -1.6693957149982452e-07, + -3.8032885640859604e-06 + ] + }, + "hidden_norms_per_layer": [ + 11572.41796875, + 170961.703125, + 761668.5625, + 2149773.25, + 3106495.5, + 3628764.75, + 3844090.5, + 4201663.0, + 4523281.0, + 4742705.5, + 4761602.0, + 4759073.5, + 4570665.0 + ], + "bp_grad_norms_per_layer": [ + 2.5100242055486888e-05, + 6.707207944600668e-07, + 2.330790778160008e-07, + 2.3257328507497732e-07, + 2.3236127333348122e-07, + 2.322931180742671e-07, + 2.323264851611384e-07, + 2.3226489531680272e-07, + 2.3237980428802985e-07, + 2.3232749413182319e-07, + 2.322744165894619e-07, + 2.3242134261636238e-07, + 2.2277541233961529e-07 + ] + }, + "drift": { + "embed.weight": 67.4183636503971, + "embed.bias": 12.639534947709132, + "blocks.0.ln.weight": 1.4281653737839686, + "blocks.0.w1.weight": 22.149022125188107, + "blocks.0.w1.bias": 14.43109450707943, + "blocks.0.w2.weight": 73.54315291854846, + "blocks.1.ln.weight": 1.3725125667153013, + "blocks.1.w1.weight": 28.041388459852282, + "blocks.1.w1.bias": 16.737535867388896, + "blocks.1.w2.weight": 48.10538352181479, + "blocks.2.ln.weight": 0.9692921428242081, + "blocks.2.w1.weight": 29.126634688998955, + "blocks.2.w1.bias": 28.160099907002987, + "blocks.2.w2.weight": 30.43668721582192, + "blocks.3.ln.weight": 0.8517630406355475, + "blocks.3.w1.weight": 30.698936977122976, + "blocks.3.w1.bias": 30.008595592413137, + "blocks.3.w2.weight": 25.53125025342729, + "blocks.4.ln.weight": 0.8000730489190893, + "blocks.4.w1.weight": 28.294012201626252, + "blocks.4.w1.bias": 27.681602003963505, + "blocks.4.w2.weight": 31.236916248398295, + "blocks.5.ln.weight": 0.7726872906934783, + "blocks.5.w1.weight": 27.361431129938182, + "blocks.5.w1.bias": 27.206264858292855, + "blocks.5.w2.weight": 30.441424834889585, + "blocks.6.ln.weight": 0.836359631646653, + "blocks.6.w1.weight": 26.47170019895073, + "blocks.6.w1.bias": 25.167535850732502, + "blocks.6.w2.weight": 27.854377209764337, + "blocks.7.ln.weight": 0.7549166617706077, + "blocks.7.w1.weight": 26.946400539940182, + "blocks.7.w1.bias": 26.486714874573558, + "blocks.7.w2.weight": 31.532602647761905, + "blocks.8.ln.weight": 0.7532469721914246, + "blocks.8.w1.weight": 26.170218010343223, + "blocks.8.w1.bias": 27.052147560472978, + "blocks.8.w2.weight": 31.57827072835312, + "blocks.9.ln.weight": 0.5491388007855835, + "blocks.9.w1.weight": 17.942432655743463, + "blocks.9.w1.bias": 16.168189388817968, + "blocks.9.w2.weight": 38.437476826047074, + "blocks.10.ln.weight": 0.44584462833770383, + "blocks.10.w1.weight": 14.61733255882144, + "blocks.10.w1.bias": 9.906228706328685, + "blocks.10.w2.weight": 63.74207530119445, + "blocks.11.ln.weight": 0.4261726286054206, + "blocks.11.w1.weight": 16.108342263191403, + "blocks.11.w1.bias": 10.380547619047618, + "blocks.11.w2.weight": 90.22616819735002, + "out_ln.weight": 0.4534589624136927, + "out_head.weight": 7.737075841258268, + "out_head.bias": 0.6965283841940496 + } + } + }, + "config": { + "dataset": "cifar10", + "d_hidden": 512, + "num_blocks": 12, + "batch_size": 128, + "epochs": 100, + "lr": 0.001, + "lr_fb": 0.001, + "wd": 0.01, + "lam": 0.1, + "K": 4, + "sigma_bridge": 0.05, + "ema_momentum": 0.995, + "term_grad_weight": 1.0, + "seeds": [ + 5 + ], + "gpu": 0, + "output_dir": "results/fa_dfa_d512_L12_seed5", + "methods": [ + "fa", + "dfa" + ], + "random_targets": false, + "penalty_lam": 0.0, + "num_classes": 10 + } +}
\ No newline at end of file |