diff options
Diffstat (limited to 'results/fa_dfa_d512_L12_seed9/results_cifar10.json')
| -rw-r--r-- | results/fa_dfa_d512_L12_seed9/results_cifar10.json | 969 |
1 files changed, 969 insertions, 0 deletions
diff --git a/results/fa_dfa_d512_L12_seed9/results_cifar10.json b/results/fa_dfa_d512_L12_seed9/results_cifar10.json new file mode 100644 index 0000000..974d42a --- /dev/null +++ b/results/fa_dfa_d512_L12_seed9/results_cifar10.json @@ -0,0 +1,969 @@ +{ + "9": { + "dfa": { + "log": { + "train_loss": [ + 2.086543095245361, + 2.049035365447998, + 2.0382854364776612, + 2.0298845935058596, + 2.0281315744018555, + 2.0237447773742674, + 2.0194822165679933, + 2.019161204986572, + 2.015748862915039, + 2.0162334437561036, + 2.0126681255340575, + 2.012035894165039, + 2.009547615966797, + 2.011429062194824, + 2.0080011304473877, + 2.0085202750396727, + 2.0067744227600097, + 2.0047744031143186, + 2.004743169174194, + 2.001354461517334, + 2.0038469007873534, + 2.0027169017028807, + 1.9995321035003661, + 1.9969835998916625, + 1.9975377194595336, + 1.9985439079284668, + 1.9960253533172607, + 1.9974639236831666, + 1.9950202367401122, + 1.9968991885375977, + 1.9976718412017822, + 1.9939492235565186, + 1.9947053424835206, + 1.995585137901306, + 1.9956871920776367, + 1.9927901021575927, + 1.992329425201416, + 1.9917698760604858, + 1.9922489316558838, + 1.9921125721740722, + 1.9919519045257568, + 1.9915309338378906, + 1.9927355318450928, + 1.9899454330825805, + 1.9910321056365967, + 1.9931654618835448, + 1.9889872648620606, + 1.9899863806915283, + 1.9893665783309937, + 1.989453801651001, + 1.9890669647216797, + 1.9882669693374633, + 1.9893041844177246, + 1.9891512668609619, + 1.9883573954772948, + 1.9891788550186158, + 1.9858848489379883, + 1.9877911152648926, + 1.987854264564514, + 1.9873136006164551, + 1.9871731573486329, + 1.9879692407989502, + 1.987359190750122, + 1.9866330204772948, + 1.98554376373291, + 1.9857093753051758, + 1.986483130493164, + 1.9841259546279908, + 1.98654155418396, + 1.9857370235824585, + 1.9854363512420654, + 1.9837415993881227, + 1.9846706900024413, + 1.9846486893463136, + 1.9852562395477296, + 1.985122490158081, + 1.9856149569702148, + 1.9855838787460327, + 1.985352071609497, + 1.9836603102874757, + 1.984020567855835, + 1.9842208641052246, + 1.9867661368179321, + 1.9825923907852172, + 1.9839700649261474, + 1.985303124923706, + 1.984482919921875, + 1.9825283669281006, + 1.9836111863708497, + 1.9835795726013183, + 1.9827597061920166, + 1.98389965385437, + 1.9854092786407471, + 1.9815458943939208, + 1.9817595046615601, + 1.9855254986190796, + 1.9835386077880859, + 1.9825453355407714, + 1.9826166521835327, + 1.9829287356567382 + ], + "train_acc": [ + 0.22926, + 0.24766, + 0.24966, + 0.25552, + 0.25304, + 0.25922, + 0.25868, + 0.25894, + 0.2602, + 0.26156, + 0.26166, + 0.26588, + 0.26356, + 0.26296, + 0.2648, + 0.26752, + 0.26888, + 0.26516, + 0.2687, + 0.2697, + 0.27084, + 0.26762, + 0.26938, + 0.27176, + 0.2698, + 0.27094, + 0.2722, + 0.2721, + 0.27342, + 0.27132, + 0.27128, + 0.2735, + 0.27358, + 0.27302, + 0.2743, + 0.27702, + 0.27592, + 0.2762, + 0.27586, + 0.27454, + 0.27708, + 0.27588, + 0.27488, + 0.27664, + 0.27778, + 0.27754, + 0.27744, + 0.27638, + 0.2782, + 0.27892, + 0.27888, + 0.27894, + 0.27778, + 0.2784, + 0.2806, + 0.2788, + 0.27942, + 0.27968, + 0.27888, + 0.2771, + 0.27966, + 0.27918, + 0.28102, + 0.2799, + 0.27998, + 0.27866, + 0.28044, + 0.28208, + 0.28314, + 0.28018, + 0.2797, + 0.28004, + 0.2795, + 0.28196, + 0.28128, + 0.2824, + 0.28326, + 0.28176, + 0.28084, + 0.2816, + 0.28268, + 0.28344, + 0.28014, + 0.283, + 0.2824, + 0.28114, + 0.28384, + 0.28424, + 0.28492, + 0.28422, + 0.28338, + 0.28276, + 0.28152, + 0.28358, + 0.28352, + 0.28086, + 0.2832, + 0.28484, + 0.28126, + 0.28462 + ], + "test_acc": [ + 0.2701, + 0.2621, + 0.2798, + 0.281, + 0.2593, + 0.2849, + 0.2797, + 0.2768, + 0.272, + 0.2901, + 0.2986, + 0.2839, + 0.2741, + 0.2916, + 0.2865, + 0.2818, + 0.3006, + 0.2972, + 0.2972, + 0.3049, + 0.2762, + 0.2939, + 0.2754, + 0.2954, + 0.2984, + 0.3, + 0.2821, + 0.2798, + 0.2887, + 0.3018, + 0.2833, + 0.3076, + 0.2891, + 0.291, + 0.2959, + 0.284, + 0.2942, + 0.3106, + 0.2895, + 0.3004, + 0.3048, + 0.3058, + 0.303, + 0.2951, + 0.3031, + 0.2955, + 0.2961, + 0.3036, + 0.3045, + 0.2976, + 0.2995, + 0.2965, + 0.2912, + 0.3007, + 0.2987, + 0.308, + 0.3073, + 0.3, + 0.3117, + 0.2891, + 0.2945, + 0.3064, + 0.2987, + 0.2929, + 0.2984, + 0.2945, + 0.2945, + 0.3028, + 0.2994, + 0.3008, + 0.3, + 0.3069, + 0.3033, + 0.3088, + 0.3052, + 0.3023, + 0.2992, + 0.2946, + 0.305, + 0.3024, + 0.3081, + 0.3014, + 0.3053, + 0.3039, + 0.3079, + 0.305, + 0.3035, + 0.3018, + 0.3052, + 0.3063, + 0.3025, + 0.3056, + 0.3037, + 0.3046, + 0.3045, + 0.3041, + 0.3043, + 0.3044, + 0.3044, + 0.3043 + ] + }, + "diagnostics": { + "bp_cosine": [ + 0.3948982357978821, + 8.632082608528435e-05, + -0.00022141945373732597, + -0.0004791135434061289, + 0.00010106388799613342, + 7.346954225795344e-05, + -7.074545283103362e-05, + -5.661595787387341e-05, + 0.00012481751036830246, + -0.0005871393950656056, + 0.00015422290016431361, + -0.0005310022970661521 + ], + "perturbation_rho": [ + 0.0077664791606366634, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "nudging": { + "0.001": [ + -3.594905138015747e-07, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "0.003": [ + -1.3024546205997467e-06, + -3.725290298461914e-09, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "0.01": [ + -4.144385457038879e-06, + -3.725290298461914e-09, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + -9.313225746154785e-10, + 0.0, + 0.0, + 0.0, + 0.0 + ] + }, + "hidden_norms_per_layer": [ + 56626.58984375, + 921777728.0, + 2723525376.0, + 5562149376.0, + 6096715776.0, + 6586259968.0, + 9732721664.0, + 9669530624.0, + 10051646464.0, + 10812058624.0, + 10900319232.0, + 11575299072.0, + 12058745856.0 + ], + "bp_grad_norms_per_layer": [ + 2.953412661099719e-07, + 1.7980278377294212e-10, + 1.7774372251810888e-10, + 1.7754615833087684e-10, + 1.77546435886633e-10, + 1.7753745695792134e-10, + 1.775574409723646e-10, + 1.7755288905796363e-10, + 1.7755619197146189e-10, + 1.7750646785774649e-10, + 1.775143226856457e-10, + 1.7751018710487898e-10, + 1.775422031613516e-10 + ] + }, + "drift": { + "embed.weight": 330.97935370781335, + "embed.bias": 242.7352200338295, + "blocks.0.ln.weight": 10.118660070740958, + "blocks.0.w1.weight": 268.02039354554194, + "blocks.0.w1.bias": 236.60527981405795, + "blocks.0.w2.weight": 479.95056180068593, + "blocks.1.ln.weight": 8.173523635910902, + "blocks.1.w1.weight": 314.00233907878504, + "blocks.1.w1.bias": 303.2790322538351, + "blocks.1.w2.weight": 332.3008689530667, + "blocks.2.ln.weight": 9.503933776173088, + "blocks.2.w1.weight": 407.2148671186256, + "blocks.2.w1.bias": 377.7677645021183, + "blocks.2.w2.weight": 405.2326915964652, + "blocks.3.ln.weight": 8.261827691373458, + "blocks.3.w1.weight": 324.0550552979263, + "blocks.3.w1.bias": 305.54524457584483, + "blocks.3.w2.weight": 300.9480221394592, + "blocks.4.ln.weight": 8.47590401409121, + "blocks.4.w1.weight": 338.56186029144186, + "blocks.4.w1.bias": 322.0375352385375, + "blocks.4.w2.weight": 319.77753486779426, + "blocks.5.ln.weight": 11.22139697754829, + "blocks.5.w1.weight": 458.2444114055581, + "blocks.5.w1.bias": 424.3745814588518, + "blocks.5.w2.weight": 450.83821734650985, + "blocks.6.ln.weight": 7.796159629674181, + "blocks.6.w1.weight": 306.7435557133061, + "blocks.6.w1.bias": 295.1474090093065, + "blocks.6.w2.weight": 287.3353674909529, + "blocks.7.ln.weight": 8.077037696900597, + "blocks.7.w1.weight": 320.0610116694296, + "blocks.7.w1.bias": 297.55111326321224, + "blocks.7.w2.weight": 294.8277249235264, + "blocks.8.ln.weight": 9.437782779016567, + "blocks.8.w1.weight": 381.01065244180916, + "blocks.8.w1.bias": 346.5987668046489, + "blocks.8.w2.weight": 354.1595052479188, + "blocks.9.ln.weight": 7.2340406951431575, + "blocks.9.w1.weight": 277.2401387021752, + "blocks.9.w1.bias": 262.2340251864824, + "blocks.9.w2.weight": 255.1111568089683, + "blocks.10.ln.weight": 9.961816471882193, + "blocks.10.w1.weight": 400.6058052128251, + "blocks.10.w1.bias": 369.00187630886694, + "blocks.10.w2.weight": 388.64956558131564, + "blocks.11.ln.weight": 8.356343855238048, + "blocks.11.w1.weight": 316.91850723934095, + "blocks.11.w1.bias": 297.45861181164963, + "blocks.11.w2.weight": 290.23124955665594, + "out_ln.weight": 0.6534850850308469, + "out_head.weight": 9.848963989978573, + "out_head.bias": 0.41699188717651064 + } + }, + "fa": { + "log": { + "train_loss": [ + 2.0459222147369385, + 1.9676443615722656, + 1.9359465097045898, + 1.9049253499984742, + 1.8857684122085572, + 1.8734163967514037, + 1.866455598678589, + 1.8637321212387086, + 1.8555012069320678, + 1.8584735760879516, + 1.84622566532135, + 1.8429653158950805, + 1.8373867583847046, + 1.837373632888794, + 1.8269891134643554, + 1.8270549670791627, + 1.8241438305282593, + 1.8183621447372436, + 1.8176076436233521, + 1.8132607741546631, + 1.8171432834243775, + 1.8106282345199585, + 1.8068263914489746, + 1.8001747159194947, + 1.797019803085327, + 1.799570650062561, + 1.7992187567520141, + 1.7956272640228272, + 1.7893521131134034, + 1.7947827518463135, + 1.792629102745056, + 1.7834515933990478, + 1.784611135520935, + 1.7758458992767334, + 1.77485143119812, + 1.7703186184692383, + 1.7679582196044923, + 1.7615163513946532, + 1.760532038192749, + 1.7643762328338624, + 1.7579109369659425, + 1.755184786720276, + 1.7560388659286499, + 1.7505403981781005, + 1.7491300173950195, + 1.7485786712646485, + 1.7421914972686767, + 1.7423645967864991, + 1.7408842813110352, + 1.7322719787979126, + 1.7337682635498046, + 1.7296864183807372, + 1.7320984534454347, + 1.7284361850357055, + 1.7295022455596925, + 1.7249998714447021, + 1.7229057648086548, + 1.7212110678863526, + 1.72249207862854, + 1.7190113651275636, + 1.717387024230957, + 1.7169812508392335, + 1.7162931756210327, + 1.7126198623275757, + 1.7117092602539064, + 1.7112801861190796, + 1.7114282751846313, + 1.7101980803680419, + 1.7129489760971068, + 1.7100260018539428, + 1.7092081976699829, + 1.703683881187439, + 1.7068217246246338, + 1.7049527828216553, + 1.7057187732315064, + 1.7080302432250976, + 1.7038126770401, + 1.7049555835723877, + 1.7032458050155639, + 1.7034490664672852, + 1.7006872876739503, + 1.7008496173095704, + 1.7032257048797608, + 1.6975849852371216, + 1.7016590633392334, + 1.7036013817596436, + 1.7003546350097656, + 1.6987783340835572, + 1.7004486318969727, + 1.6981003344726562, + 1.6952576486968993, + 1.6982201540374755, + 1.6985680112075805, + 1.6973068224334718, + 1.698023046951294, + 1.6962739462661742, + 1.7007662552642822, + 1.694658960800171, + 1.6946139693069457, + 1.6979838909912108 + ], + "train_acc": [ + 0.24304, + 0.27768, + 0.29718, + 0.3114, + 0.3155, + 0.32342, + 0.32646, + 0.32848, + 0.33164, + 0.33126, + 0.33432, + 0.33792, + 0.33882, + 0.33816, + 0.34222, + 0.34396, + 0.34814, + 0.34378, + 0.34546, + 0.3467, + 0.34866, + 0.35218, + 0.3533, + 0.3518, + 0.35448, + 0.3562, + 0.35548, + 0.358, + 0.35764, + 0.35846, + 0.35716, + 0.3626, + 0.36082, + 0.36478, + 0.36298, + 0.36532, + 0.36858, + 0.36974, + 0.37074, + 0.3673, + 0.37104, + 0.372, + 0.36924, + 0.37348, + 0.37412, + 0.37354, + 0.37672, + 0.37626, + 0.3795, + 0.3796, + 0.37736, + 0.37876, + 0.37752, + 0.38226, + 0.3828, + 0.38434, + 0.38232, + 0.38234, + 0.38028, + 0.38442, + 0.38296, + 0.38464, + 0.3859, + 0.38794, + 0.38652, + 0.3878, + 0.38752, + 0.38512, + 0.3885, + 0.38786, + 0.38678, + 0.3892, + 0.38872, + 0.39076, + 0.38828, + 0.38848, + 0.38858, + 0.38922, + 0.38946, + 0.39026, + 0.38864, + 0.38984, + 0.39182, + 0.3923, + 0.39104, + 0.39152, + 0.39114, + 0.3885, + 0.39122, + 0.39272, + 0.39182, + 0.39102, + 0.39104, + 0.3916, + 0.39454, + 0.39062, + 0.38996, + 0.39216, + 0.39174, + 0.39228 + ], + "test_acc": [ + 0.2729, + 0.3136, + 0.3293, + 0.3473, + 0.3313, + 0.3519, + 0.3629, + 0.3562, + 0.3516, + 0.3661, + 0.3733, + 0.3723, + 0.3632, + 0.3667, + 0.3676, + 0.3685, + 0.3723, + 0.3608, + 0.3781, + 0.3773, + 0.3671, + 0.3711, + 0.3595, + 0.377, + 0.3849, + 0.3701, + 0.3727, + 0.3603, + 0.3725, + 0.382, + 0.3783, + 0.3762, + 0.3832, + 0.3806, + 0.387, + 0.3873, + 0.3862, + 0.3864, + 0.3908, + 0.3939, + 0.3997, + 0.396, + 0.393, + 0.3846, + 0.3913, + 0.3967, + 0.3972, + 0.4013, + 0.3994, + 0.4043, + 0.3988, + 0.4024, + 0.3951, + 0.3982, + 0.3985, + 0.4081, + 0.4033, + 0.4029, + 0.4082, + 0.3986, + 0.4028, + 0.405, + 0.4054, + 0.4051, + 0.408, + 0.4069, + 0.4066, + 0.4085, + 0.4073, + 0.4096, + 0.4103, + 0.4066, + 0.4131, + 0.4106, + 0.4108, + 0.41, + 0.4121, + 0.41, + 0.4102, + 0.4076, + 0.4129, + 0.4109, + 0.4094, + 0.4103, + 0.41, + 0.4094, + 0.4121, + 0.4119, + 0.4126, + 0.4125, + 0.4116, + 0.4109, + 0.4128, + 0.4123, + 0.4129, + 0.4113, + 0.411, + 0.4114, + 0.4114, + 0.4115 + ] + }, + "diagnostics": { + "bp_cosine": [ + 0.03677885979413986, + 0.03784121572971344, + 0.08952198922634125, + 0.04931114614009857, + -0.00528424559161067, + 0.03716714680194855, + -0.02366219088435173, + -0.042923711240291595, + -0.07109162956476212, + -0.028443587943911552, + 0.01791619509458542, + 0.9962450861930847 + ], + "perturbation_rho": [ + 0.0005302987992763519, + 0.012386747635900974, + 0.023949606344103813, + 0.006188662722706795, + 0.05080874264240265, + -0.01257331669330597, + -0.003029853105545044, + 0.003304736688733101, + 0.043240927159786224, + 0.0031258384697139263, + -0.018814031034708023, + -0.018657812848687172 + ], + "nudging": { + "0.001": [ + -3.0247028917074203e-06, + -1.51805579662323e-07, + -1.8300488591194153e-07, + -1.150183379650116e-07, + -8.847564458847046e-09, + -3.64379957318306e-08, + 9.778887033462524e-09, + 3.096647560596466e-08, + 5.3551048040390015e-08, + -1.3737007975578308e-08, + -5.820766091346741e-09, + -7.356284186244011e-07 + ], + "0.003": [ + -9.037903510034084e-06, + -4.105968400835991e-07, + -4.7474168241024017e-07, + -2.3958273231983185e-07, + -1.0593794286251068e-08, + -1.551816239953041e-07, + 3.8067810237407684e-08, + 1.2014061212539673e-07, + 1.5227124094963074e-07, + 1.1094380170106888e-07, + -8.183997124433517e-08, + -2.6408815756440163e-06 + ], + "0.01": [ + -3.033224493265152e-05, + -1.35018490254879e-06, + -1.6904668882489204e-06, + -6.724148988723755e-07, + 3.341119736433029e-08, + -5.158362910151482e-07, + 1.4738179743289948e-07, + 3.360910341143608e-07, + 6.420304998755455e-07, + 2.4598557502031326e-07, + -2.3667234927415848e-07, + -9.222421795129776e-06 + ] + }, + "hidden_norms_per_layer": [ + 6368.87646484375, + 60059.984375, + 180441.8125, + 274150.96875, + 517201.65625, + 857839.4375, + 862732.0, + 1219729.5, + 1498813.125, + 1794381.5, + 1887047.25, + 1900776.375, + 1523186.0 + ], + "bp_grad_norms_per_layer": [ + 3.379595364094712e-05, + 2.1159430616535246e-06, + 6.931001053089858e-07, + 4.934564117320406e-07, + 4.4718211711369804e-07, + 4.4051026293345785e-07, + 4.3748320877057267e-07, + 4.3679304440047417e-07, + 4.368349380001746e-07, + 4.3717255948649836e-07, + 4.362396737178642e-07, + 4.269320186267578e-07, + 4.208905863833934e-07 + ] + }, + "drift": { + "embed.weight": 43.40305026092655, + "embed.bias": 12.884723360242011, + "blocks.0.ln.weight": 1.1396957317801488, + "blocks.0.w1.weight": 15.815325196455687, + "blocks.0.w1.bias": 9.997007586540093, + "blocks.0.w2.weight": 51.54342945808938, + "blocks.1.ln.weight": 0.9665440839605698, + "blocks.1.w1.weight": 17.964098193399778, + "blocks.1.w1.bias": 7.662859769714302, + "blocks.1.w2.weight": 45.06560123113467, + "blocks.2.ln.weight": 0.7981284775892137, + "blocks.2.w1.weight": 16.830439973227175, + "blocks.2.w1.bias": 8.949450279000775, + "blocks.2.w2.weight": 46.74550325782192, + "blocks.3.ln.weight": 0.7868855526754271, + "blocks.3.w1.weight": 17.044252690417927, + "blocks.3.w1.bias": 13.670590135892402, + "blocks.3.w2.weight": 38.40914010958083, + "blocks.4.ln.weight": 0.6395411371371652, + "blocks.4.w1.weight": 17.982212663546917, + "blocks.4.w1.bias": 18.982959249125763, + "blocks.4.w2.weight": 28.60564989778907, + "blocks.5.ln.weight": 0.5328988611331281, + "blocks.5.w1.weight": 14.634932988244662, + "blocks.5.w1.bias": 10.974172599914699, + "blocks.5.w2.weight": 39.20486954308308, + "blocks.6.ln.weight": 0.5485338249117044, + "blocks.6.w1.weight": 18.159621784839892, + "blocks.6.w1.bias": 18.891505344910815, + "blocks.6.w2.weight": 28.93284323285378, + "blocks.7.ln.weight": 0.5894477945101706, + "blocks.7.w1.weight": 18.009195934295505, + "blocks.7.w1.bias": 18.37430010215222, + "blocks.7.w2.weight": 30.406749000767654, + "blocks.8.ln.weight": 0.6068497649428072, + "blocks.8.w1.weight": 19.638841310683983, + "blocks.8.w1.bias": 20.60396737480574, + "blocks.8.w2.weight": 33.89217379886503, + "blocks.9.ln.weight": 0.678936766492041, + "blocks.9.w1.weight": 16.2394672260477, + "blocks.9.w1.bias": 14.562794517299237, + "blocks.9.w2.weight": 50.979251931044914, + "blocks.10.ln.weight": 0.6040394973297021, + "blocks.10.w1.weight": 14.292818466799657, + "blocks.10.w1.bias": 9.974035975062883, + "blocks.10.w2.weight": 60.410553219844616, + "blocks.11.ln.weight": 0.6493019680977011, + "blocks.11.w1.weight": 15.455381797619573, + "blocks.11.w1.bias": 12.883608145619089, + "blocks.11.w2.weight": 64.05421036694803, + "out_ln.weight": 0.3508285396209464, + "out_head.weight": 6.026469963854456, + "out_head.bias": 0.7776800281309532 + } + } + }, + "config": { + "dataset": "cifar10", + "d_hidden": 512, + "num_blocks": 12, + "batch_size": 128, + "epochs": 100, + "lr": 0.001, + "lr_fb": 0.001, + "wd": 0.01, + "lam": 0.1, + "K": 4, + "sigma_bridge": 0.05, + "ema_momentum": 0.995, + "term_grad_weight": 1.0, + "seeds": [ + 9 + ], + "gpu": 0, + "output_dir": "results/fa_dfa_d512_L12_seed9", + "methods": [ + "fa", + "dfa" + ], + "random_targets": false, + "penalty_lam": 0.0, + "num_classes": 10 + } +}
\ No newline at end of file |