diff options
Diffstat (limited to 'results/fa_dfa_d512_L12_seed0/results_cifar10.json')
| -rw-r--r-- | results/fa_dfa_d512_L12_seed0/results_cifar10.json | 969 |
1 files changed, 969 insertions, 0 deletions
diff --git a/results/fa_dfa_d512_L12_seed0/results_cifar10.json b/results/fa_dfa_d512_L12_seed0/results_cifar10.json new file mode 100644 index 0000000..0938122 --- /dev/null +++ b/results/fa_dfa_d512_L12_seed0/results_cifar10.json @@ -0,0 +1,969 @@ +{ + "0": { + "dfa": { + "log": { + "train_loss": [ + 2.0727519187164307, + 2.0449732803344727, + 2.0343947634887694, + 2.0317279723739623, + 2.029190194129944, + 2.0266742729187013, + 2.022695488433838, + 2.0220756021881106, + 2.017278121871948, + 2.013867749862671, + 2.0141937175750733, + 2.0114837906265257, + 2.0090159717559812, + 2.006871261138916, + 2.0085684817504883, + 2.004453958129883, + 2.0050127017211916, + 2.004625142745972, + 2.001847840270996, + 2.0038710110855105, + 2.0015532569885255, + 2.001803522415161, + 2.0029406197357176, + 1.9984209733581544, + 1.9999778453063966, + 1.997875029144287, + 2.002456622467041, + 2.000510860939026, + 1.9972556433105468, + 1.9980127339935303, + 1.9964525595092772, + 1.9945909706878662, + 1.994441053237915, + 1.9951845182037353, + 1.995466745147705, + 1.992624706878662, + 1.9937170362091063, + 1.9947755416870117, + 1.9944197441101075, + 1.9952830042266845, + 1.9904139096832276, + 1.991399830932617, + 1.9951595304870606, + 1.9930326363754272, + 1.9904268531799316, + 1.9908876788330079, + 1.9936137868499755, + 1.9875998815917968, + 1.992842328338623, + 1.990330719833374, + 1.9921186029815674, + 1.9902390911102295, + 1.9888944667053223, + 1.9905146474456787, + 1.9888609869384766, + 1.9911943308258058, + 1.9893802894592285, + 1.9878305652618409, + 1.9877847812652587, + 1.9883021939849854, + 1.988304048690796, + 1.9873467050170899, + 1.9866977008438111, + 1.9872942990875244, + 1.9863059562683105, + 1.9860430270004272, + 1.9863528005981446, + 1.9857840370941162, + 1.9861036871337892, + 1.985937850112915, + 1.9876243210601807, + 1.9859837552642823, + 1.9863339701080323, + 1.9869920307922364, + 1.9855867569732666, + 1.984609903526306, + 1.9867486194229127, + 1.9843175145721434, + 1.9860691648101807, + 1.9842630282592773, + 1.986168493347168, + 1.9858933181762695, + 1.98474868850708, + 1.9866244787979126, + 1.9833790422058106, + 1.9852294052505493, + 1.9821201675033568, + 1.9854196563339233, + 1.9846102389526368, + 1.9820537503433227, + 1.981782052307129, + 1.984344421157837, + 1.983909532699585, + 1.9853303438949585, + 1.984236333694458, + 1.9829996084594728, + 1.984313102684021, + 1.9833407458496093, + 1.983147784729004, + 1.9833076025390626 + ], + "train_acc": [ + 0.2332, + 0.24532, + 0.2542, + 0.2564, + 0.25326, + 0.25504, + 0.25822, + 0.26022, + 0.26324, + 0.26234, + 0.26278, + 0.26454, + 0.26348, + 0.26876, + 0.26432, + 0.26712, + 0.268, + 0.26852, + 0.27126, + 0.26796, + 0.2701, + 0.27052, + 0.26954, + 0.27056, + 0.27256, + 0.27258, + 0.26872, + 0.27102, + 0.2732, + 0.27298, + 0.27452, + 0.27374, + 0.27488, + 0.273, + 0.27604, + 0.2756, + 0.27576, + 0.27472, + 0.27344, + 0.27474, + 0.27776, + 0.27826, + 0.27318, + 0.2762, + 0.27666, + 0.27602, + 0.2763, + 0.2777, + 0.27724, + 0.27716, + 0.27566, + 0.27832, + 0.28138, + 0.27918, + 0.27922, + 0.27668, + 0.27846, + 0.27786, + 0.27796, + 0.2762, + 0.27946, + 0.27984, + 0.2795, + 0.27926, + 0.28092, + 0.27924, + 0.27978, + 0.28098, + 0.28132, + 0.2802, + 0.27814, + 0.28184, + 0.28178, + 0.28284, + 0.28128, + 0.2829, + 0.28048, + 0.28194, + 0.28236, + 0.2801, + 0.28106, + 0.28278, + 0.28308, + 0.27886, + 0.2815, + 0.28188, + 0.28254, + 0.28202, + 0.28158, + 0.28336, + 0.2825, + 0.28316, + 0.28312, + 0.28218, + 0.28288, + 0.2816, + 0.28218, + 0.2807, + 0.2835, + 0.28354 + ], + "test_acc": [ + 0.2507, + 0.2596, + 0.274, + 0.2704, + 0.2841, + 0.2803, + 0.2788, + 0.2781, + 0.303, + 0.2941, + 0.2905, + 0.3016, + 0.2779, + 0.3052, + 0.2819, + 0.2911, + 0.2931, + 0.2886, + 0.2812, + 0.2924, + 0.2947, + 0.2701, + 0.28, + 0.2912, + 0.3037, + 0.2894, + 0.3012, + 0.3011, + 0.3056, + 0.2981, + 0.2923, + 0.2961, + 0.3082, + 0.3035, + 0.2868, + 0.2981, + 0.3042, + 0.2956, + 0.3012, + 0.296, + 0.3077, + 0.3082, + 0.3, + 0.3141, + 0.2967, + 0.3024, + 0.3004, + 0.3016, + 0.308, + 0.3028, + 0.3072, + 0.2933, + 0.2907, + 0.3126, + 0.3036, + 0.2973, + 0.3001, + 0.3054, + 0.3115, + 0.3004, + 0.3026, + 0.3022, + 0.3051, + 0.3109, + 0.2968, + 0.3092, + 0.2998, + 0.3062, + 0.3081, + 0.3035, + 0.3032, + 0.3114, + 0.3147, + 0.308, + 0.3035, + 0.3098, + 0.3091, + 0.3065, + 0.3094, + 0.3071, + 0.3072, + 0.309, + 0.3012, + 0.3067, + 0.3016, + 0.3028, + 0.3048, + 0.3083, + 0.3082, + 0.308, + 0.3097, + 0.3087, + 0.3117, + 0.3104, + 0.3086, + 0.3085, + 0.3088, + 0.3084, + 0.3085, + 0.3086 + ] + }, + "diagnostics": { + "bp_cosine": [ + 0.395052969455719, + 0.0004171400796622038, + 8.336821338161826e-06, + 0.0005622187163680792, + 0.00013701531861443073, + -0.00035643568844534457, + 0.00012600264744833112, + -0.00027335749473422766, + 0.00016040689661167562, + -6.0928698076168075e-05, + 3.278384974692017e-05, + 0.0003701794194057584 + ], + "perturbation_rho": [ + 0.018501652404665947, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "nudging": { + "0.001": [ + -4.1211023926734924e-07, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "0.003": [ + -1.2032687664031982e-06, + 0.0, + 0.0, + 0.0, + 0.0, + 3.725290298461914e-09, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "0.01": [ + -4.2067840695381165e-06, + -9.313225746154785e-10, + -2.7939677238464355e-09, + 9.313225746154785e-10, + 0.0, + 4.6566128730773926e-09, + 1.862645149230957e-09, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ] + }, + "hidden_norms_per_layer": [ + 52639.97265625, + 1088866944.0, + 4048803840.0, + 5257923584.0, + 5847087616.0, + 5938909184.0, + 7423066624.0, + 9731870720.0, + 11342387200.0, + 11317005312.0, + 11353008128.0, + 12463034368.0, + 13189959680.0 + ], + "bp_grad_norms_per_layer": [ + 2.804916334753216e-07, + 1.9205773083008637e-10, + 1.9170887100017353e-10, + 1.9189858035950635e-10, + 1.919657766080718e-10, + 1.919622516499686e-10, + 1.919622516499686e-10, + 1.9191119526862366e-10, + 1.9192891720365424e-10, + 1.919416292572862e-10, + 1.9193641120907046e-10, + 1.9192181177629664e-10, + 1.9190903033372564e-10 + ] + }, + "drift": { + "embed.weight": 330.26007887921685, + "embed.bias": 274.2168771279626, + "blocks.0.ln.weight": 10.94628999409354, + "blocks.0.w1.weight": 277.3643033410537, + "blocks.0.w1.bias": 261.86497048407, + "blocks.0.w2.weight": 484.6109947045105, + "blocks.1.ln.weight": 9.35886499546542, + "blocks.1.w1.weight": 360.26286845371646, + "blocks.1.w1.bias": 355.66126981713177, + "blocks.1.w2.weight": 392.1776082515924, + "blocks.2.ln.weight": 9.350239389888953, + "blocks.2.w1.weight": 370.0336198354507, + "blocks.2.w1.bias": 337.8932267522991, + "blocks.2.w2.weight": 352.1258021274022, + "blocks.3.ln.weight": 9.767929501471052, + "blocks.3.w1.weight": 331.5909899943081, + "blocks.3.w1.bias": 303.3604214345198, + "blocks.3.w2.weight": 312.3336342294667, + "blocks.4.ln.weight": 6.8894143383412505, + "blocks.4.w1.weight": 245.69582220582208, + "blocks.4.w1.bias": 226.6077326792198, + "blocks.4.w2.weight": 237.1578303477898, + "blocks.5.ln.weight": 10.087708773364, + "blocks.5.w1.weight": 391.6451916277919, + "blocks.5.w1.bias": 364.5077034653599, + "blocks.5.w2.weight": 375.80268056062386, + "blocks.6.ln.weight": 10.894168165759849, + "blocks.6.w1.weight": 446.1083385424109, + "blocks.6.w1.bias": 408.24079100200436, + "blocks.6.w2.weight": 421.6066446657385, + "blocks.7.ln.weight": 10.280446652077675, + "blocks.7.w1.weight": 417.5786268538863, + "blocks.7.w1.bias": 412.3650615213133, + "blocks.7.w2.weight": 413.1057523107626, + "blocks.8.ln.weight": 7.628061136196448, + "blocks.8.w1.weight": 280.87597195145304, + "blocks.8.w1.bias": 261.4439841235217, + "blocks.8.w2.weight": 269.85556299140023, + "blocks.9.ln.weight": 7.035433277618009, + "blocks.9.w1.weight": 236.54101277402214, + "blocks.9.w1.bias": 227.70919757896647, + "blocks.9.w2.weight": 225.58244999977697, + "blocks.10.ln.weight": 10.07787136157841, + "blocks.10.w1.weight": 405.6014059372381, + "blocks.10.w1.bias": 387.7928293241623, + "blocks.10.w2.weight": 391.3899843395933, + "blocks.11.ln.weight": 9.615189319853535, + "blocks.11.w1.weight": 379.5363902874608, + "blocks.11.w1.bias": 358.37006314701347, + "blocks.11.w2.weight": 355.59941733335387, + "out_ln.weight": 0.6824993975107903, + "out_head.weight": 9.861581662622491, + "out_head.bias": 0.35061659338833756 + } + }, + "fa": { + "log": { + "train_loss": [ + 2.0293911264038087, + 1.950565330429077, + 1.917781063232422, + 1.9023740824127198, + 1.8861673935699463, + 1.8825682291412353, + 1.873201879234314, + 1.8714005821990967, + 1.8628303081512452, + 1.8579199737548828, + 1.8572168459320069, + 1.8525300827789306, + 1.845312434463501, + 1.8409330677032472, + 1.8335270806121826, + 1.822513469543457, + 1.8199686001205444, + 1.8136085485458373, + 1.8064511907196046, + 1.803108533859253, + 1.7962512873077392, + 1.7911701647949219, + 1.7911696646118165, + 1.7887402313995362, + 1.783379939842224, + 1.7759505123901367, + 1.780573166732788, + 1.7734921991729737, + 1.7670989339828491, + 1.7612831104278563, + 1.7572847888183594, + 1.753526630859375, + 1.750715301132202, + 1.7472047467041016, + 1.7457971502304077, + 1.7424137032699585, + 1.7345080102920531, + 1.734222209701538, + 1.7304457287216186, + 1.7239187670898437, + 1.7218291164398194, + 1.7199537884902953, + 1.7217532263565063, + 1.7196246533584594, + 1.7135296297836304, + 1.706606083908081, + 1.7079890319824218, + 1.7041183599472045, + 1.705560855026245, + 1.7011269199371337, + 1.7035633499908447, + 1.6992430791854858, + 1.6950508019638062, + 1.7004030841064453, + 1.691479129295349, + 1.691718464050293, + 1.6919280102157592, + 1.6887580330657959, + 1.6883217154693604, + 1.688851436805725, + 1.6853341622543334, + 1.680581435546875, + 1.678009653892517, + 1.6784401907730102, + 1.6803946784210204, + 1.6761323783493043, + 1.676516137046814, + 1.6778211946868897, + 1.6767865496444703, + 1.6731242126083374, + 1.675690205078125, + 1.669817829246521, + 1.6760586986923218, + 1.670416958580017, + 1.6687565314483643, + 1.668258262901306, + 1.6666069919204711, + 1.666242490005493, + 1.6675299571990967, + 1.663490775527954, + 1.6672910994720458, + 1.6639445114517213, + 1.6617960903167726, + 1.6636806215667725, + 1.6643552365112304, + 1.6602695792007447, + 1.6589048775100708, + 1.6614284314727783, + 1.6610555962753295, + 1.6578762405014038, + 1.6580958423614502, + 1.6621724097061157, + 1.6586555422210694, + 1.657684158668518, + 1.6587366760253905, + 1.657404851913452, + 1.6599628324127198, + 1.6565218897247314, + 1.6586303088760377, + 1.6615223189544677 + ], + "train_acc": [ + 0.2526, + 0.29054, + 0.30538, + 0.31188, + 0.31418, + 0.31852, + 0.32254, + 0.3243, + 0.32728, + 0.33104, + 0.33096, + 0.33338, + 0.33726, + 0.33962, + 0.3428, + 0.34614, + 0.34728, + 0.34656, + 0.35102, + 0.35328, + 0.35598, + 0.35864, + 0.35488, + 0.35678, + 0.35954, + 0.36188, + 0.36066, + 0.36522, + 0.36488, + 0.36676, + 0.36986, + 0.3702, + 0.3739, + 0.37346, + 0.37434, + 0.37574, + 0.3752, + 0.3788, + 0.37904, + 0.38092, + 0.38054, + 0.38164, + 0.37934, + 0.38172, + 0.38678, + 0.3875, + 0.38734, + 0.38996, + 0.38752, + 0.39106, + 0.38978, + 0.39084, + 0.39086, + 0.39326, + 0.39296, + 0.39256, + 0.39342, + 0.39414, + 0.39416, + 0.39432, + 0.39614, + 0.39536, + 0.3965, + 0.39922, + 0.39722, + 0.39968, + 0.3982, + 0.39886, + 0.40146, + 0.4016, + 0.40122, + 0.4017, + 0.39858, + 0.40386, + 0.40168, + 0.4028, + 0.4026, + 0.4035, + 0.40378, + 0.40566, + 0.4031, + 0.4049, + 0.40506, + 0.4035, + 0.40618, + 0.40566, + 0.406, + 0.40476, + 0.40564, + 0.40724, + 0.40646, + 0.4061, + 0.4079, + 0.40568, + 0.40498, + 0.40842, + 0.40674, + 0.40746, + 0.40728, + 0.40684 + ], + "test_acc": [ + 0.2924, + 0.3363, + 0.3349, + 0.3386, + 0.347, + 0.3457, + 0.3423, + 0.3555, + 0.3653, + 0.3606, + 0.3632, + 0.3617, + 0.3575, + 0.3723, + 0.3703, + 0.3715, + 0.3741, + 0.3774, + 0.374, + 0.3745, + 0.3836, + 0.3907, + 0.3702, + 0.3831, + 0.385, + 0.3843, + 0.3922, + 0.4003, + 0.3967, + 0.3978, + 0.3897, + 0.3828, + 0.401, + 0.4012, + 0.3991, + 0.3999, + 0.4055, + 0.4085, + 0.3968, + 0.4064, + 0.4066, + 0.4082, + 0.404, + 0.4042, + 0.4085, + 0.4091, + 0.4104, + 0.4155, + 0.4157, + 0.4125, + 0.4139, + 0.4166, + 0.4116, + 0.4187, + 0.4186, + 0.4123, + 0.4161, + 0.4171, + 0.4184, + 0.414, + 0.4179, + 0.4184, + 0.4197, + 0.4193, + 0.421, + 0.4205, + 0.423, + 0.4233, + 0.4201, + 0.422, + 0.4204, + 0.4151, + 0.4182, + 0.4218, + 0.4239, + 0.4198, + 0.425, + 0.4239, + 0.4204, + 0.4215, + 0.4253, + 0.4199, + 0.4235, + 0.4206, + 0.4235, + 0.4255, + 0.4222, + 0.423, + 0.4216, + 0.426, + 0.4245, + 0.4245, + 0.4239, + 0.4248, + 0.4224, + 0.4245, + 0.4246, + 0.4241, + 0.4242, + 0.4239 + ] + }, + "diagnostics": { + "bp_cosine": [ + 0.005966864060610533, + 0.04795818775892258, + 0.043208200484514236, + -0.05298762395977974, + -0.11785908043384552, + -0.05540306121110916, + -0.10012105107307434, + -0.09065192937850952, + -0.04056818038225174, + -0.03824863210320473, + -0.07007172703742981, + 0.9948829412460327 + ], + "perturbation_rho": [ + 0.028052061796188354, + 0.03165658190846443, + -0.006041648797690868, + -0.027337782084941864, + 0.01562969572842121, + 0.01757640205323696, + -0.025423342362046242, + -0.024989785626530647, + -0.03021504357457161, + 0.021996641531586647, + 0.03194758668541908, + 0.01976931467652321 + ], + "nudging": { + "0.001": [ + 3.7101563066244125e-07, + -6.126938387751579e-07, + -3.066379576921463e-07, + 1.6076955944299698e-07, + 3.3061951398849487e-07, + 1.5937257558107376e-07, + 2.9441434890031815e-07, + 2.859160304069519e-07, + 1.284060999751091e-07, + 1.5040859580039978e-07, + 2.1443702280521393e-07, + -3.2152747735381126e-06 + ], + "0.003": [ + 1.007108949124813e-06, + -1.6828998923301697e-06, + -6.601912900805473e-07, + 4.4493936002254486e-07, + 1.1789379641413689e-06, + 5.08967787027359e-07, + 1.0151416063308716e-06, + 8.791685104370117e-07, + 3.866152837872505e-07, + 3.688037395477295e-07, + 6.126938387751579e-07, + -1.0434770956635475e-05 + ], + "0.01": [ + 3.6874553188681602e-06, + -5.368026904761791e-06, + -2.2364547476172447e-06, + 1.735752448439598e-06, + 3.992114216089249e-06, + 1.860782504081726e-06, + 3.5137636587023735e-06, + 3.010733053088188e-06, + 1.437612809240818e-06, + 1.2825476005673409e-06, + 2.246466465294361e-06, + -3.491528332233429e-05 + ] + }, + "hidden_norms_per_layer": [ + 4702.6123046875, + 44530.10546875, + 108911.2109375, + 164933.125, + 283322.40625, + 377155.46875, + 407664.6875, + 517485.0625, + 562702.0625, + 564809.5, + 577993.1875, + 617885.5625, + 296847.03125 + ], + "bp_grad_norms_per_layer": [ + 4.941995575791225e-05, + 3.9530186768388376e-06, + 1.528447455712012e-06, + 1.3358264823182253e-06, + 1.3305660786500084e-06, + 1.3287758520164061e-06, + 1.3350502285902621e-06, + 1.339361574537179e-06, + 1.3375677099247696e-06, + 1.3184284171074978e-06, + 1.3012635236009373e-06, + 1.3201109823057777e-06, + 1.27659131976543e-06 + ] + }, + "drift": { + "embed.weight": 36.945692873367385, + "embed.bias": 13.300137685811876, + "blocks.0.ln.weight": 0.901147599699004, + "blocks.0.w1.weight": 13.367758537525685, + "blocks.0.w1.bias": 10.72336098907488, + "blocks.0.w2.weight": 46.304733049049524, + "blocks.1.ln.weight": 0.7424575473753766, + "blocks.1.w1.weight": 15.226852479991834, + "blocks.1.w1.bias": 7.345119575250527, + "blocks.1.w2.weight": 40.81729281725618, + "blocks.2.ln.weight": 0.6752024292095198, + "blocks.2.w1.weight": 13.877425620596014, + "blocks.2.w1.bias": 7.829951186600198, + "blocks.2.w2.weight": 39.52886223656965, + "blocks.3.ln.weight": 0.6072219216215275, + "blocks.3.w1.weight": 13.849387698880346, + "blocks.3.w1.bias": 11.302805784789012, + "blocks.3.w2.weight": 31.48991587682597, + "blocks.4.ln.weight": 0.3513996294280428, + "blocks.4.w1.weight": 12.017687634070212, + "blocks.4.w1.bias": 12.545759692163585, + "blocks.4.w2.weight": 19.78672753090035, + "blocks.5.ln.weight": 0.27340079470413037, + "blocks.5.w1.weight": 10.049144366681992, + "blocks.5.w1.bias": 9.943637751012846, + "blocks.5.w2.weight": 19.227980350358227, + "blocks.6.ln.weight": 0.3688735234588941, + "blocks.6.w1.weight": 12.460300257534458, + "blocks.6.w1.bias": 13.383277160534908, + "blocks.6.w2.weight": 19.760938996380943, + "blocks.7.ln.weight": 0.34359755221214905, + "blocks.7.w1.weight": 11.768674295896531, + "blocks.7.w1.bias": 13.150434923761187, + "blocks.7.w2.weight": 21.174060669259852, + "blocks.8.ln.weight": 0.3997077790355334, + "blocks.8.w1.weight": 11.48289450791943, + "blocks.8.w1.bias": 8.028224303872328, + "blocks.8.w2.weight": 35.011661428887535, + "blocks.9.ln.weight": 0.43894400900661307, + "blocks.9.w1.weight": 11.84935422288265, + "blocks.9.w1.bias": 8.899824293598352, + "blocks.9.w2.weight": 34.666647832790865, + "blocks.10.ln.weight": 0.4701858246153278, + "blocks.10.w1.weight": 12.779979416376127, + "blocks.10.w1.bias": 12.40660396279328, + "blocks.10.w2.weight": 30.770398182409856, + "blocks.11.ln.weight": 0.502840402610306, + "blocks.11.w1.weight": 15.51482213849637, + "blocks.11.w1.bias": 16.777463254010712, + "blocks.11.w2.weight": 27.32990799926724, + "out_ln.weight": 0.2952693196309993, + "out_head.weight": 4.4773532602382, + "out_head.bias": 1.8150038464128413 + } + } + }, + "config": { + "dataset": "cifar10", + "d_hidden": 512, + "num_blocks": 12, + "batch_size": 128, + "epochs": 100, + "lr": 0.001, + "lr_fb": 0.001, + "wd": 0.01, + "lam": 0.1, + "K": 4, + "sigma_bridge": 0.05, + "ema_momentum": 0.995, + "term_grad_weight": 1.0, + "seeds": [ + 0 + ], + "gpu": 0, + "output_dir": "results/fa_dfa_d512_L12_seed0", + "methods": [ + "fa", + "dfa" + ], + "random_targets": false, + "penalty_lam": 0.0, + "num_classes": 10 + } +}
\ No newline at end of file |