diff options
Diffstat (limited to 'results/fa_dfa_d256_L12_seed4/results_cifar10.json')
| -rw-r--r-- | results/fa_dfa_d256_L12_seed4/results_cifar10.json | 969 |
1 files changed, 969 insertions, 0 deletions
diff --git a/results/fa_dfa_d256_L12_seed4/results_cifar10.json b/results/fa_dfa_d256_L12_seed4/results_cifar10.json new file mode 100644 index 0000000..6ce051d --- /dev/null +++ b/results/fa_dfa_d256_L12_seed4/results_cifar10.json @@ -0,0 +1,969 @@ +{ + "4": { + "dfa": { + "log": { + "train_loss": [ + 2.08256451877594, + 2.0663339112854002, + 2.065528285903931, + 2.0641389405822754, + 2.063766109161377, + 2.0643044331359865, + 2.061108066558838, + 2.0643930868148805, + 2.061966312408447, + 2.060680084762573, + 2.058289337158203, + 2.05898098777771, + 2.0577542728424074, + 2.0568669985198973, + 2.056451043395996, + 2.054366501312256, + 2.0567925134277343, + 2.053527456817627, + 2.05231096206665, + 2.054566929168701, + 2.0525731983947755, + 2.0498655393218996, + 2.0522367533874513, + 2.0521925598526, + 2.05059351852417, + 2.0510073263931274, + 2.0483419913482668, + 2.0495694082641602, + 2.049738536529541, + 2.0502711264038087, + 2.0500267957305907, + 2.0491143457794188, + 2.050376922607422, + 2.049438688735962, + 2.048919423675537, + 2.0511557712554933, + 2.050678148727417, + 2.051541336364746, + 2.0506073697280884, + 2.0506765252685546, + 2.0498380197906494, + 2.049806067466736, + 2.0485455421447756, + 2.0491862691497804, + 2.0492898570251463, + 2.0476144123840334, + 2.0490683988189695, + 2.0513844109344483, + 2.0487480349731446, + 2.04999566116333, + 2.047838335647583, + 2.049221919898987, + 2.0486966360092165, + 2.0475039793395995, + 2.0491980659484863, + 2.04866181350708, + 2.047751593399048, + 2.0454024687194825, + 2.0468619202423097, + 2.0493145092010496, + 2.048052859725952, + 2.04698764465332, + 2.0469833773040773, + 2.0464803858947755, + 2.0463906023406984, + 2.047911610183716, + 2.048891339149475, + 2.046190006866455, + 2.047377264175415, + 2.0476539112091063, + 2.0468983917617796, + 2.046835005149841, + 2.047859336853027, + 2.044651624069214, + 2.0467517770385744, + 2.04683432472229, + 2.0458193408966063, + 2.0458727869415285, + 2.0462171686553954, + 2.045814103355408, + 2.0447903966522216, + 2.0451703567504884, + 2.046430616149902, + 2.044895040740967, + 2.0480339848327636, + 2.044828313446045, + 2.0458129290008547, + 2.0457765758514403, + 2.0444045190811155, + 2.044279132156372, + 2.046977679824829, + 2.047140994644165, + 2.0433586880111694, + 2.045490126800537, + 2.0467801708221436, + 2.0449837641906736, + 2.0445874029541016, + 2.0448050390625, + 2.0441628652191164, + 2.0462842389678957 + ], + "train_acc": [ + 0.23122, + 0.23646, + 0.23786, + 0.2353, + 0.23812, + 0.23576, + 0.23744, + 0.23778, + 0.23848, + 0.24008, + 0.23958, + 0.24138, + 0.23976, + 0.24192, + 0.2421, + 0.24358, + 0.24318, + 0.246, + 0.2423, + 0.24362, + 0.24622, + 0.24714, + 0.24544, + 0.24618, + 0.24696, + 0.24756, + 0.2518, + 0.24818, + 0.24998, + 0.24534, + 0.24778, + 0.25146, + 0.25022, + 0.24854, + 0.25064, + 0.24894, + 0.25102, + 0.2495, + 0.25118, + 0.24816, + 0.2509, + 0.25058, + 0.25294, + 0.24896, + 0.24964, + 0.25192, + 0.25154, + 0.2498, + 0.25128, + 0.25344, + 0.25054, + 0.25238, + 0.2517, + 0.25266, + 0.2531, + 0.25136, + 0.25094, + 0.25286, + 0.25358, + 0.25226, + 0.25338, + 0.25504, + 0.2549, + 0.2549, + 0.25466, + 0.25336, + 0.25376, + 0.25556, + 0.25546, + 0.2527, + 0.25446, + 0.25494, + 0.2543, + 0.25412, + 0.25538, + 0.25468, + 0.25604, + 0.25698, + 0.2563, + 0.25708, + 0.25672, + 0.25804, + 0.2562, + 0.25584, + 0.25736, + 0.25728, + 0.25424, + 0.2564, + 0.25502, + 0.25512, + 0.25496, + 0.25608, + 0.25874, + 0.25556, + 0.25554, + 0.25712, + 0.2583, + 0.25732, + 0.25662, + 0.25448 + ], + "test_acc": [ + 0.2652, + 0.2592, + 0.2598, + 0.24, + 0.2463, + 0.2675, + 0.2477, + 0.2644, + 0.2193, + 0.2485, + 0.2463, + 0.2506, + 0.2573, + 0.261, + 0.2639, + 0.2619, + 0.2542, + 0.2491, + 0.2607, + 0.2572, + 0.2644, + 0.2546, + 0.2564, + 0.2631, + 0.2536, + 0.2618, + 0.2623, + 0.2654, + 0.2622, + 0.259, + 0.2608, + 0.2555, + 0.2803, + 0.2565, + 0.2723, + 0.2608, + 0.2661, + 0.2633, + 0.2512, + 0.2564, + 0.2686, + 0.2678, + 0.2687, + 0.266, + 0.2684, + 0.2691, + 0.2695, + 0.2661, + 0.2716, + 0.269, + 0.2681, + 0.2739, + 0.2661, + 0.2627, + 0.2673, + 0.264, + 0.2703, + 0.2719, + 0.2653, + 0.2734, + 0.2601, + 0.2622, + 0.2775, + 0.2672, + 0.2615, + 0.2804, + 0.2608, + 0.2719, + 0.2634, + 0.2672, + 0.2714, + 0.2629, + 0.2664, + 0.2664, + 0.2719, + 0.2688, + 0.2724, + 0.2737, + 0.2737, + 0.2663, + 0.2694, + 0.272, + 0.271, + 0.2699, + 0.2654, + 0.2695, + 0.2711, + 0.2699, + 0.2695, + 0.2719, + 0.2682, + 0.2674, + 0.2688, + 0.2692, + 0.2683, + 0.2681, + 0.2691, + 0.2695, + 0.2694, + 0.2695 + ] + }, + "diagnostics": { + "bp_cosine": [ + 0.3369702696800232, + 0.0017256045248359442, + 0.0003115047584287822, + 0.0012309218291193247, + 0.0004706716863438487, + -0.0003515754360705614, + 0.0010358416475355625, + 0.00042472450877539814, + -0.00047611017362214625, + -0.001211655791848898, + -0.0013567307032644749, + 0.0005671238759532571 + ], + "perturbation_rho": [ + 0.008996784687042236, + 0.0, + 0.0, + -0.004853670950978994, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "nudging": { + "0.001": [ + -1.7974525690078735e-07, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "0.003": [ + -6.239861249923706e-07, + 0.0, + 0.0, + -1.862645149230957e-09, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "0.01": [ + -2.148095518350601e-06, + 7.450580596923828e-09, + -3.725290298461914e-09, + -5.587935447692871e-09, + 0.0, + 0.0, + 1.862645149230957e-09, + 0.0, + 0.0, + 0.0, + -1.862645149230957e-09, + 0.0 + ] + }, + "hidden_norms_per_layer": [ + 50053.5078125, + 506126208.0, + 839627776.0, + 1433945984.0, + 2151404288.0, + 2646801152.0, + 2618418688.0, + 2616808960.0, + 2656328704.0, + 2846460672.0, + 3113679360.0, + 3486919424.0, + 3484421376.0 + ], + "bp_grad_norms_per_layer": [ + 2.0373111908611463e-07, + 5.87830117915189e-10, + 5.875596120752391e-10, + 5.888394216668758e-10, + 5.894798538186308e-10, + 5.895269272748749e-10, + 5.894864596456273e-10, + 5.894754129265323e-10, + 5.893150412106252e-10, + 5.894901788927598e-10, + 5.895146593104528e-10, + 5.898119770364474e-10, + 5.898875832244244e-10 + ] + }, + "drift": { + "embed.weight": 406.5158433791385, + "embed.bias": 328.00460641981374, + "blocks.0.ln.weight": 10.621206283569336, + "blocks.0.w1.weight": 260.1775263539264, + "blocks.0.w1.bias": 261.5553235769218, + "blocks.0.w2.weight": 546.8623769606548, + "blocks.1.ln.weight": 9.573291778564453, + "blocks.1.w1.weight": 254.1321472468791, + "blocks.1.w1.bias": 229.33671096787808, + "blocks.1.w2.weight": 375.8430206058194, + "blocks.2.ln.weight": 10.723036766052246, + "blocks.2.w1.weight": 312.23440184847425, + "blocks.2.w1.bias": 301.75305024697, + "blocks.2.w2.weight": 465.9606227271888, + "blocks.3.ln.weight": 11.675829887390137, + "blocks.3.w1.weight": 329.12792372114694, + "blocks.3.w1.bias": 336.1713580136849, + "blocks.3.w2.weight": 479.62371125689094, + "blocks.4.ln.weight": 12.166597366333008, + "blocks.4.w1.weight": 342.5184155405496, + "blocks.4.w1.bias": 331.25442650567624, + "blocks.4.w2.weight": 473.58945710721196, + "blocks.5.ln.weight": 8.257390022277832, + "blocks.5.w1.weight": 224.17786513425023, + "blocks.5.w1.bias": 206.55166933090487, + "blocks.5.w2.weight": 310.58552361575914, + "blocks.6.ln.weight": 9.497365951538086, + "blocks.6.w1.weight": 257.28476030419915, + "blocks.6.w1.bias": 241.49147853808202, + "blocks.6.w2.weight": 326.8691329729593, + "blocks.7.ln.weight": 8.255859375, + "blocks.7.w1.weight": 200.14594495243745, + "blocks.7.w1.bias": 191.91816529358212, + "blocks.7.w2.weight": 257.1419920590999, + "blocks.8.ln.weight": 11.01574993133545, + "blocks.8.w1.weight": 307.85736409735574, + "blocks.8.w1.bias": 281.3908916662528, + "blocks.8.w2.weight": 408.1843979772796, + "blocks.9.ln.weight": 11.364969253540039, + "blocks.9.w1.weight": 305.9120817067219, + "blocks.9.w1.bias": 279.3079525981705, + "blocks.9.w2.weight": 396.62586594267685, + "blocks.10.ln.weight": 11.884243965148926, + "blocks.10.w1.weight": 334.91020121291007, + "blocks.10.w1.bias": 324.23156540936054, + "blocks.10.w2.weight": 491.4126848947247, + "blocks.11.ln.weight": 11.255571365356445, + "blocks.11.w1.weight": 308.006589638926, + "blocks.11.w1.bias": 292.5786848876635, + "blocks.11.w2.weight": 425.11018085204205, + "out_ln.weight": 0.7832708358764648, + "out_head.weight": 8.078985322489435, + "out_head.bias": 0.29867781036736046 + } + }, + "fa": { + "log": { + "train_loss": [ + 2.049650134963989, + 1.973029051513672, + 1.9459494330596925, + 1.9263736043548585, + 1.918336088027954, + 1.9090399783706664, + 1.9018680041122435, + 1.9004452823638915, + 1.8939352771759033, + 1.8905867196655273, + 1.88599944606781, + 1.8800309131622315, + 1.8739088614273072, + 1.8683558203125, + 1.8653341689300538, + 1.8583481198120118, + 1.8549240982055664, + 1.8496921838760376, + 1.8435221142959595, + 1.8398676053619385, + 1.8327734576416015, + 1.8336225759124756, + 1.8276614254379273, + 1.8216393673324585, + 1.8191773874664308, + 1.8113161963653563, + 1.8087092595672607, + 1.8040201818084718, + 1.7976001425170898, + 1.795108130493164, + 1.7909190823745726, + 1.7871669284820557, + 1.7862169911956787, + 1.7830527084350587, + 1.7802653800201416, + 1.7863243228912353, + 1.786163843460083, + 1.7841941988754273, + 1.7805864434814453, + 1.7733078927230834, + 1.7747596154022216, + 1.772979263381958, + 1.7702470769882201, + 1.766930933494568, + 1.7668507308197021, + 1.7599455773544312, + 1.7646458086395265, + 1.7616525787734985, + 1.760625428390503, + 1.7602823944473267, + 1.7544173859024048, + 1.7526818951797485, + 1.7567130640029907, + 1.7525626901245117, + 1.7489151987075806, + 1.7496014617919922, + 1.750472166748047, + 1.7476551281738282, + 1.7476194573974608, + 1.7486230560302733, + 1.7486302239227296, + 1.7496078895950318, + 1.745147467918396, + 1.745377778968811, + 1.7445024374771119, + 1.746578332748413, + 1.7441591314697265, + 1.7423080096817016, + 1.7424391454315185, + 1.7420111348724365, + 1.7456162438964844, + 1.740698592147827, + 1.74056429561615, + 1.7388315572738648, + 1.7365661661529541, + 1.7376085388565063, + 1.7377186611557007, + 1.7358070825958252, + 1.7348824306488038, + 1.7347428383636474, + 1.734776619567871, + 1.7334602453994752, + 1.7353667811965943, + 1.7360192542266846, + 1.7331659270477295, + 1.7365653960800171, + 1.735952963256836, + 1.7346072133255004, + 1.7329498879623413, + 1.7314477652740479, + 1.736275334815979, + 1.736558514175415, + 1.7318700998306273, + 1.7343199029159546, + 1.7351305802154542, + 1.7342404050064086, + 1.7299409299087525, + 1.730475319519043, + 1.7300176064300536, + 1.7322037688064575 + ], + "train_acc": [ + 0.24164, + 0.27262, + 0.28936, + 0.2973, + 0.2997, + 0.30452, + 0.30772, + 0.30798, + 0.31336, + 0.31634, + 0.3196, + 0.31996, + 0.32114, + 0.32864, + 0.32748, + 0.3284, + 0.33336, + 0.33396, + 0.33538, + 0.33648, + 0.34296, + 0.34042, + 0.3434, + 0.34368, + 0.34544, + 0.35258, + 0.35284, + 0.35284, + 0.3565, + 0.35634, + 0.35946, + 0.35858, + 0.3638, + 0.36276, + 0.36302, + 0.36144, + 0.36354, + 0.3642, + 0.36578, + 0.36918, + 0.36566, + 0.36788, + 0.36728, + 0.36864, + 0.3674, + 0.37124, + 0.36972, + 0.37258, + 0.37202, + 0.3714, + 0.37326, + 0.37558, + 0.37402, + 0.37572, + 0.37648, + 0.3754, + 0.37748, + 0.37704, + 0.37678, + 0.3756, + 0.3764, + 0.37632, + 0.37764, + 0.37716, + 0.37722, + 0.37784, + 0.37756, + 0.37846, + 0.3795, + 0.38056, + 0.37696, + 0.378, + 0.37894, + 0.37918, + 0.38124, + 0.37876, + 0.38168, + 0.38232, + 0.3829, + 0.3832, + 0.38284, + 0.3823, + 0.3804, + 0.38132, + 0.3823, + 0.3801, + 0.3822, + 0.38338, + 0.38094, + 0.38014, + 0.38042, + 0.38178, + 0.3816, + 0.38086, + 0.38378, + 0.3806, + 0.38276, + 0.38406, + 0.38342, + 0.38124 + ], + "test_acc": [ + 0.285, + 0.3059, + 0.321, + 0.3096, + 0.3312, + 0.3302, + 0.3228, + 0.3481, + 0.3321, + 0.3399, + 0.3381, + 0.3452, + 0.3487, + 0.353, + 0.3556, + 0.3522, + 0.3601, + 0.3548, + 0.3545, + 0.3601, + 0.3639, + 0.3662, + 0.3599, + 0.3704, + 0.376, + 0.384, + 0.3756, + 0.3748, + 0.3822, + 0.3781, + 0.3928, + 0.383, + 0.3873, + 0.3871, + 0.3898, + 0.3911, + 0.3906, + 0.3919, + 0.3913, + 0.3917, + 0.3974, + 0.3933, + 0.3945, + 0.3961, + 0.4018, + 0.3979, + 0.4008, + 0.4002, + 0.3904, + 0.3997, + 0.3978, + 0.4013, + 0.4005, + 0.3988, + 0.3977, + 0.3977, + 0.3988, + 0.4007, + 0.4037, + 0.4013, + 0.3994, + 0.4027, + 0.401, + 0.4011, + 0.4005, + 0.4059, + 0.4028, + 0.4058, + 0.4074, + 0.4029, + 0.4066, + 0.4041, + 0.4077, + 0.4053, + 0.4073, + 0.4054, + 0.4047, + 0.4053, + 0.406, + 0.4084, + 0.4073, + 0.4093, + 0.4042, + 0.4078, + 0.4072, + 0.4083, + 0.4063, + 0.4072, + 0.4064, + 0.4048, + 0.4085, + 0.4079, + 0.4088, + 0.4081, + 0.4075, + 0.4085, + 0.4084, + 0.4087, + 0.4089, + 0.409 + ] + }, + "diagnostics": { + "bp_cosine": [ + -0.002191243227571249, + 0.0590691976249218, + 0.0698356181383133, + 0.13605648279190063, + 0.08923564106225967, + 0.015668585896492004, + -0.07714484632015228, + 0.01213260181248188, + -0.0025911303237080574, + -0.04502572491765022, + -0.009707875549793243, + 0.9995251893997192 + ], + "perturbation_rho": [ + -0.043511975556612015, + -0.037262365221977234, + -0.03251894563436508, + 0.004958000499755144, + -0.02477085031569004, + 0.03360602259635925, + -0.005741700530052185, + 0.060581792145967484, + -0.0035040113143622875, + 0.014635683037340641, + -0.027158895507454872, + -0.040769003331661224 + ], + "nudging": { + "0.001": [ + 3.6729034036397934e-07, + -4.2887404561042786e-07, + -1.8009450286626816e-07, + -2.9650982469320297e-07, + -1.4586839824914932e-07, + -2.0139850676059723e-08, + 1.0035000741481781e-07, + 1.641456037759781e-08, + -3.4924596548080444e-09, + -2.0954757928848267e-08, + -5.9371814131736755e-09, + -1.025269739329815e-06 + ], + "0.003": [ + 1.1167721822857857e-06, + -1.264386810362339e-06, + -6.126938387751579e-07, + -7.883645594120026e-07, + -4.4063199311494827e-07, + -3.41096892952919e-08, + 2.60770320892334e-07, + -5.6694261729717255e-08, + -5.587935447692871e-09, + 1.5133991837501526e-07, + 2.8405338525772095e-08, + -3.802357241511345e-06 + ], + "0.01": [ + 3.6997953429818153e-06, + -4.11586370319128e-06, + -1.989188604056835e-06, + -2.64833215624094e-06, + -1.3904646039009094e-06, + -2.3422762751579285e-07, + 9.041978046298027e-07, + -2.2142194211483002e-07, + 1.0477378964424133e-08, + 5.260808393359184e-07, + 3.888271749019623e-08, + -1.2977398000657558e-05 + ] + }, + "hidden_norms_per_layer": [ + 4480.74365234375, + 31138.357421875, + 92759.8359375, + 106458.953125, + 140787.828125, + 222385.875, + 450712.65625, + 609736.25, + 665552.8125, + 762367.8125, + 785119.4375, + 812034.75, + 413068.875 + ], + "bp_grad_norms_per_layer": [ + 2.8362848752294667e-05, + 2.841563627953292e-06, + 1.476502120567602e-06, + 1.0035049626822001e-06, + 8.074727020357386e-07, + 7.039782303763786e-07, + 6.961119538573257e-07, + 6.961448093534273e-07, + 6.940574621694395e-07, + 6.940469461369503e-07, + 6.940644539099594e-07, + 6.928050879650982e-07, + 6.924681770215102e-07 + ] + }, + "drift": { + "embed.weight": 49.91851185408626, + "embed.bias": 17.86594759988079, + "blocks.0.ln.weight": 1.3650614023208618, + "blocks.0.w1.weight": 15.146734124129205, + "blocks.0.w1.bias": 11.038570361897584, + "blocks.0.w2.weight": 73.17905982531522, + "blocks.1.ln.weight": 1.2106239795684814, + "blocks.1.w1.weight": 15.587515080559209, + "blocks.1.w1.bias": 8.439216877390802, + "blocks.1.w2.weight": 58.68756094487398, + "blocks.2.ln.weight": 1.2199766635894775, + "blocks.2.w1.weight": 16.452854749419053, + "blocks.2.w1.bias": 7.664067486265118, + "blocks.2.w2.weight": 57.4145143245269, + "blocks.3.ln.weight": 0.9999213218688965, + "blocks.3.w1.weight": 15.45370831650007, + "blocks.3.w1.bias": 9.701139813225756, + "blocks.3.w2.weight": 50.62047330019419, + "blocks.4.ln.weight": 0.9458633661270142, + "blocks.4.w1.weight": 17.169635878784078, + "blocks.4.w1.bias": 12.344329529016134, + "blocks.4.w2.weight": 46.31906604368206, + "blocks.5.ln.weight": 0.9830385446548462, + "blocks.5.w1.weight": 19.826223976248237, + "blocks.5.w1.bias": 17.42019988793343, + "blocks.5.w2.weight": 54.57298040844193, + "blocks.6.ln.weight": 0.7866309881210327, + "blocks.6.w1.weight": 18.567239259563237, + "blocks.6.w1.bias": 18.51093768753787, + "blocks.6.w2.weight": 37.907911398349064, + "blocks.7.ln.weight": 0.664797842502594, + "blocks.7.w1.weight": 16.223960253839458, + "blocks.7.w1.bias": 17.78980163184235, + "blocks.7.w2.weight": 33.05609365656536, + "blocks.8.ln.weight": 0.7077108025550842, + "blocks.8.w1.weight": 17.30902246074257, + "blocks.8.w1.bias": 18.027686468269295, + "blocks.8.w2.weight": 34.213298812674346, + "blocks.9.ln.weight": 0.6533149480819702, + "blocks.9.w1.weight": 15.761533820678208, + "blocks.9.w1.bias": 16.051257049533785, + "blocks.9.w2.weight": 28.711206238147582, + "blocks.10.ln.weight": 0.6785529255867004, + "blocks.10.w1.weight": 14.454064107657125, + "blocks.10.w1.bias": 14.910736898942519, + "blocks.10.w2.weight": 33.53805959707834, + "blocks.11.ln.weight": 0.790199339389801, + "blocks.11.w1.weight": 21.739355051784727, + "blocks.11.w1.bias": 26.75768188676886, + "blocks.11.w2.weight": 38.60387777810455, + "out_ln.weight": 0.26305317878723145, + "out_head.weight": 4.037201408884711, + "out_head.bias": 0.8009701595278154 + } + } + }, + "config": { + "dataset": "cifar10", + "d_hidden": 256, + "num_blocks": 12, + "batch_size": 128, + "epochs": 100, + "lr": 0.001, + "lr_fb": 0.001, + "wd": 0.01, + "lam": 0.1, + "K": 4, + "sigma_bridge": 0.05, + "ema_momentum": 0.995, + "term_grad_weight": 1.0, + "seeds": [ + 4 + ], + "gpu": 0, + "output_dir": "results/fa_dfa_d256_L12_seed4", + "methods": [ + "fa", + "dfa" + ], + "random_targets": false, + "penalty_lam": 0.0, + "num_classes": 10 + } +}
\ No newline at end of file |