diff options
Diffstat (limited to 'results/fa_dfa_d256_L8_seed2/results_cifar10.json')
| -rw-r--r-- | results/fa_dfa_d256_L8_seed2/results_cifar10.json | 881 |
1 files changed, 881 insertions, 0 deletions
diff --git a/results/fa_dfa_d256_L8_seed2/results_cifar10.json b/results/fa_dfa_d256_L8_seed2/results_cifar10.json new file mode 100644 index 0000000..d34561f --- /dev/null +++ b/results/fa_dfa_d256_L8_seed2/results_cifar10.json @@ -0,0 +1,881 @@ +{ + "2": { + "dfa": { + "log": { + "train_loss": [ + 2.0886882679748533, + 2.068717489356995, + 2.0682498848724364, + 2.069725094680786, + 2.070216057891846, + 2.0698515621948244, + 2.0687220024108885, + 2.070030203552246, + 2.0674163632965086, + 2.0676918890380858, + 2.0673313298797606, + 2.064816045455933, + 2.0655937913513185, + 2.0663441319274902, + 2.0637396141815185, + 2.062943668060303, + 2.0631902577209473, + 2.0599184128189085, + 2.061999180755615, + 2.06542762260437, + 2.0647084296417235, + 2.065485874633789, + 2.0667521308898924, + 2.0664935392761232, + 2.0673532804107664, + 2.0684957303619385, + 2.066869210281372, + 2.067966806564331, + 2.0697209989929197, + 2.070055845565796, + 2.0691994259643556, + 2.0712842699432374, + 2.0721969300079346, + 2.0708646242523194, + 2.071090523452759, + 2.0724222064208986, + 2.0733718238830567, + 2.0746167260742188, + 2.075273102645874, + 2.0737432322692873, + 2.0741693158721923, + 2.0734868059539795, + 2.074445921783447, + 2.074121329498291, + 2.07596641456604, + 2.077426067504883, + 2.076066472320557, + 2.0742275380706787, + 2.076295921859741, + 2.076496145629883, + 2.0769809690856933, + 2.077219148712158, + 2.0779221090698243, + 2.0786613733673094, + 2.0790778132629395, + 2.077897124710083, + 2.0780438188934327, + 2.077808642272949, + 2.077776354217529, + 2.0779004010772706, + 2.079731927947998, + 2.0795092359924316, + 2.077359133682251, + 2.080645820541382, + 2.0798381079101564, + 2.0776988359069826, + 2.0795934693145752, + 2.080012099685669, + 2.0793860891723632, + 2.0781388677215578, + 2.0804970370483398, + 2.078412676239014, + 2.0773618979644777, + 2.07952963760376, + 2.0799946071624755, + 2.0800561015319823, + 2.0800602492523192, + 2.0796355754089357, + 2.0790224867248535, + 2.078683383331299, + 2.080645339508057, + 2.0792727799224853, + 2.0785955645751955, + 2.0800429849243165, + 2.078200587310791, + 2.0787713687896727, + 2.079665564956665, + 2.0787830973052976, + 2.0781467710113524, + 2.0787809454345703, + 2.0801569499206543, + 2.0799850133514406, + 2.07929296005249, + 2.079213014297485, + 2.0788697573852537, + 2.0782224908828737, + 2.080356029815674, + 2.078121220932007, + 2.078316292877197, + 2.077635722503662 + ], + "train_acc": [ + 0.23002, + 0.23622, + 0.2345, + 0.2358, + 0.23586, + 0.23384, + 0.2376, + 0.23746, + 0.23776, + 0.23716, + 0.24034, + 0.2422, + 0.24098, + 0.23872, + 0.24112, + 0.2408, + 0.24232, + 0.24592, + 0.24418, + 0.24304, + 0.24244, + 0.24248, + 0.24314, + 0.24312, + 0.2393, + 0.24304, + 0.24282, + 0.2429, + 0.24196, + 0.24328, + 0.24174, + 0.24158, + 0.24166, + 0.2407, + 0.24348, + 0.24048, + 0.24018, + 0.23818, + 0.24048, + 0.23914, + 0.2415, + 0.2415, + 0.24158, + 0.24146, + 0.24178, + 0.24136, + 0.24252, + 0.2435, + 0.24262, + 0.23852, + 0.24158, + 0.23898, + 0.23892, + 0.24054, + 0.24006, + 0.23972, + 0.24122, + 0.24014, + 0.24046, + 0.24122, + 0.23972, + 0.23902, + 0.24022, + 0.23936, + 0.2416, + 0.24268, + 0.2365, + 0.23816, + 0.23982, + 0.24116, + 0.24232, + 0.24096, + 0.24274, + 0.23918, + 0.23968, + 0.24092, + 0.23912, + 0.2421, + 0.2393, + 0.24116, + 0.24206, + 0.2393, + 0.2402, + 0.23988, + 0.2409, + 0.24032, + 0.24144, + 0.2411, + 0.24234, + 0.24238, + 0.23996, + 0.2394, + 0.23952, + 0.23958, + 0.24116, + 0.2415, + 0.23976, + 0.24184, + 0.24056, + 0.24148 + ], + "test_acc": [ + 0.2516, + 0.257, + 0.242, + 0.2516, + 0.2568, + 0.2492, + 0.247, + 0.2534, + 0.2538, + 0.2516, + 0.259, + 0.2616, + 0.241, + 0.2622, + 0.2649, + 0.263, + 0.254, + 0.2557, + 0.2605, + 0.2406, + 0.2404, + 0.2602, + 0.2541, + 0.2557, + 0.2459, + 0.2537, + 0.2639, + 0.2592, + 0.2641, + 0.2661, + 0.2526, + 0.2479, + 0.2497, + 0.2529, + 0.2595, + 0.2577, + 0.2523, + 0.2523, + 0.2578, + 0.2396, + 0.2647, + 0.2567, + 0.2657, + 0.2477, + 0.259, + 0.2492, + 0.2469, + 0.246, + 0.2515, + 0.2638, + 0.2481, + 0.2607, + 0.2516, + 0.2447, + 0.2457, + 0.2547, + 0.2481, + 0.2426, + 0.2434, + 0.2491, + 0.2579, + 0.2605, + 0.2537, + 0.2654, + 0.2601, + 0.2484, + 0.249, + 0.2515, + 0.2555, + 0.2502, + 0.2538, + 0.258, + 0.2569, + 0.254, + 0.2533, + 0.2544, + 0.2589, + 0.2566, + 0.2566, + 0.2521, + 0.2485, + 0.254, + 0.2493, + 0.2517, + 0.2532, + 0.252, + 0.2492, + 0.2531, + 0.2524, + 0.2491, + 0.2538, + 0.2526, + 0.251, + 0.2548, + 0.2537, + 0.2541, + 0.2543, + 0.2532, + 0.2533, + 0.2533 + ] + }, + "diagnostics": { + "bp_cosine": [ + 0.20111382007598877, + 0.0004621353873517364, + -0.0012605632655322552, + 5.168101051822305e-05, + -0.0015679890057072043, + 0.0019785298500210047, + 0.0006557029555551708, + 0.0007036488968878984 + ], + "perturbation_rho": [ + -0.02028709650039673, + 0.0, + 0.0, + 0.0, + -0.0007397841545753181, + 0.0, + 0.0, + 0.0 + ], + "nudging": { + "0.001": [ + -1.695007085800171e-07, + 0.0, + 0.0, + 0.0, + 9.313225746154785e-10, + 0.0, + 0.0, + 0.0 + ], + "0.003": [ + -3.8463622331619263e-07, + 0.0, + 0.0, + 0.0, + 9.313225746154785e-10, + 0.0, + 0.0, + 0.0 + ], + "0.01": [ + -1.1408701539039612e-06, + 0.0, + 0.0, + 0.0, + 2.7939677238464355e-09, + 0.0, + 0.0, + 0.0 + ] + }, + "hidden_norms_per_layer": [ + 45184.2421875, + 1068381440.0, + 1778018944.0, + 2116394752.0, + 2182007296.0, + 2562655488.0, + 2566907136.0, + 2985686272.0, + 3052116480.0 + ], + "bp_grad_norms_per_layer": [ + 1.4686037275168928e-07, + 6.425820986422082e-10, + 6.384120454505648e-10, + 6.389382356530859e-10, + 6.396255192164801e-10, + 6.406807306902351e-10, + 6.405915242702065e-10, + 6.406805641567814e-10, + 6.40737352064491e-10 + ] + }, + "drift": { + "embed.weight": 418.6113426367787, + "embed.bias": 437.5668398161194, + "blocks.0.ln.weight": 10.016462326049805, + "blocks.0.w1.weight": 315.59584333536606, + "blocks.0.w1.bias": 338.3304059680062, + "blocks.0.w2.weight": 607.5484294308882, + "blocks.1.ln.weight": 11.81142807006836, + "blocks.1.w1.weight": 335.3681839945845, + "blocks.1.w1.bias": 316.5131919451052, + "blocks.1.w2.weight": 481.7556503680824, + "blocks.2.ln.weight": 10.658580780029297, + "blocks.2.w1.weight": 312.4649885568403, + "blocks.2.w1.bias": 293.43148423074234, + "blocks.2.w2.weight": 428.14647887743376, + "blocks.3.ln.weight": 9.960590362548828, + "blocks.3.w1.weight": 241.4254307025639, + "blocks.3.w1.bias": 217.00406676688746, + "blocks.3.w2.weight": 345.4810531131192, + "blocks.4.ln.weight": 11.285172462463379, + "blocks.4.w1.weight": 325.03446821544236, + "blocks.4.w1.bias": 313.6381406887335, + "blocks.4.w2.weight": 455.8087708526798, + "blocks.5.ln.weight": 8.04609489440918, + "blocks.5.w1.weight": 215.75566383550878, + "blocks.5.w1.bias": 207.62392148123524, + "blocks.5.w2.weight": 294.26989490791993, + "blocks.6.ln.weight": 11.42038631439209, + "blocks.6.w1.weight": 327.5629239121232, + "blocks.6.w1.bias": 305.4508358753746, + "blocks.6.w2.weight": 477.6831296534087, + "blocks.7.ln.weight": 8.823966979980469, + "blocks.7.w1.weight": 247.08109127723426, + "blocks.7.w1.bias": 236.30426713334924, + "blocks.7.w2.weight": 344.7289213462053, + "out_ln.weight": 0.8588130474090576, + "out_head.weight": 8.666066942387875, + "out_head.bias": 1.137642620895565 + } + }, + "fa": { + "log": { + "train_loss": [ + 2.0575581959533693, + 1.982899369468689, + 1.949610625076294, + 1.9338219098281861, + 1.9279949721527099, + 1.9190433423614501, + 1.9120057677459716, + 1.9073227895355225, + 1.9010615240478517, + 1.8946472421264648, + 1.8895224981307983, + 1.8831638266372681, + 1.8827376470184327, + 1.8782546444320678, + 1.87282203704834, + 1.8714419341278077, + 1.867955844154358, + 1.8605742862701415, + 1.8630245657348632, + 1.8617111119842529, + 1.858997180404663, + 1.8575258152770997, + 1.853901057510376, + 1.851965792312622, + 1.849754028968811, + 1.8477436004638672, + 1.8454922165679932, + 1.8445022313690185, + 1.8395921088409424, + 1.8344731448745728, + 1.8315038693237304, + 1.8307365502929687, + 1.8269501232528687, + 1.824952223854065, + 1.816781127052307, + 1.815732438735962, + 1.81667452003479, + 1.8134329107666016, + 1.8117710926055908, + 1.8103601391220092, + 1.8055513787841797, + 1.8014423720550536, + 1.8024774380111694, + 1.7986959079360962, + 1.7953778457641603, + 1.7985560263442992, + 1.7965025356674194, + 1.7882651119613648, + 1.7905249483489991, + 1.789016272354126, + 1.7867180441665649, + 1.787857604675293, + 1.7842913775253295, + 1.7844246352386475, + 1.7800830935287475, + 1.7805310357284545, + 1.7760378722763062, + 1.7760090646743774, + 1.7754922837066651, + 1.7761539294815063, + 1.7764367975234985, + 1.7729884731292724, + 1.7713473574829102, + 1.7740464125823974, + 1.775451347427368, + 1.771372494544983, + 1.7704123861312866, + 1.774370786743164, + 1.770715291481018, + 1.7711526647949218, + 1.7708271207046509, + 1.7675350022125245, + 1.7677011463165284, + 1.7680682544708253, + 1.769882447128296, + 1.7696199462127686, + 1.7676654192352295, + 1.7694056000137328, + 1.769216322402954, + 1.7660821619415283, + 1.766729666442871, + 1.7692560305786134, + 1.7672280679702759, + 1.766029937133789, + 1.7672864461517335, + 1.7673147164535523, + 1.766981872253418, + 1.7642782439804077, + 1.765021160812378, + 1.7666088375091553, + 1.7657413306045533, + 1.7673192971801759, + 1.7683989611434936, + 1.7645942099380494, + 1.7642021070098877, + 1.7653449111175537, + 1.7663517474365233, + 1.7659291017913819, + 1.7664799518203735, + 1.763619673538208 + ], + "train_acc": [ + 0.23966, + 0.27212, + 0.28776, + 0.29538, + 0.30048, + 0.30252, + 0.30566, + 0.30792, + 0.3088, + 0.31204, + 0.31266, + 0.31664, + 0.31756, + 0.31608, + 0.32266, + 0.32434, + 0.32378, + 0.3291, + 0.32652, + 0.32948, + 0.33328, + 0.33224, + 0.33312, + 0.33614, + 0.33746, + 0.33856, + 0.3381, + 0.33856, + 0.34324, + 0.3451, + 0.34418, + 0.34646, + 0.34888, + 0.34762, + 0.35072, + 0.35438, + 0.35212, + 0.35348, + 0.35384, + 0.35278, + 0.35586, + 0.35542, + 0.3554, + 0.35636, + 0.35712, + 0.35806, + 0.35824, + 0.36148, + 0.35998, + 0.36142, + 0.361, + 0.35922, + 0.36284, + 0.36258, + 0.3628, + 0.3614, + 0.36514, + 0.36556, + 0.36786, + 0.36512, + 0.36446, + 0.36606, + 0.36762, + 0.3651, + 0.3634, + 0.3675, + 0.36398, + 0.36604, + 0.36612, + 0.36696, + 0.3681, + 0.36822, + 0.37026, + 0.36726, + 0.36898, + 0.36782, + 0.36792, + 0.36732, + 0.3675, + 0.36906, + 0.3676, + 0.36604, + 0.36928, + 0.36834, + 0.36678, + 0.36774, + 0.37054, + 0.36758, + 0.36964, + 0.36992, + 0.36908, + 0.37016, + 0.36498, + 0.36648, + 0.37084, + 0.37, + 0.36922, + 0.3694, + 0.36746, + 0.37072 + ], + "test_acc": [ + 0.2762, + 0.2993, + 0.3248, + 0.3131, + 0.3217, + 0.324, + 0.3154, + 0.3369, + 0.3261, + 0.3276, + 0.3409, + 0.3459, + 0.327, + 0.3408, + 0.3397, + 0.3309, + 0.3524, + 0.3538, + 0.3522, + 0.3438, + 0.3473, + 0.3506, + 0.3502, + 0.3546, + 0.3579, + 0.3615, + 0.3654, + 0.3661, + 0.3669, + 0.3655, + 0.3694, + 0.3655, + 0.3657, + 0.3646, + 0.3701, + 0.3684, + 0.3688, + 0.3636, + 0.3718, + 0.3745, + 0.3806, + 0.3699, + 0.3824, + 0.3806, + 0.3814, + 0.3851, + 0.3805, + 0.3745, + 0.3867, + 0.3821, + 0.3766, + 0.3863, + 0.3824, + 0.38, + 0.3827, + 0.3808, + 0.3811, + 0.3814, + 0.3822, + 0.3857, + 0.3854, + 0.3832, + 0.3813, + 0.3866, + 0.3831, + 0.3794, + 0.384, + 0.3861, + 0.385, + 0.3877, + 0.386, + 0.3854, + 0.3875, + 0.3869, + 0.3868, + 0.388, + 0.3888, + 0.3899, + 0.3869, + 0.3904, + 0.3891, + 0.3886, + 0.386, + 0.39, + 0.3882, + 0.3885, + 0.3862, + 0.3886, + 0.39, + 0.3888, + 0.3891, + 0.3892, + 0.3888, + 0.3899, + 0.391, + 0.3904, + 0.3893, + 0.3894, + 0.3893, + 0.3894 + ] + }, + "diagnostics": { + "bp_cosine": [ + 0.03124001994729042, + 0.08908016234636307, + 0.04665760695934296, + -0.04777219519019127, + -0.07556367665529251, + -0.0461275577545166, + -0.03507336974143982, + 0.9990285634994507 + ], + "perturbation_rho": [ + 0.027517501264810562, + 0.010178674943745136, + -0.034707143902778625, + 0.005213148891925812, + -0.07512903958559036, + -0.00475400872528553, + 9.842216968536377e-06, + 0.02905265800654888 + ], + "nudging": { + "0.001": [ + -1.698266714811325e-06, + -2.1292362362146378e-07, + 3.958120942115784e-08, + 4.831235855817795e-08, + 8.055940270423889e-08, + 2.3748725652694702e-08, + 5.098991096019745e-08, + -9.502982720732689e-07 + ], + "0.003": [ + -5.119014531373978e-06, + -9.03732143342495e-07, + -1.7078127712011337e-07, + 2.859160304069519e-07, + 2.7811620384454727e-07, + 2.0384322851896286e-07, + 1.5203841030597687e-07, + -3.3261021599173546e-06 + ], + "0.01": [ + -1.702900044620037e-05, + -3.168359398841858e-06, + -6.603077054023743e-07, + 5.726469680666924e-07, + 8.258502930402756e-07, + 5.200272426009178e-07, + 3.9814040064811707e-07, + -1.166516449302435e-05 + ] + }, + "hidden_norms_per_layer": [ + 6265.44482421875, + 53904.69921875, + 118099.265625, + 566143.375, + 935321.5, + 1106561.75, + 1203173.875, + 1238320.5, + 740976.125 + ], + "bp_grad_norms_per_layer": [ + 2.1726980776293203e-05, + 1.636821252759546e-06, + 7.93243316365988e-07, + 6.795298759243451e-07, + 6.810526542722073e-07, + 6.809283945585776e-07, + 6.807385943830013e-07, + 6.802036978115211e-07, + 6.770658842469857e-07 + ] + }, + "drift": { + "embed.weight": 63.21845610957774, + "embed.bias": 25.90407451226601, + "blocks.0.ln.weight": 1.8451513051986694, + "blocks.0.w1.weight": 19.346207451992306, + "blocks.0.w1.bias": 14.068566093574962, + "blocks.0.w2.weight": 82.80276521804117, + "blocks.1.ln.weight": 1.3143744468688965, + "blocks.1.w1.weight": 20.043434543563325, + "blocks.1.w1.bias": 13.614899313814618, + "blocks.1.w2.weight": 70.67866920669934, + "blocks.2.ln.weight": 1.2612940073013306, + "blocks.2.w1.weight": 24.037240913049125, + "blocks.2.w1.bias": 22.307086193211962, + "blocks.2.w2.weight": 47.56871336729849, + "blocks.3.ln.weight": 1.0761433839797974, + "blocks.3.w1.weight": 25.473873360543205, + "blocks.3.w1.bias": 25.35175214203562, + "blocks.3.w2.weight": 40.85251406771216, + "blocks.4.ln.weight": 0.9428171515464783, + "blocks.4.w1.weight": 23.293363798467443, + "blocks.4.w1.bias": 24.428185562790627, + "blocks.4.w2.weight": 37.128127593959505, + "blocks.5.ln.weight": 0.8093754649162292, + "blocks.5.w1.weight": 19.89014687425912, + "blocks.5.w1.bias": 21.460830616828144, + "blocks.5.w2.weight": 34.639394288754545, + "blocks.6.ln.weight": 0.7578156590461731, + "blocks.6.w1.weight": 17.50016127400267, + "blocks.6.w1.bias": 18.172019570522167, + "blocks.6.w2.weight": 28.78778456757253, + "blocks.7.ln.weight": 0.9525559544563293, + "blocks.7.w1.weight": 21.92857716495514, + "blocks.7.w1.bias": 22.77505483247343, + "blocks.7.w2.weight": 58.30518938223301, + "out_ln.weight": 0.38212674856185913, + "out_head.weight": 5.330482873019479, + "out_head.bias": 0.8164980229115041 + } + } + }, + "config": { + "dataset": "cifar10", + "d_hidden": 256, + "num_blocks": 8, + "batch_size": 128, + "epochs": 100, + "lr": 0.001, + "lr_fb": 0.001, + "wd": 0.01, + "lam": 0.1, + "K": 4, + "sigma_bridge": 0.05, + "ema_momentum": 0.995, + "term_grad_weight": 1.0, + "seeds": [ + 2 + ], + "gpu": 0, + "output_dir": "results/fa_dfa_d256_L8_seed2", + "methods": [ + "fa", + "dfa" + ], + "random_targets": false, + "penalty_lam": 0.0, + "num_classes": 10 + } +}
\ No newline at end of file |