diff options
Diffstat (limited to 'results/fa_dfa_d512_L12_seed3/results_cifar10.json')
| -rw-r--r-- | results/fa_dfa_d512_L12_seed3/results_cifar10.json | 969 |
1 files changed, 969 insertions, 0 deletions
diff --git a/results/fa_dfa_d512_L12_seed3/results_cifar10.json b/results/fa_dfa_d512_L12_seed3/results_cifar10.json new file mode 100644 index 0000000..b0d47db --- /dev/null +++ b/results/fa_dfa_d512_L12_seed3/results_cifar10.json @@ -0,0 +1,969 @@ +{ + "3": { + "dfa": { + "log": { + "train_loss": [ + 2.0758749111938477, + 2.0449016705703738, + 2.0355339753723145, + 2.033782032623291, + 2.0313166973876955, + 2.0272680778503416, + 2.0246013010406494, + 2.0257189083099365, + 2.0217166822052004, + 2.0203594020080566, + 2.0174152685165407, + 2.018405201034546, + 2.0147499544525145, + 2.014394051055908, + 2.016394521636963, + 2.012079556236267, + 2.0137909897613526, + 2.014015982284546, + 2.0109746593475344, + 2.014003984375, + 2.0116691515350342, + 2.0092717248535155, + 2.008394868850708, + 2.0091352093887327, + 2.0081222701644896, + 2.008312914390564, + 2.0076249710464475, + 2.007539535140991, + 2.006514790344238, + 2.0067833641052246, + 2.002747939300537, + 2.0058067621612548, + 2.0007595862197878, + 2.0042390581512453, + 2.0029916175842284, + 2.0036084170150756, + 2.000785510559082, + 2.0037371703720095, + 2.001269952163696, + 2.0031689390563967, + 2.002326145706177, + 2.0042856772232054, + 2.001963946914673, + 1.9995613592147827, + 2.000965595703125, + 2.0001352091979983, + 2.0018488079071046, + 2.00168183052063, + 2.002008575668335, + 1.9990670106506347, + 2.000030261154175, + 2.0025694244766234, + 2.000111082458496, + 2.0006912731170656, + 1.9988810729980468, + 1.9988630979156494, + 1.9988078353118897, + 1.9998760187530518, + 1.999488633041382, + 1.9987801266479492, + 2.0005314921569823, + 1.9992797033691405, + 1.9988488864135743, + 1.9994260061645508, + 1.99964618309021, + 1.997838856201172, + 1.9974179007339476, + 1.9974302843475342, + 1.9990705501556396, + 1.9974287271499633, + 1.9969496033477783, + 1.9965652766418458, + 1.9951063619613647, + 1.997466404724121, + 1.9966685424804687, + 1.9972340461730957, + 1.9965911829376222, + 1.9977326393127441, + 1.9962291263580323, + 1.9955562910461426, + 1.9967020976257324, + 1.9967882759094238, + 1.9950329151916504, + 1.99679522026062, + 1.9955974411773683, + 1.9971681539154054, + 1.996667978439331, + 1.99505258934021, + 1.9948117360687256, + 1.9953517591094971, + 1.994836011428833, + 1.9956397792816163, + 1.9950232530212402, + 1.9973082242584228, + 1.9949973779296875, + 1.995920922012329, + 1.9955849953460694, + 1.9965253795623779, + 1.995772368774414, + 1.9941626037216187 + ], + "train_acc": [ + 0.2279, + 0.24072, + 0.24646, + 0.24854, + 0.25044, + 0.25614, + 0.25482, + 0.25184, + 0.25618, + 0.25788, + 0.25794, + 0.25986, + 0.25826, + 0.26054, + 0.26198, + 0.26164, + 0.26276, + 0.26056, + 0.26338, + 0.26482, + 0.26374, + 0.27, + 0.26544, + 0.26496, + 0.26726, + 0.26608, + 0.2677, + 0.26784, + 0.26822, + 0.26688, + 0.27298, + 0.2674, + 0.27316, + 0.26942, + 0.27202, + 0.2707, + 0.27076, + 0.2708, + 0.27202, + 0.27326, + 0.27412, + 0.27078, + 0.27212, + 0.27422, + 0.27278, + 0.27348, + 0.27218, + 0.27472, + 0.27368, + 0.27624, + 0.2752, + 0.27146, + 0.27238, + 0.27332, + 0.27558, + 0.27502, + 0.274, + 0.27478, + 0.2765, + 0.27494, + 0.2753, + 0.2748, + 0.27594, + 0.27726, + 0.27356, + 0.27716, + 0.27596, + 0.27972, + 0.277, + 0.27616, + 0.27622, + 0.2769, + 0.27584, + 0.27614, + 0.27692, + 0.27662, + 0.27758, + 0.27612, + 0.27734, + 0.27778, + 0.2763, + 0.2761, + 0.2776, + 0.27728, + 0.27692, + 0.27838, + 0.27818, + 0.2777, + 0.27922, + 0.27792, + 0.27694, + 0.27868, + 0.27922, + 0.2776, + 0.27966, + 0.27774, + 0.28024, + 0.27544, + 0.27852, + 0.28028 + ], + "test_acc": [ + 0.2245, + 0.2534, + 0.2569, + 0.2594, + 0.2706, + 0.2666, + 0.2759, + 0.276, + 0.2493, + 0.2911, + 0.2656, + 0.2816, + 0.2759, + 0.2916, + 0.302, + 0.2776, + 0.2854, + 0.2814, + 0.2838, + 0.2796, + 0.2881, + 0.2831, + 0.2907, + 0.2949, + 0.2872, + 0.2857, + 0.3067, + 0.301, + 0.2915, + 0.2936, + 0.2727, + 0.2971, + 0.2768, + 0.2756, + 0.2752, + 0.2918, + 0.2942, + 0.2706, + 0.2698, + 0.3002, + 0.2978, + 0.2896, + 0.2879, + 0.2922, + 0.2897, + 0.2955, + 0.2922, + 0.2979, + 0.3049, + 0.2838, + 0.2864, + 0.2929, + 0.3024, + 0.2947, + 0.2939, + 0.3001, + 0.2966, + 0.2875, + 0.288, + 0.2917, + 0.2948, + 0.2922, + 0.2926, + 0.2969, + 0.2958, + 0.2974, + 0.3028, + 0.2923, + 0.2926, + 0.2949, + 0.2849, + 0.2976, + 0.299, + 0.3005, + 0.291, + 0.297, + 0.2947, + 0.294, + 0.2974, + 0.2953, + 0.2958, + 0.293, + 0.292, + 0.3, + 0.2927, + 0.2969, + 0.2968, + 0.297, + 0.296, + 0.2958, + 0.2974, + 0.2971, + 0.2965, + 0.2957, + 0.2976, + 0.2961, + 0.2969, + 0.2969, + 0.2968, + 0.297 + ] + }, + "diagnostics": { + "bp_cosine": [ + 0.3807877004146576, + -5.573713860940188e-05, + 0.00026205976610071957, + 5.7832341553876176e-05, + -0.0006758540403097868, + 9.82169367489405e-05, + -0.0004489597922656685, + 0.00012056646664859727, + 0.00039241870399564505, + 0.0004099192447029054, + -0.00038764585042372346, + 9.827437315834686e-05 + ], + "perturbation_rho": [ + 0.0020497534424066544, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "nudging": { + "0.001": [ + -3.2549723982810974e-07, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "0.003": [ + -1.0975636541843414e-06, + 0.0, + -5.122274160385132e-09, + -9.313225746154785e-10, + 0.0, + 0.0, + 0.0, + -2.7939677238464355e-09, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "0.01": [ + -3.623310476541519e-06, + 2.7939677238464355e-09, + -3.259629011154175e-09, + -2.7939677238464355e-09, + 0.0, + 0.0, + 0.0, + -1.862645149230957e-09, + -9.313225746154785e-10, + 0.0, + 0.0, + -1.862645149230957e-09 + ] + }, + "hidden_norms_per_layer": [ + 53572.55078125, + 1372884736.0, + 1772287744.0, + 3241988608.0, + 5805380608.0, + 6317006848.0, + 6381533184.0, + 6840990720.0, + 8109968896.0, + 8596752384.0, + 8790169600.0, + 9575925760.0, + 10357586944.0 + ], + "bp_grad_norms_per_layer": [ + 2.5510743739687314e-07, + 2.2633883656197895e-10, + 2.2594967563627222e-10, + 2.2604293437034073e-10, + 2.2608663552414754e-10, + 2.2616231110106355e-10, + 2.2606677640979456e-10, + 2.2607062055701732e-10, + 2.2602890392686703e-10, + 2.2608405425561529e-10, + 2.2606443106365504e-10, + 2.26052662699594e-10, + 2.2604719485119773e-10 + ] + }, + "drift": { + "embed.weight": 327.43241116948093, + "embed.bias": 223.24151222733428, + "blocks.0.ln.weight": 9.973600730927519, + "blocks.0.w1.weight": 298.8679532598482, + "blocks.0.w1.bias": 266.1095556704987, + "blocks.0.w2.weight": 476.45333568353897, + "blocks.1.ln.weight": 7.306781436470832, + "blocks.1.w1.weight": 230.21983409061758, + "blocks.1.w1.bias": 212.39608458469817, + "blocks.1.w2.weight": 257.8517804786634, + "blocks.2.ln.weight": 8.853410982440527, + "blocks.2.w1.weight": 338.8735798549029, + "blocks.2.w1.bias": 298.9157804791883, + "blocks.2.w2.weight": 337.76450040174166, + "blocks.3.ln.weight": 9.715802278340659, + "blocks.3.w1.weight": 404.66894924511104, + "blocks.3.w1.bias": 372.47402373621406, + "blocks.3.w2.weight": 404.8085148871707, + "blocks.4.ln.weight": 8.048402116007853, + "blocks.4.w1.weight": 327.8249969525614, + "blocks.4.w1.bias": 304.9268816131629, + "blocks.4.w2.weight": 317.1280523416376, + "blocks.5.ln.weight": 6.247399813356893, + "blocks.5.w1.weight": 221.81137144150296, + "blocks.5.w1.bias": 207.34062578724078, + "blocks.5.w2.weight": 219.43839912847176, + "blocks.6.ln.weight": 8.827579347133504, + "blocks.6.w1.weight": 332.6040891021855, + "blocks.6.w1.bias": 301.2387289035309, + "blocks.6.w2.weight": 301.27670926969427, + "blocks.7.ln.weight": 10.285108430777552, + "blocks.7.w1.weight": 406.88401472243066, + "blocks.7.w1.bias": 378.73147854832933, + "blocks.7.w2.weight": 385.868939931865, + "blocks.8.ln.weight": 8.800845432078118, + "blocks.8.w1.weight": 349.4923999527594, + "blocks.8.w1.bias": 314.4618082483143, + "blocks.8.w2.weight": 312.34600503490987, + "blocks.9.ln.weight": 7.894181737045302, + "blocks.9.w1.weight": 310.5073132382578, + "blocks.9.w1.bias": 286.9185161607037, + "blocks.9.w2.weight": 283.5451705498602, + "blocks.10.ln.weight": 9.369016654158887, + "blocks.10.w1.weight": 380.3135567962096, + "blocks.10.w1.bias": 366.5904487269434, + "blocks.10.w2.weight": 364.5707271107058, + "blocks.11.ln.weight": 9.823259210793506, + "blocks.11.w1.weight": 387.5149132383442, + "blocks.11.w1.bias": 358.0799340264483, + "blocks.11.w2.weight": 346.4609492966194, + "out_ln.weight": 0.671169228619093, + "out_head.weight": 9.171587508256877, + "out_head.bias": 0.43138812866351306 + } + }, + "fa": { + "log": { + "train_loss": [ + 2.0396657500457764, + 1.9608277139282226, + 1.9326482776641847, + 1.9163381662750245, + 1.9034836812591553, + 1.890145294265747, + 1.8797314615631104, + 1.8768412320327759, + 1.867796627883911, + 1.8607218304824829, + 1.8534223027801513, + 1.8497662923431397, + 1.8423005596923827, + 1.836013819656372, + 1.8322983874511718, + 1.8273781113052368, + 1.8277745475006104, + 1.8251262734985352, + 1.820411690635681, + 1.8177711023712158, + 1.8161355939102173, + 1.811638991355896, + 1.8096345615005494, + 1.809067364845276, + 1.8052070534896851, + 1.806075843887329, + 1.8016806897735596, + 1.8036705466079712, + 1.8012689702606202, + 1.7954118814086915, + 1.7932103118515015, + 1.7936329647064209, + 1.7884179528427124, + 1.7905398846817016, + 1.7887225045013428, + 1.7856135736846923, + 1.7844085126495361, + 1.7840350121307373, + 1.774809980545044, + 1.778959913368225, + 1.771520369796753, + 1.775222759399414, + 1.7740441353607177, + 1.7697435302734374, + 1.7660783060073852, + 1.7702557649993897, + 1.7655815365600587, + 1.767026780052185, + 1.7612031787490845, + 1.7588948153686523, + 1.7596543489837646, + 1.7577299001312257, + 1.7578089023590089, + 1.7566554468154907, + 1.7528888186264038, + 1.7549751110076903, + 1.7530736223983765, + 1.7534073949813842, + 1.7489603903198243, + 1.7487286296844482, + 1.7519352731704712, + 1.7486104052734375, + 1.7467161587142945, + 1.748152449569702, + 1.7439867239379883, + 1.7447851418304443, + 1.7409665993499757, + 1.7442577163314819, + 1.7420410083770752, + 1.7413771923446655, + 1.7416430898666382, + 1.7407120821762085, + 1.7343293948745728, + 1.7396172916412354, + 1.7371713684844972, + 1.7387296481323242, + 1.736112067642212, + 1.733493857727051, + 1.7360445062637329, + 1.7351057822036744, + 1.734473055152893, + 1.7352992589569092, + 1.7343194945907592, + 1.735215076599121, + 1.733106077194214, + 1.7345250820159912, + 1.7314182668304443, + 1.731066604309082, + 1.730784634399414, + 1.7321079010391236, + 1.7304316757583618, + 1.7294625887298585, + 1.730209889831543, + 1.7305309671401978, + 1.7275332722854615, + 1.7291330585098266, + 1.7289624264526366, + 1.7314536703109742, + 1.7267744286346436, + 1.7254689588165284 + ], + "train_acc": [ + 0.2449, + 0.27998, + 0.2977, + 0.30396, + 0.3107, + 0.31656, + 0.32312, + 0.31882, + 0.32358, + 0.3291, + 0.32752, + 0.33508, + 0.33836, + 0.33794, + 0.34144, + 0.34536, + 0.34406, + 0.34554, + 0.34712, + 0.34862, + 0.34952, + 0.35122, + 0.35074, + 0.35326, + 0.3523, + 0.35342, + 0.35338, + 0.35376, + 0.35524, + 0.35476, + 0.35774, + 0.3577, + 0.3623, + 0.3571, + 0.35802, + 0.36086, + 0.3604, + 0.36042, + 0.36226, + 0.3633, + 0.3655, + 0.36332, + 0.36392, + 0.3675, + 0.36718, + 0.36608, + 0.36696, + 0.36686, + 0.37226, + 0.37178, + 0.37022, + 0.3701, + 0.3706, + 0.37212, + 0.375, + 0.37474, + 0.37334, + 0.37296, + 0.37436, + 0.37478, + 0.37246, + 0.37528, + 0.37576, + 0.37532, + 0.37498, + 0.37548, + 0.3776, + 0.37842, + 0.37692, + 0.37798, + 0.37794, + 0.37798, + 0.38094, + 0.38068, + 0.37932, + 0.37848, + 0.38182, + 0.37976, + 0.38048, + 0.381, + 0.3807, + 0.37908, + 0.38194, + 0.38326, + 0.38306, + 0.38104, + 0.3811, + 0.3813, + 0.38256, + 0.38198, + 0.38276, + 0.3842, + 0.3819, + 0.38484, + 0.38298, + 0.38164, + 0.38186, + 0.38012, + 0.38642, + 0.38538 + ], + "test_acc": [ + 0.2636, + 0.3156, + 0.3187, + 0.3379, + 0.3384, + 0.3469, + 0.3529, + 0.3526, + 0.3431, + 0.3598, + 0.3512, + 0.3619, + 0.3655, + 0.3755, + 0.3804, + 0.3699, + 0.3784, + 0.3643, + 0.3669, + 0.3644, + 0.3728, + 0.376, + 0.3802, + 0.3798, + 0.388, + 0.3631, + 0.3848, + 0.3802, + 0.3861, + 0.3829, + 0.3808, + 0.3865, + 0.3601, + 0.3915, + 0.371, + 0.3833, + 0.3932, + 0.3933, + 0.3827, + 0.3896, + 0.394, + 0.3837, + 0.3944, + 0.388, + 0.3896, + 0.3983, + 0.3933, + 0.3988, + 0.3997, + 0.3956, + 0.3967, + 0.3981, + 0.4041, + 0.4021, + 0.3873, + 0.4015, + 0.4031, + 0.3996, + 0.4016, + 0.401, + 0.403, + 0.4044, + 0.4021, + 0.4006, + 0.4062, + 0.4035, + 0.4027, + 0.4066, + 0.4081, + 0.4059, + 0.4023, + 0.4111, + 0.4095, + 0.406, + 0.4046, + 0.4048, + 0.4081, + 0.4067, + 0.4088, + 0.4074, + 0.4045, + 0.4073, + 0.408, + 0.4073, + 0.4066, + 0.4076, + 0.4064, + 0.4063, + 0.4088, + 0.4087, + 0.4081, + 0.4092, + 0.4088, + 0.4086, + 0.4093, + 0.4103, + 0.4091, + 0.4094, + 0.4087, + 0.4088 + ] + }, + "diagnostics": { + "bp_cosine": [ + 0.033972106873989105, + 0.070818230509758, + 0.01647794246673584, + -0.04525557532906532, + -0.03256663307547569, + -0.03431400656700134, + 0.011294779367744923, + -0.034145478159189224, + -0.05482051521539688, + 0.018059976398944855, + -0.030339818447828293, + 0.9974545836448669 + ], + "perturbation_rho": [ + 0.06674668192863464, + 0.0040874360129237175, + -0.028121206909418106, + 0.033479828387498856, + -0.023402733728289604, + -0.02146291360259056, + 0.01769183948636055, + -0.01436183787882328, + 0.009981930255889893, + 0.04906001687049866, + 0.004424326121807098, + 0.01695885881781578 + ], + "nudging": { + "0.001": [ + -2.7818605303764343e-06, + -2.752931322902441e-07, + -6.530899554491043e-08, + -3.67872416973114e-08, + 3.2247044146060944e-08, + 4.889443516731262e-09, + -1.862645149230957e-09, + -4.237517714500427e-08, + -4.889443516731262e-09, + -1.3737007975578308e-08, + -2.7008354663848877e-08, + -6.719492375850677e-07 + ], + "0.003": [ + -8.350558346137404e-06, + -6.812333595007658e-07, + -1.474982127547264e-07, + 9.033828973770142e-08, + 5.844049155712128e-08, + 7.008202373981476e-08, + 6.984919309616089e-09, + 1.0186340659856796e-07, + 1.2223608791828156e-07, + -8.253846317529678e-08, + 7.729977369308472e-08, + -2.513494109734893e-06 + ], + "0.01": [ + -2.8048030799254775e-05, + -2.1727464627474546e-06, + -3.3923424780368805e-07, + 3.7439167499542236e-07, + 2.1606683731079102e-07, + 2.60770320892334e-07, + -9.790528565645218e-08, + 2.130400389432907e-07, + 3.862660378217697e-07, + -1.123407855629921e-07, + 2.7267378754913807e-07, + -8.881674148142338e-06 + ] + }, + "hidden_norms_per_layer": [ + 6954.1630859375, + 111134.6796875, + 557813.6875, + 996516.875, + 1403786.0, + 1685541.375, + 2023717.25, + 2137581.0, + 2278237.0, + 2302800.5, + 2343287.75, + 2358363.25, + 1859393.25 + ], + "bp_grad_norms_per_layer": [ + 2.7945565307163633e-05, + 1.218010652337398e-06, + 4.458847513433284e-07, + 4.3005411498597823e-07, + 4.3150785700163397e-07, + 4.261477215550258e-07, + 4.2959365487149626e-07, + 4.2929926280521613e-07, + 4.298903206745308e-07, + 4.3081271883238514e-07, + 4.2645430653465155e-07, + 4.254479506471398e-07, + 4.048590369620797e-07 + ] + }, + "drift": { + "embed.weight": 48.84206517073718, + "embed.bias": 16.190805729960193, + "blocks.0.ln.weight": 1.1044050790999422, + "blocks.0.w1.weight": 16.916577725001215, + "blocks.0.w1.bias": 13.172179767406938, + "blocks.0.w2.weight": 57.6138058078061, + "blocks.1.ln.weight": 1.006892864310832, + "blocks.1.w1.weight": 20.474460499736093, + "blocks.1.w1.bias": 14.809778597172114, + "blocks.1.w2.weight": 56.78242938569469, + "blocks.2.ln.weight": 0.7251084712385265, + "blocks.2.w1.weight": 20.378823416992166, + "blocks.2.w1.bias": 19.146090983258652, + "blocks.2.w2.weight": 43.602875378949854, + "blocks.3.ln.weight": 0.5778874616748163, + "blocks.3.w1.weight": 19.87346377423969, + "blocks.3.w1.bias": 20.45559649243161, + "blocks.3.w2.weight": 29.273505770749868, + "blocks.4.ln.weight": 0.5907735236527697, + "blocks.4.w1.weight": 19.310944988230343, + "blocks.4.w1.bias": 19.98019865055604, + "blocks.4.w2.weight": 35.158811303772694, + "blocks.5.ln.weight": 0.5664442970803276, + "blocks.5.w1.weight": 21.492887279178838, + "blocks.5.w1.bias": 23.514956599646894, + "blocks.5.w2.weight": 31.178552638993263, + "blocks.6.ln.weight": 0.5602955775395394, + "blocks.6.w1.weight": 19.61043120210454, + "blocks.6.w1.bias": 21.095333255326253, + "blocks.6.w2.weight": 35.811762604119366, + "blocks.7.ln.weight": 0.5625727296567063, + "blocks.7.w1.weight": 18.807090394262456, + "blocks.7.w1.bias": 21.21463438586343, + "blocks.7.w2.weight": 37.013283750347604, + "blocks.8.ln.weight": 0.6306802678482845, + "blocks.8.w1.weight": 18.641671150147772, + "blocks.8.w1.bias": 18.967183417640936, + "blocks.8.w2.weight": 48.3826747365419, + "blocks.9.ln.weight": 0.587358912368524, + "blocks.9.w1.weight": 16.64124632095154, + "blocks.9.w1.bias": 15.919474132420179, + "blocks.9.w2.weight": 52.31462590866729, + "blocks.10.ln.weight": 0.5508428821937401, + "blocks.10.w1.weight": 15.626845109574424, + "blocks.10.w1.bias": 15.545502528098213, + "blocks.10.w2.weight": 49.44591721207919, + "blocks.11.ln.weight": 0.5806268804343621, + "blocks.11.w1.weight": 16.372036995921366, + "blocks.11.w1.bias": 13.944650611793428, + "blocks.11.w2.weight": 55.112106178217005, + "out_ln.weight": 0.3649173251620146, + "out_head.weight": 6.863263504617185, + "out_head.bias": 0.881340072926514 + } + } + }, + "config": { + "dataset": "cifar10", + "d_hidden": 512, + "num_blocks": 12, + "batch_size": 128, + "epochs": 100, + "lr": 0.001, + "lr_fb": 0.001, + "wd": 0.01, + "lam": 0.1, + "K": 4, + "sigma_bridge": 0.05, + "ema_momentum": 0.995, + "term_grad_weight": 1.0, + "seeds": [ + 3 + ], + "gpu": 0, + "output_dir": "results/fa_dfa_d512_L12_seed3", + "methods": [ + "fa", + "dfa" + ], + "random_targets": false, + "penalty_lam": 0.0, + "num_classes": 10 + } +}
\ No newline at end of file |