diff options
Diffstat (limited to 'results/fa_dfa_d512_L12_seed7/results_cifar10.json')
| -rw-r--r-- | results/fa_dfa_d512_L12_seed7/results_cifar10.json | 969 |
1 files changed, 969 insertions, 0 deletions
diff --git a/results/fa_dfa_d512_L12_seed7/results_cifar10.json b/results/fa_dfa_d512_L12_seed7/results_cifar10.json new file mode 100644 index 0000000..d1c18ce --- /dev/null +++ b/results/fa_dfa_d512_L12_seed7/results_cifar10.json @@ -0,0 +1,969 @@ +{ + "7": { + "dfa": { + "log": { + "train_loss": [ + 2.0717703549194337, + 2.041298008880615, + 2.029167268066406, + 2.0265366765975954, + 2.0233016680145264, + 2.021149669342041, + 2.015149740600586, + 2.014331086883545, + 2.011163010597229, + 2.00646972366333, + 2.0077928015899658, + 2.0069499212265014, + 2.0002946398925783, + 2.002099399795532, + 1.9983450998687744, + 1.9959249508666992, + 1.9950883417510987, + 1.9943942831802368, + 1.993684328918457, + 1.9930962839508057, + 1.9938797711181642, + 1.986891388015747, + 1.9910820366668702, + 1.9907979718780517, + 1.989411315689087, + 1.9852583177947998, + 1.981110195388794, + 1.9853213675308228, + 1.9821887520599366, + 1.9824637595367431, + 1.9810781491470337, + 1.978963046951294, + 1.9793350392913818, + 1.9822494831848145, + 1.9770296029281615, + 1.9799606243515016, + 1.979586162147522, + 1.978336156539917, + 1.9767711865234374, + 1.9781431746673583, + 1.97637092502594, + 1.9791246725845337, + 1.9776694247055053, + 1.9774175772857665, + 1.974560000038147, + 1.97618024269104, + 1.9750861001968383, + 1.9740860368347168, + 1.975063199005127, + 1.975644574661255, + 1.974392618637085, + 1.976005935974121, + 1.974110288734436, + 1.9724776383209228, + 1.9728016277313232, + 1.973982884864807, + 1.972937359275818, + 1.971426241798401, + 1.970759379310608, + 1.97244817237854, + 1.9711870779800416, + 1.9723809194564819, + 1.970773144493103, + 1.9708155387115478, + 1.9702037267684938, + 1.9704236566925049, + 1.9704266147613525, + 1.9674897221374512, + 1.969771149520874, + 1.9684535709381104, + 1.972261046295166, + 1.9708796706390381, + 1.9701639178848267, + 1.970189701461792, + 1.9690214687728882, + 1.970269794769287, + 1.9686412525177002, + 1.9686531394195557, + 1.9690335285186769, + 1.9683185077667236, + 1.9699563415527344, + 1.9683909454345703, + 1.9685154306793213, + 1.966210771751404, + 1.969103935775757, + 1.9681334160614015, + 1.9663462246322632, + 1.9694912530899047, + 1.9677201830291748, + 1.9683091250610352, + 1.9689178394317628, + 1.966240083847046, + 1.968869041519165, + 1.96721680809021, + 1.9639848593139648, + 1.9679099475097657, + 1.9662394170379638, + 1.966985820236206, + 1.9662641632843016, + 1.9668700101089478 + ], + "train_acc": [ + 0.23978, + 0.24322, + 0.25136, + 0.25116, + 0.25264, + 0.25374, + 0.26024, + 0.2603, + 0.26, + 0.26422, + 0.26226, + 0.26102, + 0.26648, + 0.2665, + 0.26764, + 0.26976, + 0.27012, + 0.27084, + 0.27144, + 0.27138, + 0.26928, + 0.2735, + 0.27452, + 0.27408, + 0.27312, + 0.276, + 0.27856, + 0.27458, + 0.27752, + 0.27688, + 0.27972, + 0.2782, + 0.2808, + 0.27646, + 0.28084, + 0.28012, + 0.2808, + 0.2793, + 0.28086, + 0.2807, + 0.27956, + 0.27968, + 0.28138, + 0.28184, + 0.28282, + 0.28208, + 0.28344, + 0.2835, + 0.27944, + 0.28146, + 0.28096, + 0.28186, + 0.28142, + 0.2833, + 0.28372, + 0.28258, + 0.28264, + 0.28458, + 0.28644, + 0.2822, + 0.2842, + 0.28418, + 0.2849, + 0.28526, + 0.28614, + 0.28488, + 0.2833, + 0.28626, + 0.28546, + 0.28682, + 0.28376, + 0.2851, + 0.28242, + 0.28406, + 0.28672, + 0.28582, + 0.28482, + 0.28584, + 0.28422, + 0.28666, + 0.28424, + 0.28532, + 0.28416, + 0.28974, + 0.28814, + 0.28682, + 0.28648, + 0.28618, + 0.28602, + 0.2883, + 0.2841, + 0.28796, + 0.28582, + 0.28556, + 0.29042, + 0.28562, + 0.28642, + 0.28704, + 0.28854, + 0.28752 + ], + "test_acc": [ + 0.2673, + 0.2593, + 0.2397, + 0.2765, + 0.284, + 0.2617, + 0.2814, + 0.2855, + 0.277, + 0.2629, + 0.287, + 0.2628, + 0.2838, + 0.2918, + 0.29, + 0.3014, + 0.2878, + 0.2773, + 0.2764, + 0.299, + 0.2977, + 0.2922, + 0.2967, + 0.2866, + 0.2876, + 0.2988, + 0.296, + 0.3046, + 0.3027, + 0.2977, + 0.3168, + 0.2957, + 0.2942, + 0.3079, + 0.2781, + 0.3022, + 0.294, + 0.3143, + 0.3169, + 0.3066, + 0.3105, + 0.3005, + 0.3035, + 0.3135, + 0.3035, + 0.2983, + 0.2928, + 0.3032, + 0.299, + 0.3029, + 0.2978, + 0.3047, + 0.3054, + 0.2984, + 0.3004, + 0.3, + 0.3146, + 0.3113, + 0.3079, + 0.3115, + 0.3101, + 0.3124, + 0.3111, + 0.3014, + 0.3064, + 0.3106, + 0.3105, + 0.3112, + 0.3019, + 0.3067, + 0.3096, + 0.2973, + 0.3033, + 0.3053, + 0.3119, + 0.3105, + 0.3093, + 0.3053, + 0.31, + 0.3055, + 0.3091, + 0.3076, + 0.3069, + 0.3131, + 0.3058, + 0.3119, + 0.3091, + 0.3084, + 0.3126, + 0.3089, + 0.3102, + 0.3105, + 0.31, + 0.3113, + 0.309, + 0.3103, + 0.3103, + 0.3109, + 0.3108, + 0.3108 + ] + }, + "diagnostics": { + "bp_cosine": [ + 0.4225231409072876, + 0.000332531810272485, + -0.00028742029098793864, + -0.0005942026618868113, + 0.0011352845467627048, + 5.1041941333096474e-05, + 4.8770005378173664e-05, + -0.0001115800259867683, + -0.00025915325386449695, + -0.0001579285744810477, + -0.00034282656270079315, + -0.001362925162538886 + ], + "perturbation_rho": [ + 0.005214178003370762, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "nudging": { + "0.001": [ + -6.705522537231445e-07, + -1.862645149230957e-09, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "0.003": [ + -1.5972182154655457e-06, + 5.587935447692871e-09, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "0.01": [ + -5.433335900306702e-06, + 0.0, + 3.725290298461914e-09, + 0.0, + -1.862645149230957e-09, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ] + }, + "hidden_norms_per_layer": [ + 52073.1640625, + 624600448.0, + 2354930944.0, + 3962745600.0, + 4607439360.0, + 6940876800.0, + 10061239296.0, + 10270245888.0, + 10501487616.0, + 10991346688.0, + 11392520192.0, + 11497131008.0, + 11767929856.0 + ], + "bp_grad_norms_per_layer": [ + 3.116530820079788e-07, + 2.391457587513912e-10, + 2.334610560428274e-10, + 2.3252627601166864e-10, + 2.3254012604390084e-10, + 2.3251603420426648e-10, + 2.3252098857451386e-10, + 2.324976183798455e-10, + 2.3242860414107724e-10, + 2.3244653424292494e-10, + 2.3248170055722994e-10, + 2.3248254710228622e-10, + 2.324954950783109e-10 + ] + }, + "drift": { + "embed.weight": 314.5708156367549, + "embed.bias": 173.06181327916602, + "blocks.0.ln.weight": 10.608293168088462, + "blocks.0.w1.weight": 248.85531881291706, + "blocks.0.w1.bias": 197.42523217269257, + "blocks.0.w2.weight": 461.7533133123923, + "blocks.1.ln.weight": 8.793695977749161, + "blocks.1.w1.weight": 307.77495139687693, + "blocks.1.w1.bias": 271.79664056051047, + "blocks.1.w2.weight": 356.28012985496673, + "blocks.2.ln.weight": 8.954547910557306, + "blocks.2.w1.weight": 349.2950006278737, + "blocks.2.w1.bias": 313.10534483438977, + "blocks.2.w2.weight": 340.97526886427016, + "blocks.3.ln.weight": 7.664064660684567, + "blocks.3.w1.weight": 303.1221009157576, + "blocks.3.w1.bias": 276.7648976947458, + "blocks.3.w2.weight": 276.8184093796655, + "blocks.4.ln.weight": 10.090295578382477, + "blocks.4.w1.weight": 416.58752319563774, + "blocks.4.w1.bias": 396.4529594058789, + "blocks.4.w2.weight": 411.8714989953588, + "blocks.5.ln.weight": 11.222968212087666, + "blocks.5.w1.weight": 460.41052960846133, + "blocks.5.w1.bias": 446.32020235508855, + "blocks.5.w2.weight": 461.9898764832566, + "blocks.6.ln.weight": 9.029697428612298, + "blocks.6.w1.weight": 363.0948995365021, + "blocks.6.w1.bias": 335.6612472751257, + "blocks.6.w2.weight": 358.45859133995083, + "blocks.7.ln.weight": 8.68285178731722, + "blocks.7.w1.weight": 347.30737462136295, + "blocks.7.w1.bias": 319.98700238133193, + "blocks.7.w2.weight": 330.4186701898697, + "blocks.8.ln.weight": 8.786812217157763, + "blocks.8.w1.weight": 347.961851087332, + "blocks.8.w1.bias": 341.22606981083646, + "blocks.8.w2.weight": 315.32952774172026, + "blocks.9.ln.weight": 9.311416409774596, + "blocks.9.w1.weight": 343.96115770328134, + "blocks.9.w1.bias": 321.3536311695321, + "blocks.9.w2.weight": 315.28472186823876, + "blocks.10.ln.weight": 6.752703446423726, + "blocks.10.w1.weight": 265.99193594775096, + "blocks.10.w1.bias": 250.46671872343975, + "blocks.10.w2.weight": 243.26857071036574, + "blocks.11.ln.weight": 9.311969376436522, + "blocks.11.w1.weight": 375.3561731401925, + "blocks.11.w1.bias": 348.39190827099014, + "blocks.11.w2.weight": 350.6780634465036, + "out_ln.weight": 0.7156480208724685, + "out_head.weight": 9.90130634348424, + "out_head.bias": 0.6155786911344596 + } + }, + "fa": { + "log": { + "train_loss": [ + 2.030823752403259, + 1.934500171585083, + 1.9079237392807007, + 1.8947180710220337, + 1.8812753409576417, + 1.8702804034423828, + 1.8603069189834596, + 1.854407600440979, + 1.8499454795074464, + 1.8409444109344482, + 1.8404085549545288, + 1.8384266778182983, + 1.8317015438842774, + 1.8293640671157836, + 1.8229795181655883, + 1.8220855783081054, + 1.8241054327392578, + 1.8256892261123656, + 1.825209083328247, + 1.8221923596572875, + 1.827516168899536, + 1.8242645990371704, + 1.8307866805648805, + 1.8350352270126342, + 1.8312144650268554, + 1.8282124835205078, + 1.822820876083374, + 1.8260399611663818, + 1.8204992460632323, + 1.8168495791625976, + 1.810229619064331, + 1.811380881576538, + 1.805999729537964, + 1.8007601963043214, + 1.7961979135513306, + 1.7947537366485595, + 1.7928297024536133, + 1.788902147216797, + 1.7878808304214477, + 1.7845082584381105, + 1.7827687833023071, + 1.7805255111694336, + 1.7768453490829468, + 1.7761483609390258, + 1.7748139315414428, + 1.7749142253875732, + 1.7714660557174682, + 1.770267225265503, + 1.7713737234497071, + 1.7634125463867187, + 1.7577275490570068, + 1.7656006281280519, + 1.7626705096435547, + 1.7589925038909913, + 1.756216689376831, + 1.7574205046844482, + 1.7555658932113647, + 1.7512263652801514, + 1.7533771200180053, + 1.7522863424301147, + 1.743942953414917, + 1.7457706503677368, + 1.744990401649475, + 1.7453525008773805, + 1.743213934249878, + 1.7438006609344483, + 1.7438085187530517, + 1.738191604385376, + 1.7366509014129639, + 1.739820227279663, + 1.7397972118377685, + 1.7396622115325928, + 1.7390950318145753, + 1.7385543502044678, + 1.733802723083496, + 1.735742342300415, + 1.7329303577041626, + 1.732147887878418, + 1.7361309533691407, + 1.7338640795516969, + 1.7344183361053467, + 1.7314273851776123, + 1.732653318786621, + 1.726430778465271, + 1.7266450793838501, + 1.7317952347564698, + 1.7268288860321044, + 1.7301931661605836, + 1.729054831314087, + 1.728853748779297, + 1.7290257139587402, + 1.7282057024383546, + 1.7276880645370483, + 1.7283826892089844, + 1.7221429102325438, + 1.7275538332366944, + 1.7293567386627198, + 1.7251165560913087, + 1.727771372756958, + 1.7276002161026 + ], + "train_acc": [ + 0.25468, + 0.29668, + 0.3079, + 0.31446, + 0.31972, + 0.32522, + 0.32856, + 0.32842, + 0.33384, + 0.3401, + 0.33566, + 0.3388, + 0.3399, + 0.34344, + 0.3465, + 0.3477, + 0.34726, + 0.34478, + 0.34704, + 0.34592, + 0.34452, + 0.34292, + 0.34314, + 0.3423, + 0.33992, + 0.34576, + 0.3454, + 0.34362, + 0.3478, + 0.34758, + 0.35328, + 0.35146, + 0.35478, + 0.35634, + 0.35704, + 0.35806, + 0.3582, + 0.35946, + 0.3603, + 0.35836, + 0.36136, + 0.36142, + 0.36514, + 0.36682, + 0.36542, + 0.36722, + 0.36676, + 0.36892, + 0.36556, + 0.3685, + 0.37052, + 0.3698, + 0.37064, + 0.37166, + 0.37218, + 0.37034, + 0.37416, + 0.3751, + 0.37026, + 0.37354, + 0.377, + 0.37554, + 0.3771, + 0.37508, + 0.37768, + 0.37622, + 0.37688, + 0.3797, + 0.37742, + 0.3776, + 0.37718, + 0.37764, + 0.38156, + 0.3803, + 0.38212, + 0.38114, + 0.37938, + 0.38004, + 0.37876, + 0.38, + 0.37908, + 0.37994, + 0.38146, + 0.38242, + 0.38336, + 0.38234, + 0.38252, + 0.3794, + 0.38194, + 0.3836, + 0.3818, + 0.3845, + 0.3839, + 0.38346, + 0.385, + 0.38244, + 0.38364, + 0.38556, + 0.38002, + 0.38376 + ], + "test_acc": [ + 0.312, + 0.3272, + 0.3242, + 0.3601, + 0.3595, + 0.3486, + 0.3611, + 0.3545, + 0.3528, + 0.3443, + 0.3558, + 0.3534, + 0.3612, + 0.3665, + 0.3685, + 0.377, + 0.3552, + 0.3599, + 0.3585, + 0.3623, + 0.3614, + 0.3579, + 0.3684, + 0.3467, + 0.3545, + 0.3635, + 0.3553, + 0.3726, + 0.3762, + 0.3545, + 0.3576, + 0.3804, + 0.3752, + 0.3737, + 0.3726, + 0.3765, + 0.3783, + 0.3783, + 0.3863, + 0.3813, + 0.3837, + 0.3841, + 0.3885, + 0.3917, + 0.3897, + 0.3896, + 0.3819, + 0.3844, + 0.394, + 0.391, + 0.3903, + 0.3962, + 0.3984, + 0.3917, + 0.3958, + 0.3898, + 0.3995, + 0.3985, + 0.3973, + 0.3971, + 0.3961, + 0.3993, + 0.4031, + 0.396, + 0.3996, + 0.4015, + 0.4009, + 0.4025, + 0.4007, + 0.4015, + 0.4021, + 0.397, + 0.3991, + 0.405, + 0.4053, + 0.4045, + 0.4016, + 0.4024, + 0.4055, + 0.4033, + 0.4033, + 0.4071, + 0.4049, + 0.4068, + 0.4048, + 0.4049, + 0.4049, + 0.4039, + 0.4045, + 0.4055, + 0.4063, + 0.4046, + 0.406, + 0.4066, + 0.4061, + 0.4062, + 0.4059, + 0.406, + 0.4061, + 0.4062 + ] + }, + "diagnostics": { + "bp_cosine": [ + 0.010068703442811966, + 0.1245778352022171, + 0.09253278374671936, + 0.08268401026725769, + 0.05013114959001541, + 0.041134029626846313, + 0.017736738547682762, + 0.00011690100654959679, + -0.07415217161178589, + 0.024469580501317978, + -0.045304883271455765, + 0.9951483011245728 + ], + "perturbation_rho": [ + -0.006299033761024475, + 0.014822498895227909, + 0.003820352256298065, + 0.018621522933244705, + -0.0024977736175060272, + -0.0020938459783792496, + -0.018873507156968117, + 0.056286461651325226, + 0.008058521896600723, + 0.012508060783147812, + 0.03497573360800743, + -0.005283009260892868 + ], + "nudging": { + "0.001": [ + -8.329516276717186e-07, + -4.055909812450409e-07, + -1.9476283341646194e-07, + -1.578591763973236e-07, + -7.811468094587326e-08, + -4.6566128730773926e-08, + -2.0256265997886658e-08, + 1.7113052308559418e-08, + 2.9802322387695312e-08, + 6.984919309616089e-10, + 1.280568540096283e-08, + -5.175825208425522e-07 + ], + "0.003": [ + -2.361135557293892e-06, + -1.1622905731201172e-06, + -4.988396540284157e-07, + -3.294553607702255e-07, + -1.5029218047857285e-07, + -1.0221265256404877e-07, + -3.748573362827301e-08, + 1.4202669262886047e-08, + 1.280568540096283e-07, + -5.366746336221695e-08, + 6.51925802230835e-08, + -1.7427373677492142e-06 + ], + "0.01": [ + -7.986207492649555e-06, + -3.8052676245570183e-06, + -1.7613638192415237e-06, + -9.794021025300026e-07, + -5.171168595552444e-07, + -2.832384780049324e-07, + -1.4971010386943817e-07, + -7.182825356721878e-08, + 4.4121406972408295e-07, + -2.468004822731018e-07, + 2.0524021238088608e-07, + -6.320537067949772e-06 + ] + }, + "hidden_norms_per_layer": [ + 7009.81494140625, + 120237.1953125, + 234765.75, + 332169.78125, + 346779.53125, + 390112.375, + 463708.90625, + 668915.8125, + 1122988.5, + 1967430.625, + 1988171.0, + 2240154.75, + 1633562.75 + ], + "bp_grad_norms_per_layer": [ + 2.4624005163786933e-05, + 1.2478232065404882e-06, + 6.924460080881545e-07, + 5.278115509099734e-07, + 4.257075261193677e-07, + 3.652528164366231e-07, + 3.4045956454065163e-07, + 3.226113562959654e-07, + 3.1934513344822335e-07, + 3.1868384553490614e-07, + 3.1644472642256005e-07, + 3.1672169598095934e-07, + 3.163899009450688e-07 + ] + }, + "drift": { + "embed.weight": 47.005374347880874, + "embed.bias": 13.243067966808145, + "blocks.0.ln.weight": 1.2686282540844722, + "blocks.0.w1.weight": 17.553869325595105, + "blocks.0.w1.bias": 13.820052496499288, + "blocks.0.w2.weight": 60.50343593281807, + "blocks.1.ln.weight": 0.9603227875454038, + "blocks.1.w1.weight": 17.719833102219965, + "blocks.1.w1.bias": 8.605211656313518, + "blocks.1.w2.weight": 46.36885490302975, + "blocks.2.ln.weight": 0.8397655185331075, + "blocks.2.w1.weight": 17.32854478894422, + "blocks.2.w1.bias": 8.14502863318518, + "blocks.2.w2.weight": 46.8756195290007, + "blocks.3.ln.weight": 0.8514772849977531, + "blocks.3.w1.weight": 17.625201649093363, + "blocks.3.w1.bias": 4.891298958534819, + "blocks.3.w2.weight": 49.35104323212137, + "blocks.4.ln.weight": 0.7181238110900667, + "blocks.4.w1.weight": 17.236589560962, + "blocks.4.w1.bias": 6.048278381119212, + "blocks.4.w2.weight": 37.01349206570791, + "blocks.5.ln.weight": 0.7889239428916937, + "blocks.5.w1.weight": 18.624277821727198, + "blocks.5.w1.bias": 8.232516711563722, + "blocks.5.w2.weight": 34.44857991057927, + "blocks.6.ln.weight": 0.9087775281092089, + "blocks.6.w1.weight": 20.478021505238107, + "blocks.6.w1.bias": 9.991577823388699, + "blocks.6.w2.weight": 41.635213960063496, + "blocks.7.ln.weight": 0.8672691193491717, + "blocks.7.w1.weight": 23.031638161138872, + "blocks.7.w1.bias": 15.432108567671811, + "blocks.7.w2.weight": 30.494694204720314, + "blocks.8.ln.weight": 0.9279180982195232, + "blocks.8.w1.weight": 25.623083539907853, + "blocks.8.w1.bias": 21.104273424027447, + "blocks.8.w2.weight": 30.431336626547836, + "blocks.9.ln.weight": 0.6368710077977572, + "blocks.9.w1.weight": 17.55317251649144, + "blocks.9.w1.bias": 12.498837888749929, + "blocks.9.w2.weight": 63.802683742207456, + "blocks.10.ln.weight": 0.6261645701278474, + "blocks.10.w1.weight": 19.149302928700966, + "blocks.10.w1.bias": 19.63354445246865, + "blocks.10.w2.weight": 28.244603543415494, + "blocks.11.ln.weight": 0.7596072689153585, + "blocks.11.w1.weight": 19.668235342459408, + "blocks.11.w1.bias": 15.490287106269353, + "blocks.11.w2.weight": 55.03718141211597, + "out_ln.weight": 0.33939420479137183, + "out_head.weight": 5.440112536505558, + "out_head.bias": 2.010159660293023 + } + } + }, + "config": { + "dataset": "cifar10", + "d_hidden": 512, + "num_blocks": 12, + "batch_size": 128, + "epochs": 100, + "lr": 0.001, + "lr_fb": 0.001, + "wd": 0.01, + "lam": 0.1, + "K": 4, + "sigma_bridge": 0.05, + "ema_momentum": 0.995, + "term_grad_weight": 1.0, + "seeds": [ + 7 + ], + "gpu": 0, + "output_dir": "results/fa_dfa_d512_L12_seed7", + "methods": [ + "fa", + "dfa" + ], + "random_targets": false, + "penalty_lam": 0.0, + "num_classes": 10 + } +}
\ No newline at end of file |