diff options
Diffstat (limited to 'results/fa_dfa_d512_L12_seed6/results_cifar10.json')
| -rw-r--r-- | results/fa_dfa_d512_L12_seed6/results_cifar10.json | 969 |
1 files changed, 969 insertions, 0 deletions
diff --git a/results/fa_dfa_d512_L12_seed6/results_cifar10.json b/results/fa_dfa_d512_L12_seed6/results_cifar10.json new file mode 100644 index 0000000..ccf37e3 --- /dev/null +++ b/results/fa_dfa_d512_L12_seed6/results_cifar10.json @@ -0,0 +1,969 @@ +{ + "6": { + "dfa": { + "log": { + "train_loss": [ + 2.0821874879455566, + 2.0482772801208498, + 2.047201838607788, + 2.0446329682159425, + 2.0396667890930176, + 2.036854875450134, + 2.035803484649658, + 2.0357003843688966, + 2.030295910644531, + 2.027509428253174, + 2.027801731262207, + 2.026494747467041, + 2.023713701324463, + 2.0224919934082033, + 2.0218288256835937, + 2.0183722845840455, + 2.0188908798217775, + 2.0174379263305666, + 2.0145048645782473, + 2.0172035501098633, + 2.01550293838501, + 2.013378846511841, + 2.015203825531006, + 2.016428701705933, + 2.011487791137695, + 2.010817546157837, + 2.01103433883667, + 2.011530419845581, + 2.0120579361724853, + 2.0116470166778564, + 2.008692329330444, + 2.009709241294861, + 2.008519637374878, + 2.0078509032440186, + 2.0077528884124756, + 2.006431950531006, + 2.0055743618011475, + 2.005430025024414, + 2.0056805452728272, + 2.007569041595459, + 2.004868392677307, + 2.0057909046936033, + 2.0064586280822754, + 2.005987978057861, + 2.00688329624176, + 2.00672064743042, + 2.0048528481292727, + 2.005383738861084, + 2.0018666375732423, + 2.0059955949401855, + 2.003435557899475, + 2.001197138328552, + 2.0012222803497313, + 2.003032783164978, + 2.003918945236206, + 2.0043967723083496, + 2.0035451693725586, + 2.003359081878662, + 2.001155726585388, + 2.001671184616089, + 2.0038728633880614, + 2.00498755947113, + 2.002938235092163, + 2.000668586883545, + 2.0018935997772216, + 2.0039958557128905, + 2.001610350112915, + 2.000534170913696, + 2.001975674972534, + 2.002237940979004, + 2.0009369605255127, + 2.003139477081299, + 2.00191648399353, + 2.003141210899353, + 1.9998547933578492, + 2.000947899513245, + 2.0015833082580565, + 2.0008020196151732, + 2.0014297840118407, + 2.00010564907074, + 2.0010018926239015, + 2.0015807587051393, + 2.0006349613952636, + 2.0010920279693605, + 2.000114615020752, + 2.001875145263672, + 2.001294381980896, + 2.001243436965942, + 1.9999162873077392, + 1.998048712120056, + 2.000587540740967, + 2.0002040885162353, + 2.00153217338562, + 2.0005496265411375, + 2.000687266998291, + 2.000036004295349, + 1.9989564297866822, + 1.9997446877288818, + 1.9997422592163085, + 2.000641355895996 + ], + "train_acc": [ + 0.23088, + 0.24298, + 0.242, + 0.24386, + 0.24422, + 0.24566, + 0.24754, + 0.24998, + 0.25214, + 0.25368, + 0.25348, + 0.2556, + 0.2568, + 0.25746, + 0.2569, + 0.25902, + 0.26116, + 0.2581, + 0.26084, + 0.26254, + 0.26022, + 0.26164, + 0.26092, + 0.26324, + 0.26598, + 0.26604, + 0.26308, + 0.26346, + 0.26408, + 0.26498, + 0.2656, + 0.26436, + 0.26714, + 0.26778, + 0.26842, + 0.26646, + 0.26986, + 0.26788, + 0.26944, + 0.2679, + 0.26858, + 0.269, + 0.26802, + 0.26894, + 0.26892, + 0.26996, + 0.26784, + 0.27018, + 0.26922, + 0.2704, + 0.26998, + 0.27222, + 0.2733, + 0.27004, + 0.2688, + 0.26874, + 0.26814, + 0.27284, + 0.26872, + 0.272, + 0.26908, + 0.27148, + 0.27176, + 0.27322, + 0.2711, + 0.26946, + 0.27102, + 0.27338, + 0.27226, + 0.2703, + 0.27122, + 0.27198, + 0.27138, + 0.27356, + 0.27036, + 0.27104, + 0.27176, + 0.27334, + 0.27254, + 0.27312, + 0.27288, + 0.27076, + 0.2713, + 0.27096, + 0.27414, + 0.27154, + 0.27082, + 0.27156, + 0.27418, + 0.27186, + 0.27248, + 0.27316, + 0.27146, + 0.27196, + 0.27248, + 0.27434, + 0.27286, + 0.27322, + 0.26998, + 0.272 + ], + "test_acc": [ + 0.2459, + 0.2608, + 0.2635, + 0.2517, + 0.253, + 0.2671, + 0.2464, + 0.2581, + 0.2752, + 0.2714, + 0.2801, + 0.2721, + 0.2781, + 0.2735, + 0.2757, + 0.2879, + 0.2731, + 0.2825, + 0.271, + 0.2808, + 0.2798, + 0.2898, + 0.2713, + 0.2935, + 0.2878, + 0.2873, + 0.268, + 0.2934, + 0.2831, + 0.2992, + 0.2915, + 0.2887, + 0.2797, + 0.2998, + 0.2957, + 0.2824, + 0.2818, + 0.289, + 0.2951, + 0.2906, + 0.2842, + 0.2772, + 0.2962, + 0.2715, + 0.3031, + 0.2834, + 0.299, + 0.2881, + 0.2849, + 0.2939, + 0.2801, + 0.279, + 0.2819, + 0.2841, + 0.3013, + 0.2908, + 0.2906, + 0.2845, + 0.2931, + 0.2816, + 0.2975, + 0.2963, + 0.2953, + 0.288, + 0.2836, + 0.2878, + 0.2906, + 0.2879, + 0.2969, + 0.2892, + 0.286, + 0.282, + 0.2914, + 0.2885, + 0.2892, + 0.2904, + 0.295, + 0.2865, + 0.2949, + 0.2902, + 0.2894, + 0.2898, + 0.2913, + 0.2929, + 0.2868, + 0.2869, + 0.2905, + 0.2895, + 0.29, + 0.2896, + 0.2882, + 0.2908, + 0.2912, + 0.2901, + 0.2893, + 0.2888, + 0.2895, + 0.2894, + 0.2896, + 0.2896 + ] + }, + "diagnostics": { + "bp_cosine": [ + 0.36949819326400757, + 0.0005479701794683933, + -0.0006032834062352777, + 0.00028805271722376347, + -0.0005946755409240723, + -0.0002685927611310035, + -0.00046556672896258533, + -0.00010807066428242251, + -0.0003351868945173919, + 0.00010062567162094638, + -0.000464106589788571, + -4.078936035512015e-05 + ], + "perturbation_rho": [ + -0.011094596236944199, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "nudging": { + "0.001": [ + -2.4400651454925537e-07, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "0.003": [ + -9.699724614620209e-07, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "0.01": [ + -3.3657997846603394e-06, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ] + }, + "hidden_norms_per_layer": [ + 55177.16015625, + 1459404800.0, + 2815841792.0, + 3553956096.0, + 3947802112.0, + 6848872448.0, + 7047305728.0, + 7256152064.0, + 9300344832.0, + 10056428544.0, + 10370693120.0, + 10568456192.0, + 11512604672.0 + ], + "bp_grad_norms_per_layer": [ + 2.3892781086942705e-07, + 2.0915398579735012e-10, + 2.0916321452624231e-10, + 2.0916490761635487e-10, + 2.0915806586696561e-10, + 2.0912831188990566e-10, + 2.0913649978471227e-10, + 2.0914339704525275e-10, + 2.089406564431684e-10, + 2.0894751207034545e-10, + 2.0897203412140186e-10, + 2.0899702801724374e-10, + 2.090319445313682e-10 + ] + }, + "drift": { + "embed.weight": 337.6841411506247, + "embed.bias": 244.65544843260258, + "blocks.0.ln.weight": 9.921388539749072, + "blocks.0.w1.weight": 303.34325560691656, + "blocks.0.w1.bias": 271.5154938432058, + "blocks.0.w2.weight": 490.11291855747174, + "blocks.1.ln.weight": 8.358331837822632, + "blocks.1.w1.weight": 306.46180391751284, + "blocks.1.w1.bias": 289.91192320867975, + "blocks.1.w2.weight": 307.16882665457837, + "blocks.2.ln.weight": 8.171380552921143, + "blocks.2.w1.weight": 309.0280257845424, + "blocks.2.w1.bias": 278.09046067069085, + "blocks.2.w2.weight": 303.83002249941313, + "blocks.3.ln.weight": 7.530980420849777, + "blocks.3.w1.weight": 291.36754150240233, + "blocks.3.w1.bias": 267.02199332509406, + "blocks.3.w2.weight": 279.4411092845991, + "blocks.4.ln.weight": 10.6408318843949, + "blocks.4.w1.weight": 441.74974808763915, + "blocks.4.w1.bias": 407.5025978091534, + "blocks.4.w2.weight": 396.8841237755263, + "blocks.5.ln.weight": 7.3188489223429976, + "blocks.5.w1.weight": 282.40725775096377, + "blocks.5.w1.bias": 268.64915470838923, + "blocks.5.w2.weight": 255.5102203441866, + "blocks.6.ln.weight": 7.484712621287041, + "blocks.6.w1.weight": 289.3900145716271, + "blocks.6.w1.bias": 273.2836157437038, + "blocks.6.w2.weight": 269.0639311186208, + "blocks.7.ln.weight": 10.645635950857786, + "blocks.7.w1.weight": 435.3403361556039, + "blocks.7.w1.bias": 407.2628750706809, + "blocks.7.w2.weight": 422.88459144783326, + "blocks.8.ln.weight": 9.19535275353374, + "blocks.8.w1.weight": 359.87499481651184, + "blocks.8.w1.bias": 333.4216491141516, + "blocks.8.w2.weight": 337.7077986951993, + "blocks.9.ln.weight": 7.799912385129816, + "blocks.9.w1.weight": 297.0043322415887, + "blocks.9.w1.bias": 289.12017704180437, + "blocks.9.w2.weight": 272.7064790358395, + "blocks.10.ln.weight": 8.43728199015399, + "blocks.10.w1.weight": 330.9678935021042, + "blocks.10.w1.bias": 306.6806776473047, + "blocks.10.w2.weight": 309.957908747728, + "blocks.11.ln.weight": 10.237677378176008, + "blocks.11.w1.weight": 400.7826698559754, + "blocks.11.w1.bias": 368.5214108931406, + "blocks.11.w2.weight": 374.837703368063, + "out_ln.weight": 0.67814086539114, + "out_head.weight": 9.623079895238506, + "out_head.bias": 0.4268739303655545 + } + }, + "fa": { + "log": { + "train_loss": [ + 2.05228896446228, + 1.9686302278900147, + 1.937331941871643, + 1.9156952641296388, + 1.8984398557281494, + 1.8918078393936157, + 1.8841382266616822, + 1.8769169528198242, + 1.8724429541397094, + 1.868889035949707, + 1.8656937331390382, + 1.8661101271820069, + 1.8578856301116944, + 1.8565830435180664, + 1.856787547569275, + 1.8557513214492798, + 1.8538894052124024, + 1.8534770782089234, + 1.848385778579712, + 1.846604812889099, + 1.8453827571868897, + 1.8429374709320068, + 1.8401580144500733, + 1.8401220611190796, + 1.8288005982208253, + 1.8259171481323242, + 1.8221615615463256, + 1.8216705709075929, + 1.821623645401001, + 1.818600112838745, + 1.812706976966858, + 1.8111941592788696, + 1.8062977261734008, + 1.805511683959961, + 1.8029492014312745, + 1.800798831100464, + 1.7953546783065797, + 1.7982503284454345, + 1.7914396514511108, + 1.7916672002792358, + 1.7845456259155272, + 1.78576413356781, + 1.783857018814087, + 1.7810697037506102, + 1.7796821146011352, + 1.7769692651367188, + 1.778120286178589, + 1.7701848324966432, + 1.7680266930770874, + 1.7723119870376587, + 1.7669956749725342, + 1.764911948814392, + 1.7598207390975953, + 1.7630078075408935, + 1.7617963638687133, + 1.7589407592391968, + 1.7578953286361694, + 1.755873950805664, + 1.7555263995742798, + 1.7516939831924438, + 1.7528417974090575, + 1.7498321814346314, + 1.7489991750335694, + 1.744125429725647, + 1.7467560472869874, + 1.7484809857940673, + 1.7442166653823852, + 1.7426679946136474, + 1.7441077096939086, + 1.7404415154647828, + 1.734541618347168, + 1.7417204373931885, + 1.7390217791748046, + 1.7405015720367432, + 1.739689345970154, + 1.7386328066253662, + 1.7377501021575927, + 1.7382229836654663, + 1.7361252673721312, + 1.7335023355865478, + 1.7335243579483033, + 1.7341424390029907, + 1.7341539621734618, + 1.7341088440322876, + 1.7317719327545167, + 1.7339295767593383, + 1.7329442672729491, + 1.7313460754776, + 1.7338177963256836, + 1.7318969781112672, + 1.7321792919921875, + 1.7318955504989624, + 1.7337695608520507, + 1.7288571945953368, + 1.7327508489990235, + 1.7287590840911866, + 1.730326948890686, + 1.7341352017593383, + 1.7290034577178954, + 1.72969926902771 + ], + "train_acc": [ + 0.2384, + 0.27844, + 0.2943, + 0.30138, + 0.30742, + 0.30942, + 0.3195, + 0.31888, + 0.32204, + 0.32418, + 0.3248, + 0.32782, + 0.33146, + 0.33268, + 0.32912, + 0.33066, + 0.33278, + 0.33552, + 0.3367, + 0.34006, + 0.3393, + 0.33932, + 0.3395, + 0.33874, + 0.34818, + 0.34772, + 0.34958, + 0.3503, + 0.35032, + 0.35086, + 0.35212, + 0.35332, + 0.35488, + 0.3559, + 0.35578, + 0.35756, + 0.35926, + 0.35818, + 0.36006, + 0.3605, + 0.36376, + 0.3638, + 0.36622, + 0.36562, + 0.36488, + 0.36572, + 0.36658, + 0.3694, + 0.37002, + 0.36866, + 0.37106, + 0.37212, + 0.37664, + 0.37196, + 0.37114, + 0.37274, + 0.37342, + 0.37482, + 0.37588, + 0.37478, + 0.37494, + 0.3766, + 0.37674, + 0.3785, + 0.37762, + 0.37722, + 0.37866, + 0.38074, + 0.37858, + 0.3806, + 0.38312, + 0.37868, + 0.3786, + 0.38086, + 0.37808, + 0.37986, + 0.38126, + 0.38078, + 0.38162, + 0.3829, + 0.38342, + 0.3823, + 0.3824, + 0.38028, + 0.38296, + 0.383, + 0.38242, + 0.38388, + 0.3826, + 0.38382, + 0.38268, + 0.38512, + 0.38288, + 0.38526, + 0.38352, + 0.38476, + 0.3849, + 0.38008, + 0.3831, + 0.3847 + ], + "test_acc": [ + 0.2743, + 0.3072, + 0.3046, + 0.3269, + 0.3349, + 0.3431, + 0.3427, + 0.3386, + 0.3512, + 0.3518, + 0.3496, + 0.3458, + 0.3629, + 0.3681, + 0.364, + 0.3676, + 0.3514, + 0.3685, + 0.353, + 0.3779, + 0.3693, + 0.3647, + 0.3759, + 0.3607, + 0.374, + 0.3735, + 0.3711, + 0.3721, + 0.3821, + 0.3752, + 0.3707, + 0.3851, + 0.3838, + 0.3893, + 0.374, + 0.392, + 0.3839, + 0.3977, + 0.3956, + 0.3924, + 0.3957, + 0.3936, + 0.394, + 0.3912, + 0.3993, + 0.3945, + 0.3978, + 0.3946, + 0.398, + 0.3963, + 0.3976, + 0.3972, + 0.3971, + 0.3984, + 0.3988, + 0.3958, + 0.4012, + 0.4042, + 0.4017, + 0.3974, + 0.3975, + 0.4052, + 0.4061, + 0.4066, + 0.4056, + 0.4057, + 0.406, + 0.4017, + 0.4026, + 0.4095, + 0.4043, + 0.4072, + 0.4077, + 0.4031, + 0.4083, + 0.407, + 0.4077, + 0.4099, + 0.4067, + 0.4065, + 0.4057, + 0.4092, + 0.4093, + 0.4055, + 0.4097, + 0.4081, + 0.4083, + 0.407, + 0.407, + 0.4086, + 0.4084, + 0.4074, + 0.4073, + 0.4072, + 0.4063, + 0.4075, + 0.407, + 0.4074, + 0.4075, + 0.4076 + ] + }, + "diagnostics": { + "bp_cosine": [ + 0.0210530087351799, + 0.03225391358137131, + 0.030373934656381607, + -0.01976676471531391, + -0.039145730435848236, + -0.07764401286840439, + -0.04457290470600128, + -0.0037843044847249985, + -0.0476202666759491, + -0.000595096789766103, + 0.02137608267366886, + 0.995658278465271 + ], + "perturbation_rho": [ + -0.0567036010324955, + -0.0030030906200408936, + -0.022865712642669678, + -0.009168766438961029, + -0.0182923823595047, + 0.03648798167705536, + 0.04260994493961334, + 0.0037925627548247576, + -0.03891247510910034, + -0.052096571773290634, + -0.05355001986026764, + 0.005782400257885456 + ], + "nudging": { + "0.001": [ + -7.315538823604584e-07, + -1.1478550732135773e-07, + -6.658956408500671e-08, + 7.450580596923828e-09, + 3.702007234096527e-08, + 5.634501576423645e-08, + 5.51808625459671e-08, + 9.080395102500916e-09, + 2.7241185307502747e-08, + -1.3969838619232178e-09, + -3.3527612686157227e-08, + -7.827766239643097e-07 + ], + "0.003": [ + -2.304092049598694e-06, + -3.343448042869568e-07, + -1.4924444258213043e-07, + 1.979060471057892e-08, + 1.2828968465328217e-07, + 2.391170710325241e-07, + 1.1990778148174286e-07, + -4.959292709827423e-08, + 1.4039687812328339e-07, + -6.752088665962219e-09, + -3.050081431865692e-08, + -2.78581865131855e-06 + ], + "0.01": [ + -7.684342563152313e-06, + -1.07521191239357e-06, + -4.4563785195350647e-07, + 1.5692785382270813e-07, + 3.7904828786849976e-07, + 7.476191967725754e-07, + 4.507601261138916e-07, + -3.306195139884949e-08, + 5.138572305440903e-07, + 2.9569491744041443e-08, + -2.5634653866291046e-07, + -9.92906279861927e-06 + ] + }, + "hidden_norms_per_layer": [ + 7711.2001953125, + 86607.8359375, + 328385.03125, + 717089.75, + 1100718.375, + 1366338.125, + 1659771.0, + 1846383.5, + 2026219.75, + 2144626.25, + 2170055.0, + 2174719.5, + 1710646.75 + ], + "bp_grad_norms_per_layer": [ + 2.5522365831420757e-05, + 1.6940738305493142e-06, + 5.418342539087462e-07, + 4.801818249688949e-07, + 4.7660145696681866e-07, + 4.771524686475459e-07, + 4.772307420353172e-07, + 4.777210733664106e-07, + 4.776471200784727e-07, + 4.787051466337289e-07, + 4.790993557435286e-07, + 4.780670792570163e-07, + 4.5420676997309783e-07 + ] + }, + "drift": { + "embed.weight": 51.639575695075386, + "embed.bias": 13.420626933240067, + "blocks.0.ln.weight": 1.165106572944437, + "blocks.0.w1.weight": 17.074257617816983, + "blocks.0.w1.bias": 12.55405788253026, + "blocks.0.w2.weight": 56.84408822778898, + "blocks.1.ln.weight": 1.0814484489664455, + "blocks.1.w1.weight": 20.255692646568928, + "blocks.1.w1.bias": 12.590175436666444, + "blocks.1.w2.weight": 48.43073026110698, + "blocks.2.ln.weight": 0.9210710896806559, + "blocks.2.w1.weight": 20.147092711724806, + "blocks.2.w1.bias": 16.898392906551216, + "blocks.2.w2.weight": 46.45488641124572, + "blocks.3.ln.weight": 0.6699388356494572, + "blocks.3.w1.weight": 19.04886241071522, + "blocks.3.w1.bias": 18.822886783231823, + "blocks.3.w2.weight": 35.62237142992444, + "blocks.4.ln.weight": 0.5724221532943284, + "blocks.4.w1.weight": 18.93452177775512, + "blocks.4.w1.bias": 18.487677483264793, + "blocks.4.w2.weight": 30.11841300499853, + "blocks.5.ln.weight": 0.5318503739647785, + "blocks.5.w1.weight": 19.752732313152933, + "blocks.5.w1.bias": 21.012803450001744, + "blocks.5.w2.weight": 26.781939546320714, + "blocks.6.ln.weight": 0.4938677177304124, + "blocks.6.w1.weight": 18.79282421249984, + "blocks.6.w1.bias": 20.24614504362752, + "blocks.6.w2.weight": 23.954722502437257, + "blocks.7.ln.weight": 0.520902940610959, + "blocks.7.w1.weight": 19.15384116500715, + "blocks.7.w1.bias": 20.22154003333216, + "blocks.7.w2.weight": 28.09055042594723, + "blocks.8.ln.weight": 0.5342907608290218, + "blocks.8.w1.weight": 18.38918909035725, + "blocks.8.w1.bias": 18.8314837769314, + "blocks.8.w2.weight": 31.203969055913323, + "blocks.9.ln.weight": 0.5198835347685796, + "blocks.9.w1.weight": 17.551735719168683, + "blocks.9.w1.bias": 17.643006584631404, + "blocks.9.w2.weight": 50.56607469544128, + "blocks.10.ln.weight": 0.4499630077104289, + "blocks.10.w1.weight": 15.14684109843433, + "blocks.10.w1.bias": 12.774970121083086, + "blocks.10.w2.weight": 53.261273057743395, + "blocks.11.ln.weight": 0.4529064172202585, + "blocks.11.w1.weight": 17.52348612725844, + "blocks.11.w1.bias": 15.699238779618904, + "blocks.11.w2.weight": 54.34198491601119, + "out_ln.weight": 0.38172881823690796, + "out_head.weight": 6.667353449848178, + "out_head.bias": 0.7683726336162054 + } + } + }, + "config": { + "dataset": "cifar10", + "d_hidden": 512, + "num_blocks": 12, + "batch_size": 128, + "epochs": 100, + "lr": 0.001, + "lr_fb": 0.001, + "wd": 0.01, + "lam": 0.1, + "K": 4, + "sigma_bridge": 0.05, + "ema_momentum": 0.995, + "term_grad_weight": 1.0, + "seeds": [ + 6 + ], + "gpu": 0, + "output_dir": "results/fa_dfa_d512_L12_seed6", + "methods": [ + "fa", + "dfa" + ], + "random_targets": false, + "penalty_lam": 0.0, + "num_classes": 10 + } +}
\ No newline at end of file |