diff options
Diffstat (limited to 'results/fa_dfa_d512_L8_seed7/results_cifar10.json')
| -rw-r--r-- | results/fa_dfa_d512_L8_seed7/results_cifar10.json | 881 |
1 files changed, 881 insertions, 0 deletions
diff --git a/results/fa_dfa_d512_L8_seed7/results_cifar10.json b/results/fa_dfa_d512_L8_seed7/results_cifar10.json new file mode 100644 index 0000000..1afcf83 --- /dev/null +++ b/results/fa_dfa_d512_L8_seed7/results_cifar10.json @@ -0,0 +1,881 @@ +{ + "7": { + "dfa": { + "log": { + "train_loss": [ + 2.0667303594207764, + 2.0324611769866943, + 2.0354494356536867, + 2.029560486984253, + 2.0285463153076173, + 2.0250388471984864, + 2.024455110168457, + 2.019448572616577, + 2.016967566986084, + 2.015476321334839, + 2.0111602995300295, + 2.011849951324463, + 2.009353925704956, + 2.0070058068084715, + 2.008019479827881, + 2.004458062477112, + 2.0052689391326903, + 1.9994013024520874, + 1.9999373209381104, + 1.998676916770935, + 1.9997672403717042, + 2.0009046767807006, + 1.9968020938491822, + 1.995622604598999, + 1.996247008934021, + 1.9960846052551269, + 1.994126517906189, + 1.9923845544433594, + 1.993994869995117, + 1.9906352404403687, + 1.993604411315918, + 1.9903970510864257, + 1.9910701220321656, + 1.9923721322250367, + 1.9914608909606935, + 1.990251951751709, + 1.9879735006332397, + 1.990091123123169, + 1.989167085533142, + 1.9870023511505126, + 1.9876113708496095, + 1.987236508255005, + 1.9868223078918457, + 1.986799183616638, + 1.9872047930145265, + 1.9893735236358643, + 1.985566644592285, + 1.9861162490081787, + 1.9881098473739625, + 1.9854161669540404, + 1.9856359400939942, + 1.9845690382385255, + 1.98623692237854, + 1.9848303674316405, + 1.985502159729004, + 1.9834946334075927, + 1.9829965356063843, + 1.982283455657959, + 1.984253911705017, + 1.9829701402282716, + 1.9815960638046264, + 1.9816721053314208, + 1.9825859942245483, + 1.980166824951172, + 1.9817002365493774, + 1.9804359143447876, + 1.9821267053604126, + 1.9813222342681884, + 1.980442075805664, + 1.9812367477416992, + 1.9801365872192382, + 1.980766887664795, + 1.979991530380249, + 1.9813088734436035, + 1.9798352686309815, + 1.979272435836792, + 1.9787259462738036, + 1.9776743465805053, + 1.9783772805404662, + 1.9771632527160645, + 1.9797238985824586, + 1.9785139586639404, + 1.978402091369629, + 1.9799914836883545, + 1.9797544343566895, + 1.9781852695465088, + 1.9793489038848877, + 1.97948084274292, + 1.9786187704849243, + 1.9765155652236939, + 1.9782553648376464, + 1.980494052734375, + 1.9774739639282226, + 1.9782270874786376, + 1.9778608634948731, + 1.9789943241119385, + 1.9787702282333375, + 1.9780272876358032, + 1.9778476651000976, + 1.9772668927764891 + ], + "train_acc": [ + 0.23342, + 0.24908, + 0.24646, + 0.24752, + 0.25104, + 0.25202, + 0.2535, + 0.25812, + 0.25712, + 0.2563, + 0.26068, + 0.25952, + 0.26266, + 0.261, + 0.26174, + 0.26348, + 0.26424, + 0.26552, + 0.2655, + 0.2661, + 0.26904, + 0.26678, + 0.27024, + 0.26984, + 0.26944, + 0.27048, + 0.26884, + 0.27052, + 0.27058, + 0.26958, + 0.27028, + 0.27444, + 0.27172, + 0.27178, + 0.27452, + 0.2702, + 0.27174, + 0.27434, + 0.27366, + 0.27622, + 0.27582, + 0.2751, + 0.27744, + 0.2755, + 0.2772, + 0.27482, + 0.27754, + 0.2768, + 0.27594, + 0.27634, + 0.27546, + 0.27758, + 0.27588, + 0.27868, + 0.27538, + 0.27782, + 0.27744, + 0.27858, + 0.27892, + 0.27808, + 0.27762, + 0.28194, + 0.27864, + 0.28092, + 0.27878, + 0.28206, + 0.27952, + 0.27822, + 0.28132, + 0.28178, + 0.2819, + 0.28098, + 0.28214, + 0.2826, + 0.28144, + 0.2839, + 0.28118, + 0.28474, + 0.28238, + 0.28314, + 0.28064, + 0.27932, + 0.28206, + 0.28026, + 0.28226, + 0.2815, + 0.28228, + 0.27942, + 0.2824, + 0.28296, + 0.2822, + 0.27914, + 0.28222, + 0.2813, + 0.28184, + 0.28122, + 0.28294, + 0.28278, + 0.28304, + 0.28276 + ], + "test_acc": [ + 0.2395, + 0.2584, + 0.278, + 0.2745, + 0.2714, + 0.2845, + 0.279, + 0.2849, + 0.2824, + 0.2736, + 0.2673, + 0.2695, + 0.2977, + 0.2632, + 0.2837, + 0.2899, + 0.2993, + 0.2917, + 0.2987, + 0.2881, + 0.2865, + 0.2963, + 0.307, + 0.2818, + 0.2905, + 0.297, + 0.3003, + 0.284, + 0.2824, + 0.3007, + 0.3045, + 0.3001, + 0.2875, + 0.3014, + 0.2948, + 0.2938, + 0.2869, + 0.3033, + 0.3048, + 0.2973, + 0.2897, + 0.2889, + 0.3027, + 0.309, + 0.2915, + 0.3052, + 0.3026, + 0.3069, + 0.2847, + 0.3014, + 0.3036, + 0.3065, + 0.3041, + 0.3006, + 0.296, + 0.3047, + 0.299, + 0.2993, + 0.3054, + 0.3044, + 0.3094, + 0.3083, + 0.2916, + 0.2964, + 0.3022, + 0.3068, + 0.3073, + 0.2973, + 0.3006, + 0.2954, + 0.3078, + 0.3015, + 0.3052, + 0.2978, + 0.3006, + 0.2996, + 0.3006, + 0.3029, + 0.2958, + 0.2935, + 0.3002, + 0.2972, + 0.3032, + 0.3043, + 0.3048, + 0.3077, + 0.3024, + 0.3073, + 0.3056, + 0.3067, + 0.3072, + 0.3058, + 0.3066, + 0.3057, + 0.3058, + 0.3058, + 0.306, + 0.3059, + 0.3055, + 0.3055 + ] + }, + "diagnostics": { + "bp_cosine": [ + 0.38665446639060974, + 0.000932278111577034, + -3.8507863791892305e-05, + 0.0002074055082630366, + 0.0004886630922555923, + -2.073507675959263e-05, + 9.8392330983188e-05, + -0.00026387785328552127 + ], + "perturbation_rho": [ + -0.015509183518588543, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "nudging": { + "0.001": [ + -4.4563785195350647e-07, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "0.003": [ + -1.2372620403766632e-06, + 0.0, + 9.313225746154785e-10, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "0.01": [ + -3.957655280828476e-06, + -5.587935447692871e-09, + 9.313225746154785e-10, + -3.725290298461914e-09, + 0.0, + 0.0, + 0.0, + 0.0 + ] + }, + "hidden_norms_per_layer": [ + 54526.05078125, + 813104704.0, + 2402701568.0, + 3855742976.0, + 4565018112.0, + 7107309056.0, + 10427996160.0, + 10594944000.0, + 10802217984.0 + ], + "bp_grad_norms_per_layer": [ + 2.7473959107737755e-07, + 1.7227932969099413e-10, + 1.702108870516028e-10, + 1.6996638818600474e-10, + 1.6991698326140892e-10, + 1.6980272743438718e-10, + 1.6994737561670803e-10, + 1.6996060114848888e-10, + 1.6999159024866373e-10 + ] + }, + "drift": { + "embed.weight": 327.4346996072575, + "embed.bias": 170.27189912672551, + "blocks.0.ln.weight": 10.029448340733337, + "blocks.0.w1.weight": 271.09694985433896, + "blocks.0.w1.bias": 210.3344533990309, + "blocks.0.w2.weight": 492.9386447153367, + "blocks.1.ln.weight": 8.828382497492473, + "blocks.1.w1.weight": 304.4175908830063, + "blocks.1.w1.bias": 277.43931427160345, + "blocks.1.w2.weight": 335.16168952866406, + "blocks.2.ln.weight": 8.838151125620117, + "blocks.2.w1.weight": 344.66383116618897, + "blocks.2.w1.bias": 306.24514451886506, + "blocks.2.w2.weight": 332.4200685911524, + "blocks.3.ln.weight": 7.729321470287778, + "blocks.3.w1.weight": 307.1583996497501, + "blocks.3.w1.bias": 280.7450378625442, + "blocks.3.w2.weight": 283.2991400409648, + "blocks.4.ln.weight": 10.279472891273015, + "blocks.4.w1.weight": 425.65902304929637, + "blocks.4.w1.bias": 405.30162642773695, + "blocks.4.w2.weight": 417.0126410755384, + "blocks.5.ln.weight": 11.518133724029006, + "blocks.5.w1.weight": 467.0126802605906, + "blocks.5.w1.bias": 452.55789718669956, + "blocks.5.w2.weight": 467.65851514176296, + "blocks.6.ln.weight": 9.037508419886805, + "blocks.6.w1.weight": 358.31655559732087, + "blocks.6.w1.bias": 330.36950370925376, + "blocks.6.w2.weight": 350.477939939699, + "blocks.7.ln.weight": 8.758836150161805, + "blocks.7.w1.weight": 347.69073535976315, + "blocks.7.w1.bias": 320.1104471013453, + "blocks.7.w2.weight": 332.8564062769089, + "out_ln.weight": 0.6350668697695233, + "out_head.weight": 9.197192445712407, + "out_head.bias": 0.8845643723650614 + } + }, + "fa": { + "log": { + "train_loss": [ + 2.0300379509735107, + 1.9401999561309815, + 1.913938886642456, + 1.8904116683578491, + 1.8764363840103149, + 1.873182268371582, + 1.8717842712402344, + 1.8644564879608154, + 1.8566048559570312, + 1.850687060508728, + 1.8460483242797852, + 1.8433684701538087, + 1.833488243789673, + 1.8278405266571045, + 1.8265484925079345, + 1.818034171447754, + 1.8155431734466552, + 1.8074933514022826, + 1.8059410440444947, + 1.8039924750137328, + 1.8001109600067138, + 1.7982124895477294, + 1.7878496411514282, + 1.7872236651611328, + 1.7860211572265625, + 1.7805663021469116, + 1.779324355392456, + 1.771492137527466, + 1.772376263961792, + 1.7669432696914673, + 1.7704145129776, + 1.7629687177276612, + 1.759504497718811, + 1.7608156897735596, + 1.7560134534072875, + 1.7523908859634398, + 1.7503845543670655, + 1.7478893075180053, + 1.7472649411773682, + 1.7424788817596435, + 1.74519405002594, + 1.7384309884262086, + 1.7379801452636718, + 1.7321213195800782, + 1.732442968559265, + 1.7347256170654297, + 1.7268475212860108, + 1.7259371692276, + 1.730874881286621, + 1.7255464766311646, + 1.7273355867767335, + 1.7193929712295533, + 1.7231387823104858, + 1.7167245266342164, + 1.7185063571929933, + 1.7160026277923583, + 1.71593848777771, + 1.7101723334121703, + 1.7123066550445556, + 1.7091578268432617, + 1.710422399520874, + 1.7071148763275146, + 1.703984679031372, + 1.7055902273559571, + 1.7043205630493163, + 1.702749596977234, + 1.7015760723114013, + 1.6997603458023072, + 1.7010435842895508, + 1.7012611554718018, + 1.7004539463043213, + 1.6962661600494384, + 1.6981626427841185, + 1.698192346496582, + 1.6980322202682494, + 1.6932597018814086, + 1.6941705449676514, + 1.6931912238311768, + 1.6942447135162353, + 1.6932169602203369, + 1.6921008915328979, + 1.691394665260315, + 1.692651755027771, + 1.690492251586914, + 1.6901156521606446, + 1.6877212131118775, + 1.6918383533859254, + 1.6916584229278564, + 1.6862779132843018, + 1.690645524635315, + 1.689451340637207, + 1.6918565393829346, + 1.6878593984222412, + 1.6896748791122436, + 1.6869253600692748, + 1.6864034613037109, + 1.6866692586517333, + 1.6912582437515258, + 1.6882677157211303, + 1.6879832342147827 + ], + "train_acc": [ + 0.25058, + 0.29132, + 0.30476, + 0.31654, + 0.32498, + 0.32736, + 0.32762, + 0.3327, + 0.33604, + 0.33698, + 0.33776, + 0.33972, + 0.34396, + 0.34484, + 0.34528, + 0.34798, + 0.351, + 0.35216, + 0.35302, + 0.35184, + 0.35472, + 0.3551, + 0.35822, + 0.35894, + 0.36172, + 0.36298, + 0.36054, + 0.36104, + 0.36522, + 0.36498, + 0.364, + 0.36804, + 0.36678, + 0.36776, + 0.3693, + 0.37214, + 0.37256, + 0.37198, + 0.37316, + 0.37604, + 0.37578, + 0.37634, + 0.37724, + 0.38006, + 0.38006, + 0.3758, + 0.38288, + 0.37976, + 0.37962, + 0.38206, + 0.38228, + 0.38356, + 0.38352, + 0.38466, + 0.38582, + 0.3873, + 0.3864, + 0.38692, + 0.38786, + 0.38986, + 0.38712, + 0.38938, + 0.389, + 0.39036, + 0.38952, + 0.3926, + 0.3918, + 0.39226, + 0.39226, + 0.39486, + 0.39342, + 0.39342, + 0.39574, + 0.39404, + 0.39118, + 0.39872, + 0.39752, + 0.39638, + 0.39602, + 0.39634, + 0.39706, + 0.39678, + 0.39506, + 0.39628, + 0.39606, + 0.39918, + 0.39864, + 0.39598, + 0.39636, + 0.39802, + 0.39958, + 0.39712, + 0.39732, + 0.39682, + 0.39824, + 0.39882, + 0.39722, + 0.39738, + 0.39682, + 0.39604 + ], + "test_acc": [ + 0.2982, + 0.3236, + 0.3362, + 0.355, + 0.3615, + 0.3565, + 0.3706, + 0.3689, + 0.3752, + 0.3721, + 0.3648, + 0.3757, + 0.358, + 0.3682, + 0.3671, + 0.3888, + 0.3867, + 0.3821, + 0.3853, + 0.3871, + 0.3942, + 0.387, + 0.3844, + 0.3793, + 0.3865, + 0.3944, + 0.3914, + 0.3841, + 0.3867, + 0.3946, + 0.3922, + 0.3911, + 0.3974, + 0.3923, + 0.4016, + 0.4003, + 0.398, + 0.3965, + 0.4044, + 0.4054, + 0.3959, + 0.3964, + 0.3997, + 0.4038, + 0.4096, + 0.4134, + 0.4049, + 0.4077, + 0.4074, + 0.4021, + 0.4087, + 0.4057, + 0.4133, + 0.415, + 0.4057, + 0.414, + 0.4129, + 0.4116, + 0.4118, + 0.4059, + 0.4124, + 0.414, + 0.4067, + 0.4166, + 0.4129, + 0.4136, + 0.4099, + 0.4092, + 0.418, + 0.4107, + 0.4178, + 0.4112, + 0.4128, + 0.4178, + 0.4138, + 0.4147, + 0.4157, + 0.4178, + 0.4179, + 0.4197, + 0.4188, + 0.414, + 0.416, + 0.4182, + 0.4217, + 0.4191, + 0.4173, + 0.4185, + 0.421, + 0.42, + 0.4176, + 0.4186, + 0.421, + 0.4193, + 0.4192, + 0.42, + 0.4192, + 0.42, + 0.4203, + 0.42 + ] + }, + "diagnostics": { + "bp_cosine": [ + 0.04720165580511093, + 0.023647643625736237, + -0.033746276050806046, + -0.02436545118689537, + -0.028243303298950195, + -0.01576988771557808, + -0.08274193108081818, + 0.9948431253433228 + ], + "perturbation_rho": [ + -0.019907839596271515, + 0.026347745209932327, + 0.00029688142240047455, + -0.015195554122328758, + 0.043242912739515305, + 0.008055397309362888, + 0.003536408767104149, + -0.039388515055179596 + ], + "nudging": { + "0.001": [ + -7.520895451307297e-06, + -1.26776285469532e-07, + 9.906943887472153e-08, + -4.656612873077393e-10, + -4.0978193283081055e-08, + -3.585591912269592e-08, + 6.740447133779526e-08, + -1.3819662854075432e-06 + ], + "0.003": [ + -2.2122403606772423e-05, + -3.012828528881073e-07, + 2.1152663975954056e-07, + 6.798654794692993e-08, + 1.9441358745098114e-07, + 2.6775524020195007e-08, + 3.398745320737362e-07, + -5.154346581548452e-06 + ], + "0.01": [ + -7.367332000285387e-05, + -1.0807416401803493e-06, + 6.495974957942963e-07, + 3.817840479314327e-07, + 5.302717909216881e-07, + 2.4068867787718773e-07, + 1.5024561434984207e-06, + -1.799390884116292e-05 + ] + }, + "hidden_norms_per_layer": [ + 5806.3515625, + 63154.71875, + 345991.1875, + 741559.875, + 923550.8125, + 1119541.375, + 1242674.125, + 1367644.625, + 780196.0625 + ], + "bp_grad_norms_per_layer": [ + 5.0447550165699795e-05, + 2.6470297598280013e-06, + 8.134598488140909e-07, + 7.909251280580065e-07, + 7.877957273194625e-07, + 7.858119488446391e-07, + 7.467841669495101e-07, + 7.52233631828858e-07, + 7.457880428773933e-07 + ] + }, + "drift": { + "embed.weight": 40.32473097964277, + "embed.bias": 16.685566967564547, + "blocks.0.ln.weight": 1.0445740862794894, + "blocks.0.w1.weight": 15.133344163582892, + "blocks.0.w1.bias": 12.448585097052113, + "blocks.0.w2.weight": 47.37614824821112, + "blocks.1.ln.weight": 0.9771977116283915, + "blocks.1.w1.weight": 19.65973837310052, + "blocks.1.w1.bias": 15.073340004663, + "blocks.1.w2.weight": 45.15493814957138, + "blocks.2.ln.weight": 0.7677471742012729, + "blocks.2.w1.weight": 18.286316393259757, + "blocks.2.w1.bias": 16.668691780331002, + "blocks.2.w2.weight": 43.33024072420251, + "blocks.3.ln.weight": 0.7157196705161631, + "blocks.3.w1.weight": 17.769177715348974, + "blocks.3.w1.bias": 17.48172961098867, + "blocks.3.w2.weight": 46.41797990739132, + "blocks.4.ln.weight": 0.7158938212971907, + "blocks.4.w1.weight": 18.203867219610753, + "blocks.4.w1.bias": 18.870397314936728, + "blocks.4.w2.weight": 51.23112107531505, + "blocks.5.ln.weight": 0.6884334631484388, + "blocks.5.w1.weight": 18.258883275015975, + "blocks.5.w1.bias": 18.71714897198837, + "blocks.5.w2.weight": 51.991090351166356, + "blocks.6.ln.weight": 0.633491884160565, + "blocks.6.w1.weight": 17.95984323437285, + "blocks.6.w1.bias": 18.493378505745028, + "blocks.6.w2.weight": 38.75838405347133, + "blocks.7.ln.weight": 0.5989570930744887, + "blocks.7.w1.weight": 18.705839523958478, + "blocks.7.w1.bias": 20.505853112621466, + "blocks.7.w2.weight": 33.59326297500316, + "out_ln.weight": 0.2917390152456112, + "out_head.weight": 5.883284575632486, + "out_head.bias": 0.8881859391580925 + } + } + }, + "config": { + "dataset": "cifar10", + "d_hidden": 512, + "num_blocks": 8, + "batch_size": 128, + "epochs": 100, + "lr": 0.001, + "lr_fb": 0.001, + "wd": 0.01, + "lam": 0.1, + "K": 4, + "sigma_bridge": 0.05, + "ema_momentum": 0.995, + "term_grad_weight": 1.0, + "seeds": [ + 7 + ], + "gpu": 0, + "output_dir": "results/fa_dfa_d512_L8_seed7", + "methods": [ + "fa", + "dfa" + ], + "random_targets": false, + "penalty_lam": 0.0, + "num_classes": 10 + } +}
\ No newline at end of file |