diff options
Diffstat (limited to 'results/fa_dfa_d256_L12_seed3/results_cifar10.json')
| -rw-r--r-- | results/fa_dfa_d256_L12_seed3/results_cifar10.json | 969 |
1 files changed, 969 insertions, 0 deletions
diff --git a/results/fa_dfa_d256_L12_seed3/results_cifar10.json b/results/fa_dfa_d256_L12_seed3/results_cifar10.json new file mode 100644 index 0000000..b2495bc --- /dev/null +++ b/results/fa_dfa_d256_L12_seed3/results_cifar10.json @@ -0,0 +1,969 @@ +{ + "3": { + "dfa": { + "log": { + "train_loss": [ + 2.080998458023071, + 2.0535908667755125, + 2.0456236054992676, + 2.0437664754867555, + 2.04373485496521, + 2.0431359854125977, + 2.0434441787338256, + 2.0382570924377443, + 2.038818662261963, + 2.0345919523620606, + 2.03406599155426, + 2.031346089324951, + 2.0293883434677125, + 2.0278041764831545, + 2.0296552677154542, + 2.0259662225723267, + 2.0258015427017213, + 2.024105228805542, + 2.0241383403778075, + 2.0237019972991943, + 2.021524270172119, + 2.0192913208389283, + 2.020055650939941, + 2.0187213849639893, + 2.0182762395858767, + 2.018081063079834, + 2.0169390340805053, + 2.015762889480591, + 2.014286817321777, + 2.0149634075164795, + 2.012851970252991, + 2.0143834255218507, + 2.012589658126831, + 2.013470998764038, + 2.0132155655670165, + 2.0125345434570314, + 2.014651682510376, + 2.011008670730591, + 2.010692717514038, + 2.0098264505767824, + 2.010699557571411, + 2.010047173309326, + 2.0120795148086548, + 2.012862138290405, + 2.010627545700073, + 2.012053688697815, + 2.012078428192139, + 2.0087205919647215, + 2.010501960144043, + 2.0095251747131346, + 2.0100318467712404, + 2.009802308883667, + 2.008675379104614, + 2.009749626541138, + 2.0081673904037474, + 2.010975965423584, + 2.0075539178466797, + 2.007934888153076, + 2.0076504850769044, + 2.0063897485733033, + 2.007999542160034, + 2.0096467161560057, + 2.0087811013031005, + 2.0079173694229127, + 2.005330057106018, + 2.006340228881836, + 2.008018601989746, + 2.0075161946868896, + 2.0061075159454345, + 2.00557112739563, + 2.0092006335449217, + 2.005981811904907, + 2.008798143310547, + 2.006232444229126, + 2.0086249324798584, + 2.0055838273620608, + 2.0056044164276123, + 2.0061014500427246, + 2.00520422416687, + 2.0060690811538695, + 2.0042829446411132, + 2.007496082611084, + 2.006376414794922, + 2.007382208404541, + 2.0068653521347044, + 2.00715774230957, + 2.0046533392715453, + 2.0055563162994385, + 2.0043802153778074, + 2.00596639465332, + 2.0056586725234986, + 2.0065873177337648, + 2.006924060974121, + 2.007017402496338, + 2.007137279701233, + 2.007027236022949, + 2.005679884109497, + 2.00417349899292, + 2.006064287567139, + 2.0064190141296385 + ], + "train_acc": [ + 0.2315, + 0.2422, + 0.242, + 0.2424, + 0.24744, + 0.24618, + 0.24638, + 0.25122, + 0.2481, + 0.2553, + 0.25512, + 0.25802, + 0.2577, + 0.25796, + 0.2582, + 0.2618, + 0.26004, + 0.262, + 0.25824, + 0.26126, + 0.26322, + 0.26556, + 0.26414, + 0.26362, + 0.26414, + 0.2653, + 0.26648, + 0.26604, + 0.26904, + 0.26534, + 0.26784, + 0.26806, + 0.2689, + 0.26802, + 0.26812, + 0.26996, + 0.26976, + 0.26924, + 0.27148, + 0.26982, + 0.2701, + 0.27084, + 0.27122, + 0.27068, + 0.2739, + 0.26968, + 0.27164, + 0.27262, + 0.27076, + 0.2726, + 0.27058, + 0.26984, + 0.27202, + 0.27046, + 0.2735, + 0.27262, + 0.27144, + 0.27432, + 0.2739, + 0.27494, + 0.27442, + 0.2727, + 0.27466, + 0.2713, + 0.27138, + 0.27472, + 0.27394, + 0.27338, + 0.27562, + 0.274, + 0.27062, + 0.2747, + 0.27102, + 0.27396, + 0.27284, + 0.27616, + 0.27526, + 0.27456, + 0.27508, + 0.27278, + 0.27454, + 0.2754, + 0.27318, + 0.27526, + 0.27438, + 0.27674, + 0.27516, + 0.27372, + 0.27594, + 0.2748, + 0.27798, + 0.27314, + 0.27486, + 0.27464, + 0.2744, + 0.2761, + 0.27524, + 0.27572, + 0.27546, + 0.27596 + ], + "test_acc": [ + 0.2643, + 0.2485, + 0.2666, + 0.258, + 0.2678, + 0.2703, + 0.276, + 0.2618, + 0.2717, + 0.2773, + 0.2889, + 0.281, + 0.2829, + 0.2771, + 0.2647, + 0.2713, + 0.29, + 0.2779, + 0.2855, + 0.2833, + 0.2932, + 0.276, + 0.2898, + 0.2953, + 0.2799, + 0.2835, + 0.286, + 0.2849, + 0.2838, + 0.291, + 0.2948, + 0.2955, + 0.2873, + 0.2931, + 0.2908, + 0.2927, + 0.2896, + 0.2932, + 0.2863, + 0.2927, + 0.2886, + 0.2954, + 0.2871, + 0.2946, + 0.2925, + 0.2889, + 0.3005, + 0.2984, + 0.2954, + 0.2876, + 0.2852, + 0.2888, + 0.2943, + 0.2923, + 0.2934, + 0.2888, + 0.2941, + 0.2926, + 0.287, + 0.2969, + 0.2969, + 0.2895, + 0.2982, + 0.2961, + 0.2972, + 0.2998, + 0.2986, + 0.2932, + 0.2877, + 0.2875, + 0.2914, + 0.2958, + 0.2935, + 0.2963, + 0.2987, + 0.2948, + 0.2952, + 0.296, + 0.2993, + 0.2935, + 0.2962, + 0.2933, + 0.2948, + 0.2986, + 0.291, + 0.2968, + 0.2948, + 0.2976, + 0.2946, + 0.2958, + 0.2968, + 0.2977, + 0.2977, + 0.2966, + 0.2962, + 0.2963, + 0.2963, + 0.2966, + 0.2968, + 0.2968 + ] + }, + "diagnostics": { + "bp_cosine": [ + 0.384052038192749, + 0.00187331298366189, + 0.0014918153174221516, + -0.003990606404840946, + 0.0011513899080455303, + -0.0013353436952456832, + -0.0003460634616203606, + -0.0025712824426591396, + 0.0021760533563792706, + 0.0015363170532509685, + 0.0023321444168686867, + -0.0023729477543383837 + ], + "perturbation_rho": [ + 0.008436945267021656, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "nudging": { + "0.001": [ + -2.48197466135025e-07, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "0.003": [ + -1.144595444202423e-06, + -2.7939677238464355e-09, + 3.725290298461914e-09, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "0.01": [ + -3.5781413316726685e-06, + -4.6566128730773926e-09, + 3.725290298461914e-09, + -2.7939677238464355e-09, + -3.725290298461914e-09, + -6.51925802230835e-09, + 9.313225746154785e-10, + 0.0, + 0.0, + 0.0, + -3.725290298461914e-09, + 0.0 + ] + }, + "hidden_norms_per_layer": [ + 40862.5703125, + 302949600.0, + 779490368.0, + 866010048.0, + 1434764928.0, + 1708853376.0, + 1753628672.0, + 2062010112.0, + 2322754048.0, + 2426205952.0, + 2646477312.0, + 3046314752.0, + 3080729344.0 + ], + "bp_grad_norms_per_layer": [ + 3.3795413401094265e-07, + 9.012455781665096e-10, + 8.881588797748918e-10, + 8.880864932336863e-10, + 8.938522144674721e-10, + 8.938976781003305e-10, + 8.937018902699378e-10, + 8.93527418721618e-10, + 8.935119866215757e-10, + 8.93440765814546e-10, + 8.933446760117647e-10, + 8.934304962515682e-10, + 8.93500051724061e-10 + ] + }, + "drift": { + "embed.weight": 357.53469530895904, + "embed.bias": 290.04318739609596, + "blocks.0.ln.weight": 10.989575386047363, + "blocks.0.w1.weight": 220.1012762362677, + "blocks.0.w1.bias": 206.37217351266216, + "blocks.0.w2.weight": 501.3829415210294, + "blocks.1.ln.weight": 9.839688301086426, + "blocks.1.w1.weight": 256.6554332372174, + "blocks.1.w1.bias": 237.5977187566281, + "blocks.1.w2.weight": 396.28129110774046, + "blocks.2.ln.weight": 8.956626892089844, + "blocks.2.w1.weight": 225.58790523471438, + "blocks.2.w1.bias": 219.9504694961951, + "blocks.2.w2.weight": 332.68534972274165, + "blocks.3.ln.weight": 10.4323091506958, + "blocks.3.w1.weight": 304.362127476672, + "blocks.3.w1.bias": 271.4581804566994, + "blocks.3.w2.weight": 456.776518831265, + "blocks.4.ln.weight": 10.220683097839355, + "blocks.4.w1.weight": 294.88770495280846, + "blocks.4.w1.bias": 278.17483493032455, + "blocks.4.w2.weight": 422.86722605618826, + "blocks.5.ln.weight": 7.415699005126953, + "blocks.5.w1.weight": 190.23919300974276, + "blocks.5.w1.bias": 170.52475346607713, + "blocks.5.w2.weight": 282.6076353329658, + "blocks.6.ln.weight": 11.005321502685547, + "blocks.6.w1.weight": 315.1699742985849, + "blocks.6.w1.bias": 293.37584502861324, + "blocks.6.w2.weight": 428.46065418838674, + "blocks.7.ln.weight": 11.400951385498047, + "blocks.7.w1.weight": 320.1547970124374, + "blocks.7.w1.bias": 312.2252164433187, + "blocks.7.w2.weight": 444.7674866393189, + "blocks.8.ln.weight": 9.238995552062988, + "blocks.8.w1.weight": 261.6975850751966, + "blocks.8.w1.bias": 251.09895252390498, + "blocks.8.w2.weight": 352.9133162081499, + "blocks.9.ln.weight": 11.197455406188965, + "blocks.9.w1.weight": 315.7224672836386, + "blocks.9.w1.bias": 295.82606170712853, + "blocks.9.w2.weight": 416.36578033635686, + "blocks.10.ln.weight": 11.847606658935547, + "blocks.10.w1.weight": 338.01615076254603, + "blocks.10.w1.bias": 307.22711286489067, + "blocks.10.w2.weight": 464.68687019974544, + "blocks.11.ln.weight": 7.684996128082275, + "blocks.11.w1.weight": 188.9801145292558, + "blocks.11.w1.bias": 178.35247968031155, + "blocks.11.w2.weight": 244.64524232865998, + "out_ln.weight": 1.0227844715118408, + "out_head.weight": 10.424635519375226, + "out_head.bias": 0.7962212896591503 + } + }, + "fa": { + "log": { + "train_loss": [ + 2.1018029602050783, + 2.0231987742614748, + 1.9913426873779296, + 1.9757863534164428, + 1.9628748377227783, + 1.949777193145752, + 1.9388965420150757, + 1.9255553958511353, + 1.9167609970474244, + 1.9067581841278076, + 1.9039552599716187, + 1.899418589744568, + 1.8931139596176147, + 1.8900751802825928, + 1.8894958589935302, + 1.8822405550384522, + 1.8818315468597413, + 1.8777127625274659, + 1.8790236785125733, + 1.8758525806045532, + 1.8699211498641968, + 1.8684503372573853, + 1.868811384048462, + 1.8628138626098634, + 1.8617137921524047, + 1.859757089614868, + 1.8566949377059936, + 1.8563865858459472, + 1.8520149837493896, + 1.852164205932617, + 1.846489913673401, + 1.847879118347168, + 1.8489951482391358, + 1.8469820216369628, + 1.8453059420394897, + 1.8414616915893556, + 1.845604056930542, + 1.8367319815063476, + 1.838143812828064, + 1.8364125064468384, + 1.8365082810211182, + 1.8345620361328125, + 1.8362988974761962, + 1.8378243299102783, + 1.833498821182251, + 1.8334047562408446, + 1.8298064031600951, + 1.828000364303589, + 1.8265536352539062, + 1.829135093307495, + 1.8257994495391845, + 1.8226896044158936, + 1.826085818748474, + 1.8257194399261474, + 1.820882752685547, + 1.823325060119629, + 1.821622840309143, + 1.8205207012939453, + 1.8186096018218993, + 1.8195034896469116, + 1.8178193243026732, + 1.8202132829284667, + 1.8164496115112305, + 1.8147546480941772, + 1.8138668438720704, + 1.8105954584121704, + 1.8140293844604491, + 1.8142269690704347, + 1.8139710745239257, + 1.812066497116089, + 1.8160650396728515, + 1.8103228116607666, + 1.81037494846344, + 1.8109107693481445, + 1.8118848001480103, + 1.80986087184906, + 1.8117412914276123, + 1.8095279247283935, + 1.8082179739379882, + 1.8077979961395263, + 1.8045953066635132, + 1.8087207033920287, + 1.8068067630004883, + 1.8097841464996338, + 1.8086548376846314, + 1.8075193279266357, + 1.8061525051498413, + 1.8061482022476196, + 1.805157029685974, + 1.806752130088806, + 1.8043223245239257, + 1.8059532889175416, + 1.804686725730896, + 1.8053256253051757, + 1.8066633655166626, + 1.805836711769104, + 1.8080352004241944, + 1.803317499961853, + 1.801328492088318, + 1.8052216162109376 + ], + "train_acc": [ + 0.20742, + 0.24158, + 0.25614, + 0.26952, + 0.27686, + 0.28706, + 0.29562, + 0.30176, + 0.3048, + 0.30964, + 0.3102, + 0.31448, + 0.31506, + 0.31792, + 0.32218, + 0.3222, + 0.3216, + 0.3241, + 0.32336, + 0.32284, + 0.32752, + 0.3269, + 0.3251, + 0.33148, + 0.3297, + 0.32996, + 0.33296, + 0.333, + 0.3372, + 0.33618, + 0.336, + 0.33676, + 0.3361, + 0.33558, + 0.33656, + 0.34134, + 0.3399, + 0.34096, + 0.34498, + 0.34478, + 0.34308, + 0.34372, + 0.34374, + 0.34298, + 0.3446, + 0.34562, + 0.34566, + 0.34838, + 0.34956, + 0.34546, + 0.34762, + 0.34824, + 0.34838, + 0.34946, + 0.35068, + 0.35008, + 0.35006, + 0.35236, + 0.35148, + 0.35128, + 0.35232, + 0.3504, + 0.35356, + 0.3535, + 0.3542, + 0.35762, + 0.3522, + 0.35422, + 0.35474, + 0.3524, + 0.3533, + 0.3531, + 0.35642, + 0.356, + 0.35522, + 0.35664, + 0.35594, + 0.35494, + 0.3547, + 0.35866, + 0.35694, + 0.35732, + 0.35788, + 0.35534, + 0.35586, + 0.35614, + 0.35642, + 0.3569, + 0.35992, + 0.35596, + 0.35776, + 0.35608, + 0.35652, + 0.35726, + 0.35692, + 0.35758, + 0.35714, + 0.35754, + 0.35816, + 0.35878 + ], + "test_acc": [ + 0.2378, + 0.2506, + 0.2725, + 0.2937, + 0.3002, + 0.3155, + 0.3308, + 0.3192, + 0.3399, + 0.3382, + 0.3357, + 0.3429, + 0.3428, + 0.3431, + 0.3337, + 0.3418, + 0.3495, + 0.3491, + 0.3603, + 0.357, + 0.3506, + 0.3612, + 0.3593, + 0.3643, + 0.3621, + 0.3646, + 0.3621, + 0.3668, + 0.3625, + 0.3621, + 0.3598, + 0.3715, + 0.3627, + 0.37, + 0.373, + 0.3567, + 0.3681, + 0.3703, + 0.3744, + 0.3688, + 0.3656, + 0.3763, + 0.37, + 0.3705, + 0.3778, + 0.3681, + 0.3743, + 0.3804, + 0.3765, + 0.374, + 0.373, + 0.3763, + 0.3756, + 0.3773, + 0.3779, + 0.3734, + 0.3834, + 0.3806, + 0.3719, + 0.3795, + 0.3785, + 0.3767, + 0.3782, + 0.3827, + 0.3801, + 0.3815, + 0.3842, + 0.3829, + 0.38, + 0.3814, + 0.3821, + 0.3831, + 0.382, + 0.3796, + 0.3846, + 0.3786, + 0.3789, + 0.382, + 0.3786, + 0.3818, + 0.3831, + 0.3849, + 0.3807, + 0.3828, + 0.3799, + 0.382, + 0.3804, + 0.3823, + 0.3797, + 0.3821, + 0.3812, + 0.3807, + 0.3819, + 0.3817, + 0.3818, + 0.3809, + 0.3816, + 0.3816, + 0.3817, + 0.3818 + ] + }, + "diagnostics": { + "bp_cosine": [ + 0.027911242097616196, + 0.0825929045677185, + 0.0342608243227005, + -0.01674751378595829, + -0.02788889780640602, + -0.001998391468077898, + 0.007203143555670977, + -0.022960849106311798, + -0.001600538264028728, + -0.02186085283756256, + 0.013902194797992706, + 0.99517422914505 + ], + "perturbation_rho": [ + 0.024441849440336227, + 0.023016899824142456, + -0.07353583723306656, + 0.014823662117123604, + 0.003079652786254883, + -0.01875249855220318, + -0.01764325238764286, + -0.009929117746651173, + -0.04451639950275421, + 0.0320318341255188, + -0.0011818185448646545, + 0.022136203944683075 + ], + "nudging": { + "0.001": [ + -7.490161806344986e-07, + -2.367887645959854e-07, + -8.963979780673981e-08, + -9.546056389808655e-09, + -3.026798367500305e-09, + 2.7939677238464355e-08, + 2.60770320892334e-08, + 2.3283064365386963e-08, + 1.210719347000122e-08, + -1.3969838619232178e-08, + 1.862645149230957e-08, + -7.632188498973846e-07 + ], + "0.003": [ + -2.018176019191742e-06, + -6.172340363264084e-07, + -1.916196197271347e-07, + -3.3527612686157227e-08, + -1.210719347000122e-08, + -6.51925802230835e-08, + 2.514570951461792e-08, + 4.493631422519684e-08, + -4.912726581096649e-08, + 1.5366822481155396e-08, + -1.4901161193847656e-08, + -2.5206245481967926e-06 + ], + "0.01": [ + -6.553716957569122e-06, + -1.8514692783355713e-06, + -4.6426430344581604e-07, + 1.8603168427944183e-07, + 1.6205012798309326e-07, + -1.210719347000122e-08, + -1.2363307178020477e-07, + 2.6961788535118103e-07, + 1.83936208486557e-08, + 2.1792948246002197e-07, + -8.521601557731628e-08, + -9.013805538415909e-06 + ] + }, + "hidden_norms_per_layer": [ + 7627.2177734375, + 77014.5703125, + 261090.03125, + 402920.15625, + 531063.4375, + 650439.0625, + 771807.6875, + 876618.25, + 1137488.875, + 1214998.0, + 1291784.5, + 1356458.875, + 1186984.25 + ], + "bp_grad_norms_per_layer": [ + 1.7493783161626197e-05, + 1.0944853556793532e-06, + 5.515285579349438e-07, + 5.171301609152579e-07, + 5.162664820090868e-07, + 5.121610797687026e-07, + 5.100455382489599e-07, + 5.093034474157321e-07, + 5.131421971782402e-07, + 5.133904323884053e-07, + 5.137007974553853e-07, + 5.171222028366174e-07, + 5.011079906580562e-07 + ] + }, + "drift": { + "embed.weight": 71.59970957169052, + "embed.bias": 34.13643636527804, + "blocks.0.ln.weight": 1.511795163154602, + "blocks.0.w1.weight": 18.837866685893864, + "blocks.0.w1.bias": 13.70267468651505, + "blocks.0.w2.weight": 93.71863715934398, + "blocks.1.ln.weight": 1.3081063032150269, + "blocks.1.w1.weight": 21.240305602835985, + "blocks.1.w1.bias": 13.685504606064915, + "blocks.1.w2.weight": 61.028360088185984, + "blocks.2.ln.weight": 1.0054956674575806, + "blocks.2.w1.weight": 19.38818821685861, + "blocks.2.w1.bias": 16.089163443737224, + "blocks.2.w2.weight": 50.045867786081864, + "blocks.3.ln.weight": 0.7472347021102905, + "blocks.3.w1.weight": 19.199307011929896, + "blocks.3.w1.bias": 18.432771215086387, + "blocks.3.w2.weight": 37.33602068975751, + "blocks.4.ln.weight": 0.6882285475730896, + "blocks.4.w1.weight": 19.489052040551293, + "blocks.4.w1.bias": 20.122343743742668, + "blocks.4.w2.weight": 40.50107681410708, + "blocks.5.ln.weight": 0.7229293584823608, + "blocks.5.w1.weight": 19.290019148333055, + "blocks.5.w1.bias": 19.20270861199211, + "blocks.5.w2.weight": 44.78405827498682, + "blocks.6.ln.weight": 0.7572810649871826, + "blocks.6.w1.weight": 22.02089472340751, + "blocks.6.w1.bias": 22.158527807844386, + "blocks.6.w2.weight": 39.04536700764983, + "blocks.7.ln.weight": 0.7923915386199951, + "blocks.7.w1.weight": 22.987184790170787, + "blocks.7.w1.bias": 25.461720259715808, + "blocks.7.w2.weight": 43.420459766425545, + "blocks.8.ln.weight": 0.7664017081260681, + "blocks.8.w1.weight": 19.084502786198005, + "blocks.8.w1.bias": 19.896472419541702, + "blocks.8.w2.weight": 45.39367900417946, + "blocks.9.ln.weight": 0.7352473735809326, + "blocks.9.w1.weight": 18.514530575989, + "blocks.9.w1.bias": 19.155701487630136, + "blocks.9.w2.weight": 38.96532462321347, + "blocks.10.ln.weight": 0.7539297342300415, + "blocks.10.w1.weight": 18.499939513355947, + "blocks.10.w1.bias": 18.297859410049362, + "blocks.10.w2.weight": 33.573172219147224, + "blocks.11.ln.weight": 0.7084934711456299, + "blocks.11.w1.weight": 16.503202421262042, + "blocks.11.w1.bias": 12.571542518226021, + "blocks.11.w2.weight": 90.02901137589497, + "out_ln.weight": 0.44491103291511536, + "out_head.weight": 6.031580417677323, + "out_head.bias": 0.5341898346582782 + } + } + }, + "config": { + "dataset": "cifar10", + "d_hidden": 256, + "num_blocks": 12, + "batch_size": 128, + "epochs": 100, + "lr": 0.001, + "lr_fb": 0.001, + "wd": 0.01, + "lam": 0.1, + "K": 4, + "sigma_bridge": 0.05, + "ema_momentum": 0.995, + "term_grad_weight": 1.0, + "seeds": [ + 3 + ], + "gpu": 0, + "output_dir": "results/fa_dfa_d256_L12_seed3", + "methods": [ + "fa", + "dfa" + ], + "random_targets": false, + "penalty_lam": 0.0, + "num_classes": 10 + } +}
\ No newline at end of file |