diff options
Diffstat (limited to 'results/fa_dfa_d256_L12_seed1/results_cifar10.json')
| -rw-r--r-- | results/fa_dfa_d256_L12_seed1/results_cifar10.json | 969 |
1 files changed, 969 insertions, 0 deletions
diff --git a/results/fa_dfa_d256_L12_seed1/results_cifar10.json b/results/fa_dfa_d256_L12_seed1/results_cifar10.json new file mode 100644 index 0000000..a5b7453 --- /dev/null +++ b/results/fa_dfa_d256_L12_seed1/results_cifar10.json @@ -0,0 +1,969 @@ +{ + "1": { + "dfa": { + "log": { + "train_loss": [ + 2.067703508453369, + 2.0509845906829836, + 2.0489810423278807, + 2.0436042738342284, + 2.0423764279174805, + 2.0355014089202883, + 2.033642913208008, + 2.031632952003479, + 2.0292669165039063, + 2.0284323851013184, + 2.0254091524505617, + 2.023782984008789, + 2.024598086013794, + 2.02109279624939, + 2.0222655739593507, + 2.017897210388184, + 2.017190932922363, + 2.0189253887939453, + 2.021160142478943, + 2.017966260147095, + 2.019014611053467, + 2.0170903842163086, + 2.0153890646743773, + 2.013819185180664, + 2.014828204040527, + 2.0162787924957275, + 2.013823607635498, + 2.0116753441619872, + 2.0123462033843995, + 2.015174687728882, + 2.012867821960449, + 2.012428879699707, + 2.0133413526153565, + 2.013093120727539, + 2.011299782371521, + 2.0090217667388917, + 2.0103736295318604, + 2.010286435699463, + 2.00893989654541, + 2.0116446880722045, + 2.0098753689193725, + 2.010866590499878, + 2.00980348903656, + 2.0109012869644167, + 2.00831172958374, + 2.00788239944458, + 2.0091643276596067, + 2.008021312484741, + 2.0090034342193603, + 2.007280367126465, + 2.0106113063812256, + 2.0081184278106687, + 2.0077548957824707, + 2.0058415910339353, + 2.007537776412964, + 2.005961977920532, + 2.006186770477295, + 2.0064545154190063, + 2.006464062461853, + 2.005965493736267, + 2.004203755187988, + 2.00689825302124, + 2.007209775085449, + 2.0072729249572756, + 2.002925030517578, + 2.005165770111084, + 2.0051349970245362, + 2.0035568313598633, + 2.004871815338135, + 2.003002490501404, + 2.0050700107574464, + 2.003749114379883, + 2.0044914908599853, + 2.006470828590393, + 2.0049134774017334, + 2.003252558746338, + 2.003931010055542, + 2.002243830718994, + 2.003598410110474, + 2.0027744925689697, + 2.0048628931427004, + 2.0027463244628905, + 2.005390119895935, + 2.0036273486328127, + 2.001874176979065, + 2.0030779739379883, + 2.002777744674683, + 2.002490601806641, + 2.0044283893585204, + 2.0020725644683837, + 2.0030053114318846, + 2.002065945777893, + 2.0020910046386717, + 2.0032121129989626, + 2.0020565942382813, + 2.001598520851135, + 2.0025369876098633, + 2.0027815823745727, + 2.002781241912842, + 2.003610602493286 + ], + "train_acc": [ + 0.23808, + 0.24056, + 0.2394, + 0.24146, + 0.24426, + 0.25002, + 0.2469, + 0.24916, + 0.25386, + 0.25294, + 0.25656, + 0.2569, + 0.25622, + 0.26112, + 0.25894, + 0.26168, + 0.26178, + 0.25928, + 0.25872, + 0.26202, + 0.26178, + 0.26178, + 0.26246, + 0.26322, + 0.26474, + 0.25964, + 0.2657, + 0.264, + 0.26538, + 0.26332, + 0.26432, + 0.2647, + 0.26426, + 0.26644, + 0.26712, + 0.2672, + 0.2667, + 0.2673, + 0.26496, + 0.2698, + 0.2697, + 0.26524, + 0.26698, + 0.26778, + 0.26776, + 0.26726, + 0.26798, + 0.26834, + 0.2708, + 0.27128, + 0.2668, + 0.26872, + 0.26908, + 0.26946, + 0.26962, + 0.26966, + 0.26966, + 0.27008, + 0.27042, + 0.2712, + 0.2724, + 0.26974, + 0.27206, + 0.2707, + 0.27216, + 0.27204, + 0.2715, + 0.27046, + 0.2716, + 0.27214, + 0.27192, + 0.27188, + 0.27328, + 0.27076, + 0.2717, + 0.27228, + 0.27288, + 0.27206, + 0.27244, + 0.27522, + 0.27138, + 0.27282, + 0.27212, + 0.27274, + 0.27264, + 0.27226, + 0.27054, + 0.27412, + 0.27054, + 0.27344, + 0.27234, + 0.27526, + 0.27328, + 0.27266, + 0.27276, + 0.27616, + 0.274, + 0.27256, + 0.27362, + 0.27134 + ], + "test_acc": [ + 0.2661, + 0.2453, + 0.247, + 0.2496, + 0.2729, + 0.2614, + 0.2432, + 0.2629, + 0.2751, + 0.2657, + 0.2483, + 0.2648, + 0.2763, + 0.2843, + 0.2744, + 0.2859, + 0.2707, + 0.271, + 0.2724, + 0.2858, + 0.2834, + 0.2832, + 0.282, + 0.2795, + 0.2799, + 0.2735, + 0.2569, + 0.2847, + 0.2724, + 0.2892, + 0.2797, + 0.2809, + 0.2752, + 0.2811, + 0.2655, + 0.281, + 0.294, + 0.2816, + 0.2759, + 0.2752, + 0.2755, + 0.285, + 0.2917, + 0.2948, + 0.2796, + 0.2865, + 0.2715, + 0.2946, + 0.2846, + 0.2885, + 0.2846, + 0.286, + 0.2885, + 0.2917, + 0.2879, + 0.2853, + 0.2897, + 0.2857, + 0.2834, + 0.2904, + 0.2719, + 0.2858, + 0.2896, + 0.2764, + 0.2892, + 0.2881, + 0.2824, + 0.2841, + 0.2824, + 0.29, + 0.2826, + 0.2896, + 0.2886, + 0.2921, + 0.2871, + 0.2847, + 0.2849, + 0.2907, + 0.2886, + 0.2851, + 0.2855, + 0.2885, + 0.2933, + 0.2877, + 0.2871, + 0.2865, + 0.2891, + 0.2887, + 0.2887, + 0.2858, + 0.2862, + 0.2881, + 0.2863, + 0.2877, + 0.2874, + 0.2879, + 0.2874, + 0.2872, + 0.2873, + 0.2872 + ] + }, + "diagnostics": { + "bp_cosine": [ + 0.3827846348285675, + 0.0010349926305934787, + 0.0028652632609009743, + -0.00039675208972766995, + -3.398433909751475e-05, + 0.0006842626025900245, + -0.0005091045168228447, + -0.0008636444108560681, + 0.0010306478943675756, + -0.0014248000225052238, + -0.0008004868868738413, + -0.001347829820588231 + ], + "perturbation_rho": [ + 0.03251604735851288, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "nudging": { + "0.001": [ + -3.8463622331619263e-07, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "0.003": [ + -1.0239891707897186e-06, + 9.313225746154785e-10, + 0.0, + 0.0, + 3.725290298461914e-09, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "0.01": [ + -3.2410025596618652e-06, + 9.313225746154785e-10, + 0.0, + 1.1175870895385742e-08, + 3.725290298461914e-09, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ] + }, + "hidden_norms_per_layer": [ + 44938.71484375, + 224396224.0, + 534405824.0, + 628434240.0, + 817915136.0, + 1014971776.0, + 1195768320.0, + 1696130048.0, + 1781777664.0, + 2129590784.0, + 2360990464.0, + 2604362240.0, + 2611689216.0 + ], + "bp_grad_norms_per_layer": [ + 3.2724244647397427e-07, + 4.958744526106784e-10, + 4.933069508439303e-10, + 4.927478425287291e-10, + 4.92776874860823e-10, + 4.929293639932553e-10, + 4.929709418455275e-10, + 4.923624841168817e-10, + 4.925106433795179e-10, + 4.924368135483803e-10, + 4.92419938158406e-10, + 4.924182728238691e-10, + 4.923914054266731e-10 + ] + }, + "drift": { + "embed.weight": 366.7568618351936, + "embed.bias": 309.0473623596911, + "blocks.0.ln.weight": 11.064080238342285, + "blocks.0.w1.weight": 209.81708023371544, + "blocks.0.w1.bias": 212.90654132913863, + "blocks.0.w2.weight": 491.9328056450382, + "blocks.1.ln.weight": 9.262639999389648, + "blocks.1.w1.weight": 235.0790709887316, + "blocks.1.w1.bias": 223.63959899284364, + "blocks.1.w2.weight": 344.39905753660923, + "blocks.2.ln.weight": 8.437020301818848, + "blocks.2.w1.weight": 201.71434530769594, + "blocks.2.w1.bias": 185.02882567315115, + "blocks.2.w2.weight": 318.26352419378276, + "blocks.3.ln.weight": 8.921281814575195, + "blocks.3.w1.weight": 253.83854854822596, + "blocks.3.w1.bias": 229.6894019696064, + "blocks.3.w2.weight": 350.3011606006072, + "blocks.4.ln.weight": 9.37590503692627, + "blocks.4.w1.weight": 261.7678873002099, + "blocks.4.w1.bias": 243.5300414079033, + "blocks.4.w2.weight": 342.796891596585, + "blocks.5.ln.weight": 9.026515007019043, + "blocks.5.w1.weight": 246.11946079088935, + "blocks.5.w1.bias": 237.3468691840109, + "blocks.5.w2.weight": 337.576876810432, + "blocks.6.ln.weight": 11.87130355834961, + "blocks.6.w1.weight": 323.004633882484, + "blocks.6.w1.bias": 302.8929133787546, + "blocks.6.w2.weight": 446.231359639456, + "blocks.7.ln.weight": 8.507142066955566, + "blocks.7.w1.weight": 235.44405159319476, + "blocks.7.w1.bias": 230.35367387868078, + "blocks.7.w2.weight": 329.8095182055512, + "blocks.8.ln.weight": 10.987072944641113, + "blocks.8.w1.weight": 320.7072155931754, + "blocks.8.w1.bias": 294.7395252801994, + "blocks.8.w2.weight": 454.98281282285984, + "blocks.9.ln.weight": 10.434910774230957, + "blocks.9.w1.weight": 294.34456133171307, + "blocks.9.w1.bias": 277.98968710676087, + "blocks.9.w2.weight": 386.31162290067874, + "blocks.10.ln.weight": 10.570181846618652, + "blocks.10.w1.weight": 301.76214096118804, + "blocks.10.w1.bias": 288.8696738168496, + "blocks.10.w2.weight": 412.44020515723406, + "blocks.11.ln.weight": 6.689911842346191, + "blocks.11.w1.weight": 178.97751193731568, + "blocks.11.w1.bias": 170.64222401222574, + "blocks.11.w2.weight": 255.10714043782204, + "out_ln.weight": 0.7581733465194702, + "out_head.weight": 7.801807644921005, + "out_head.bias": 0.42687402024736276 + } + }, + "fa": { + "log": { + "train_loss": [ + 2.0875839049530027, + 2.0147405715942384, + 1.9875884157562256, + 1.9586031070709229, + 1.9435594665908813, + 1.9274922204971314, + 1.9189438362121582, + 1.909316096420288, + 1.901387437400818, + 1.8944761938858032, + 1.8878041967773438, + 1.8841591590118407, + 1.8791934803009034, + 1.8707326766204835, + 1.8738373201751708, + 1.8677770760726928, + 1.8662226232910155, + 1.8621477996444702, + 1.8617159877777099, + 1.8547751527404785, + 1.8523059939193725, + 1.8457024936676025, + 1.8389914599609376, + 1.8356397885131837, + 1.8316144234466554, + 1.8286296670150757, + 1.8218667905044557, + 1.8144276452255248, + 1.807547642478943, + 1.8062635816192627, + 1.8020675115203857, + 1.7944365270233154, + 1.7917551394271851, + 1.792857387008667, + 1.786349270362854, + 1.7846514977645873, + 1.782985519180298, + 1.7807849361801147, + 1.776029171066284, + 1.7791108153533937, + 1.7775139881134032, + 1.774389083557129, + 1.7711814474487304, + 1.7737521209716798, + 1.7688932614135742, + 1.7682257052612305, + 1.7661459450531005, + 1.7686957946014403, + 1.764788871536255, + 1.7630911065292358, + 1.7634479265975953, + 1.7625660708999633, + 1.7607597863388063, + 1.7593693128204346, + 1.7575467151641846, + 1.7593074349975586, + 1.7534115236663819, + 1.7566318701934815, + 1.7595509714508057, + 1.7525636702728271, + 1.7542430780029297, + 1.7547497830581664, + 1.75442517868042, + 1.753666476097107, + 1.750391424331665, + 1.7509596431732177, + 1.7515560512542725, + 1.7488042358016969, + 1.750752509765625, + 1.7479970736312866, + 1.7519938071060182, + 1.747468376235962, + 1.7489017395782471, + 1.7493435995864868, + 1.7475122854232787, + 1.7483077466583252, + 1.7469019915390014, + 1.7457904458999634, + 1.7442825564956665, + 1.7459553696060182, + 1.7484514234161377, + 1.745931058731079, + 1.7454334391021729, + 1.743918479309082, + 1.745262015953064, + 1.744756409225464, + 1.7426206311035157, + 1.7429217789077758, + 1.7468717276382446, + 1.7410543600082398, + 1.7426005300140381, + 1.7455276907730102, + 1.7452946990966798, + 1.743043016319275, + 1.7401957970428468, + 1.7423173105621337, + 1.7431762436294556, + 1.7445746730804443, + 1.7446096383285523, + 1.7452752307510375 + ], + "train_acc": [ + 0.22256, + 0.25346, + 0.27028, + 0.2853, + 0.2961, + 0.30292, + 0.30572, + 0.30968, + 0.31464, + 0.31728, + 0.31962, + 0.32042, + 0.32064, + 0.32604, + 0.32566, + 0.32516, + 0.32768, + 0.32772, + 0.329, + 0.3322, + 0.33392, + 0.33814, + 0.34014, + 0.34284, + 0.3429, + 0.34292, + 0.34672, + 0.35008, + 0.35292, + 0.35346, + 0.35432, + 0.36024, + 0.35984, + 0.35798, + 0.35996, + 0.36256, + 0.36294, + 0.36324, + 0.3641, + 0.36242, + 0.36302, + 0.36536, + 0.36798, + 0.36558, + 0.36696, + 0.36854, + 0.36734, + 0.36774, + 0.36798, + 0.3704, + 0.36866, + 0.37204, + 0.3707, + 0.3689, + 0.3732, + 0.36918, + 0.373, + 0.37182, + 0.37306, + 0.3734, + 0.37252, + 0.37262, + 0.37084, + 0.37442, + 0.37388, + 0.37424, + 0.37434, + 0.37284, + 0.3746, + 0.3753, + 0.37408, + 0.37856, + 0.37516, + 0.37618, + 0.37696, + 0.377, + 0.3785, + 0.37692, + 0.37544, + 0.3788, + 0.37708, + 0.37568, + 0.37792, + 0.37742, + 0.37744, + 0.37678, + 0.37664, + 0.38078, + 0.3783, + 0.37778, + 0.38034, + 0.37784, + 0.37736, + 0.38046, + 0.37762, + 0.38042, + 0.37982, + 0.37818, + 0.37718, + 0.37814 + ], + "test_acc": [ + 0.2532, + 0.2635, + 0.3067, + 0.3126, + 0.3209, + 0.3326, + 0.3161, + 0.3304, + 0.3471, + 0.3408, + 0.3412, + 0.3556, + 0.3545, + 0.3571, + 0.3562, + 0.3655, + 0.3543, + 0.3596, + 0.3661, + 0.3604, + 0.3646, + 0.3656, + 0.3708, + 0.3724, + 0.3732, + 0.3702, + 0.3718, + 0.3739, + 0.3815, + 0.3877, + 0.3723, + 0.3882, + 0.3776, + 0.3921, + 0.3859, + 0.3901, + 0.3839, + 0.39, + 0.3898, + 0.3889, + 0.392, + 0.39, + 0.395, + 0.3926, + 0.3921, + 0.3946, + 0.3925, + 0.3904, + 0.4002, + 0.3963, + 0.3983, + 0.3951, + 0.4027, + 0.3978, + 0.4034, + 0.3987, + 0.3974, + 0.4003, + 0.3996, + 0.4018, + 0.3972, + 0.4031, + 0.4023, + 0.3997, + 0.4, + 0.4038, + 0.4005, + 0.3993, + 0.4026, + 0.4018, + 0.4028, + 0.4035, + 0.4069, + 0.4041, + 0.4043, + 0.4035, + 0.4046, + 0.4039, + 0.406, + 0.4059, + 0.4049, + 0.4046, + 0.4042, + 0.4055, + 0.4033, + 0.4054, + 0.405, + 0.4049, + 0.4048, + 0.4056, + 0.408, + 0.4068, + 0.4054, + 0.4042, + 0.404, + 0.4064, + 0.4056, + 0.4067, + 0.4065, + 0.4065 + ] + }, + "diagnostics": { + "bp_cosine": [ + 0.022406980395317078, + 0.047667622566223145, + 0.09574861824512482, + 0.035870544612407684, + -0.041190385818481445, + -0.0699436366558075, + -0.02966473624110222, + -0.039145052433013916, + -0.004860554356127977, + 0.005380071699619293, + -0.027314443141222, + 0.9947109222412109 + ], + "perturbation_rho": [ + -0.022866476327180862, + -0.018013648688793182, + 0.009727759286761284, + -0.053621724247932434, + -0.02228248305618763, + 0.0316929928958416, + 0.0015144720673561096, + 0.00410531647503376, + 0.03611733764410019, + 0.00018612481653690338, + -0.02358195371925831, + 0.011200916953384876 + ], + "nudging": { + "0.001": [ + -1.3886019587516785e-06, + -1.648440957069397e-07, + -2.230517566204071e-07, + -8.614733815193176e-08, + -2.2817403078079224e-08, + 5.2386894822120667e-08, + 1.257285475730896e-08, + -1.5366822481155396e-08, + -1.280568540096283e-08, + -1.83936208486557e-08, + 1.909211277961731e-08, + -7.841736078262329e-07 + ], + "0.003": [ + -3.930879756808281e-06, + -3.627501428127289e-07, + -4.2142346501350403e-07, + -1.5157274901866913e-07, + 3.9814040064811707e-08, + 1.9534491002559662e-07, + 3.003515303134918e-08, + 4.889443516731262e-08, + -2.1420419216156006e-08, + -6.239861249923706e-08, + 9.66247171163559e-08, + -2.8829090297222137e-06 + ], + "0.01": [ + -1.2913951650261879e-05, + -1.1620577424764633e-06, + -1.3329554349184036e-06, + -4.7474168241024017e-07, + 3.6065466701984406e-07, + 6.561167538166046e-07, + 1.7811544239521027e-07, + 4.209578037261963e-07, + 6.076879799365997e-08, + -1.2945383787155151e-07, + 2.454034984111786e-07, + -1.0041752830147743e-05 + ] + }, + "hidden_norms_per_layer": [ + 5485.41796875, + 46996.3671875, + 180271.890625, + 213572.234375, + 396517.875, + 588744.375, + 881525.8125, + 983474.1875, + 1058439.5, + 1218365.25, + 1317032.625, + 1334561.375, + 932176.0 + ], + "bp_grad_norms_per_layer": [ + 2.137463707185816e-05, + 1.5701441498094937e-06, + 7.100429115780571e-07, + 5.854868732058094e-07, + 5.86559053772362e-07, + 5.756642735832429e-07, + 5.825444304718985e-07, + 5.82348206989991e-07, + 5.825494895361771e-07, + 5.821411264150811e-07, + 5.809959020552924e-07, + 5.812668746330019e-07, + 5.751632556894037e-07 + ] + }, + "drift": { + "embed.weight": 58.38236439806917, + "embed.bias": 21.442522118983266, + "blocks.0.ln.weight": 1.5033414363861084, + "blocks.0.w1.weight": 16.882547993290178, + "blocks.0.w1.bias": 13.788910852251966, + "blocks.0.w2.weight": 79.37840244647813, + "blocks.1.ln.weight": 1.2546273469924927, + "blocks.1.w1.weight": 20.18123204529634, + "blocks.1.w1.bias": 12.541139360298107, + "blocks.1.w2.weight": 57.12310972297133, + "blocks.2.ln.weight": 1.1011525392532349, + "blocks.2.w1.weight": 18.73377758773642, + "blocks.2.w1.bias": 9.29578164795983, + "blocks.2.w2.weight": 54.28694557656785, + "blocks.3.ln.weight": 0.9108357429504395, + "blocks.3.w1.weight": 19.53507030946998, + "blocks.3.w1.bias": 19.00316399101826, + "blocks.3.w2.weight": 36.56504557755405, + "blocks.4.ln.weight": 0.9345387816429138, + "blocks.4.w1.weight": 21.803911980238134, + "blocks.4.w1.bias": 22.478543755429566, + "blocks.4.w2.weight": 34.17486626170071, + "blocks.5.ln.weight": 0.8433954119682312, + "blocks.5.w1.weight": 24.829617601702147, + "blocks.5.w1.bias": 27.438215191570773, + "blocks.5.w2.weight": 33.01528044399735, + "blocks.6.ln.weight": 0.7245873212814331, + "blocks.6.w1.weight": 20.82361019510065, + "blocks.6.w1.bias": 23.280464463082836, + "blocks.6.w2.weight": 34.072389985742944, + "blocks.7.ln.weight": 0.7635722756385803, + "blocks.7.w1.weight": 21.474198757222815, + "blocks.7.w1.bias": 23.449600463397505, + "blocks.7.w2.weight": 36.144241146743035, + "blocks.8.ln.weight": 0.9644007682800293, + "blocks.8.w1.weight": 22.392143221629144, + "blocks.8.w1.bias": 22.63072974134052, + "blocks.8.w2.weight": 52.64014420220465, + "blocks.9.ln.weight": 0.9803445339202881, + "blocks.9.w1.weight": 23.99208040262014, + "blocks.9.w1.bias": 25.016494427878733, + "blocks.9.w2.weight": 50.534240681340044, + "blocks.10.ln.weight": 0.8526968955993652, + "blocks.10.w1.weight": 19.026884395163023, + "blocks.10.w1.bias": 18.57529627609461, + "blocks.10.w2.weight": 64.66072262637215, + "blocks.11.ln.weight": 0.9996767044067383, + "blocks.11.w1.weight": 20.832721864932843, + "blocks.11.w1.bias": 20.006242016386533, + "blocks.11.w2.weight": 78.70619993314986, + "out_ln.weight": 0.4153711795806885, + "out_head.weight": 5.83941550812572, + "out_head.bias": 1.058698128024849 + } + } + }, + "config": { + "dataset": "cifar10", + "d_hidden": 256, + "num_blocks": 12, + "batch_size": 128, + "epochs": 100, + "lr": 0.001, + "lr_fb": 0.001, + "wd": 0.01, + "lam": 0.1, + "K": 4, + "sigma_bridge": 0.05, + "ema_momentum": 0.995, + "term_grad_weight": 1.0, + "seeds": [ + 1 + ], + "gpu": 0, + "output_dir": "results/fa_dfa_d256_L12_seed1", + "methods": [ + "fa", + "dfa" + ], + "random_targets": false, + "penalty_lam": 0.0, + "num_classes": 10 + } +}
\ No newline at end of file |