diff options
Diffstat (limited to 'results/fa_dfa_d512_L6_seed2/results_cifar10.json')
| -rw-r--r-- | results/fa_dfa_d512_L6_seed2/results_cifar10.json | 837 |
1 files changed, 837 insertions, 0 deletions
diff --git a/results/fa_dfa_d512_L6_seed2/results_cifar10.json b/results/fa_dfa_d512_L6_seed2/results_cifar10.json new file mode 100644 index 0000000..0d6122d --- /dev/null +++ b/results/fa_dfa_d512_L6_seed2/results_cifar10.json @@ -0,0 +1,837 @@ +{ + "2": { + "dfa": { + "log": { + "train_loss": [ + 2.0704764087677003, + 2.049675481185913, + 2.0393172933197024, + 2.0385254068756105, + 2.0333104403305056, + 2.037317512435913, + 2.035111841583252, + 2.033784921951294, + 2.0276396758270265, + 2.0287634895324707, + 2.0255478504943847, + 2.0249928087997437, + 2.0251559844207763, + 2.026155082015991, + 2.0227601895141603, + 2.0238794365692137, + 2.0220592249298095, + 2.0211007302474977, + 2.0220025036621094, + 2.0187504733657837, + 2.022589133682251, + 2.0209386083221434, + 2.022369726409912, + 2.019175484046936, + 2.0184574550628662, + 2.020228290786743, + 2.0170737294769285, + 2.0200500258636476, + 2.0192940770721437, + 2.017934571380615, + 2.018111932220459, + 2.019744345703125, + 2.0181864041137696, + 2.018692120895386, + 2.019279464416504, + 2.020759454421997, + 2.0175593240356444, + 2.0189450312805177, + 2.018363641014099, + 2.0204193601989746, + 2.018774921836853, + 2.020020353240967, + 2.020256604042053, + 2.018686157913208, + 2.017777068939209, + 2.01642233127594, + 2.0193701077270507, + 2.0175296588897704, + 2.0180459953308105, + 2.0163426136016844, + 2.0169748413085937, + 2.017522889404297, + 2.0171759409332277, + 2.0182337244415285, + 2.0202614056396486, + 2.0193526766967773, + 2.0176297589874266, + 2.0176475778961183, + 2.017951414642334, + 2.0159765984725952, + 2.0174861521911622, + 2.0162583628082276, + 2.0189596444702147, + 2.017551812057495, + 2.0174899821472168, + 2.017485563354492, + 2.0167667126464845, + 2.017466169166565, + 2.017418462524414, + 2.0175614359283447, + 2.016368521194458, + 2.017146753158569, + 2.016855639877319, + 2.016222254562378, + 2.016020330963135, + 2.0157443789672853, + 2.014490191497803, + 2.0179072814559937, + 2.0176085803985595, + 2.016617900123596, + 2.016193874168396, + 2.0163377814483643, + 2.017562115097046, + 2.017051735992432, + 2.015485999984741, + 2.0135854721450808, + 2.0146356997299195, + 2.0149522621154787, + 2.0146646865844726, + 2.0153391693878175, + 2.0160440145874023, + 2.0161946767425536, + 2.014449896469116, + 2.015591969642639, + 2.01485006980896, + 2.01346685836792, + 2.0139751796722414, + 2.0144011250305174, + 2.014900064163208, + 2.016459233779907 + ], + "train_acc": [ + 0.23802, + 0.24392, + 0.24906, + 0.25122, + 0.25396, + 0.25104, + 0.2552, + 0.25244, + 0.2577, + 0.25632, + 0.25834, + 0.2557, + 0.25944, + 0.25876, + 0.26044, + 0.25964, + 0.2593, + 0.26036, + 0.26106, + 0.26136, + 0.26018, + 0.25898, + 0.25858, + 0.2613, + 0.26334, + 0.26364, + 0.26186, + 0.26224, + 0.26166, + 0.26282, + 0.26364, + 0.26358, + 0.2645, + 0.26294, + 0.26592, + 0.26282, + 0.26398, + 0.26414, + 0.26538, + 0.26506, + 0.26578, + 0.26148, + 0.26312, + 0.26586, + 0.26478, + 0.26574, + 0.26416, + 0.26488, + 0.26366, + 0.26614, + 0.2656, + 0.26508, + 0.2675, + 0.26694, + 0.26532, + 0.26614, + 0.26596, + 0.26538, + 0.2652, + 0.26606, + 0.26698, + 0.26806, + 0.26674, + 0.26714, + 0.26704, + 0.26768, + 0.26714, + 0.2651, + 0.26544, + 0.26656, + 0.2676, + 0.26678, + 0.26898, + 0.26788, + 0.26736, + 0.2677, + 0.26732, + 0.26766, + 0.26812, + 0.2675, + 0.26916, + 0.26876, + 0.26754, + 0.26748, + 0.26802, + 0.27004, + 0.27052, + 0.2671, + 0.27224, + 0.26654, + 0.26818, + 0.26902, + 0.2691, + 0.26962, + 0.2702, + 0.26948, + 0.2712, + 0.27026, + 0.26876, + 0.26952 + ], + "test_acc": [ + 0.2537, + 0.2494, + 0.2765, + 0.2473, + 0.2851, + 0.2759, + 0.2702, + 0.2833, + 0.2841, + 0.2663, + 0.2858, + 0.2432, + 0.2894, + 0.2933, + 0.2938, + 0.2901, + 0.2777, + 0.2769, + 0.2907, + 0.2833, + 0.2973, + 0.2692, + 0.277, + 0.2841, + 0.2917, + 0.2694, + 0.2929, + 0.2737, + 0.2955, + 0.2877, + 0.2891, + 0.302, + 0.272, + 0.2931, + 0.2882, + 0.2806, + 0.2849, + 0.2989, + 0.2988, + 0.3036, + 0.2869, + 0.2951, + 0.3014, + 0.289, + 0.2834, + 0.2896, + 0.2887, + 0.2932, + 0.2855, + 0.2882, + 0.2846, + 0.3018, + 0.2981, + 0.3019, + 0.2846, + 0.2968, + 0.2915, + 0.2935, + 0.2912, + 0.2978, + 0.2989, + 0.2953, + 0.2999, + 0.292, + 0.2923, + 0.2856, + 0.2956, + 0.2993, + 0.2906, + 0.2931, + 0.2925, + 0.2948, + 0.2968, + 0.295, + 0.2965, + 0.3019, + 0.2889, + 0.2941, + 0.2958, + 0.2959, + 0.2955, + 0.2926, + 0.2921, + 0.2964, + 0.2981, + 0.2912, + 0.2982, + 0.2995, + 0.2958, + 0.2963, + 0.2965, + 0.2963, + 0.2947, + 0.2957, + 0.2953, + 0.2947, + 0.2953, + 0.2951, + 0.2949, + 0.295 + ] + }, + "diagnostics": { + "bp_cosine": [ + 0.3428037762641907, + -3.763916902244091e-05, + -0.0007522967061959207, + -0.00019637049990706146, + -0.0005234384443610907, + 0.00033836261718533933 + ], + "perturbation_rho": [ + -0.03871288150548935, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "nudging": { + "0.001": [ + -3.3015385270118713e-07, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "0.003": [ + -8.167698979377747e-07, + 0.0, + 0.0, + 0.0, + 1.862645149230957e-09, + 0.0 + ], + "0.01": [ + -2.7455389499664307e-06, + 0.0, + 0.0, + 1.862645149230957e-09, + 1.862645149230957e-09, + 0.0 + ] + }, + "hidden_norms_per_layer": [ + 53848.7734375, + 2442806016.0, + 5245538816.0, + 6828209664.0, + 7018600960.0, + 9658986496.0, + 9768150016.0 + ], + "bp_grad_norms_per_layer": [ + 2.0601439132406085e-07, + 2.350288574870518e-10, + 2.3180254937749112e-10, + 2.322748937633179e-10, + 2.3222145040247e-10, + 2.3218917066802902e-10, + 2.321651759729093e-10 + ] + }, + "drift": { + "embed.weight": 347.726745764923, + "embed.bias": 320.686611820448, + "blocks.0.ln.weight": 9.997068770929598, + "blocks.0.w1.weight": 332.0183119830964, + "blocks.0.w1.bias": 361.6276454599223, + "blocks.0.w2.weight": 491.26444464340926, + "blocks.1.ln.weight": 9.760206175642947, + "blocks.1.w1.weight": 399.42793179083463, + "blocks.1.w1.bias": 382.1218468905457, + "blocks.1.w2.weight": 398.4896403314863, + "blocks.2.ln.weight": 9.863790992763134, + "blocks.2.w1.weight": 403.4218387582533, + "blocks.2.w1.bias": 369.67326277166103, + "blocks.2.w2.weight": 388.1030551430935, + "blocks.3.ln.weight": 7.7321395772148405, + "blocks.3.w1.weight": 284.159080804305, + "blocks.3.w1.bias": 261.08860693920894, + "blocks.3.w2.weight": 276.021700238577, + "blocks.4.ln.weight": 10.766653379169998, + "blocks.4.w1.weight": 441.4219333141434, + "blocks.4.w1.bias": 408.4716885612093, + "blocks.4.w2.weight": 432.48056855561003, + "blocks.5.ln.weight": 7.252784916283292, + "blocks.5.w1.weight": 280.38847615534195, + "blocks.5.w1.bias": 255.99327290804064, + "blocks.5.w2.weight": 255.63025547094998, + "out_ln.weight": 0.644730004333539, + "out_head.weight": 9.313913439210426, + "out_head.bias": 0.978185244031994 + } + }, + "fa": { + "log": { + "train_loss": [ + 2.0382984645843507, + 1.9496157778167724, + 1.9119072325897217, + 1.8929361431121827, + 1.8775380432128905, + 1.8741492847442627, + 1.8671779189300537, + 1.860015816307068, + 1.8485039702987671, + 1.844084996986389, + 1.8363114721679688, + 1.8330332998275758, + 1.8273526552581787, + 1.825867492904663, + 1.821722242088318, + 1.8222067623519898, + 1.8212668908691407, + 1.8164310580825807, + 1.8167414797592163, + 1.8158153560638428, + 1.8170397219848633, + 1.8139575901031495, + 1.814438596458435, + 1.807250874671936, + 1.806480390663147, + 1.805527213973999, + 1.7998141473770142, + 1.8049808203125, + 1.8055741637420655, + 1.796914314918518, + 1.8009305011749268, + 1.797931548538208, + 1.7981645835113524, + 1.793643822402954, + 1.7937120150375365, + 1.7967921892929077, + 1.7892387997436523, + 1.785188671875, + 1.786605785293579, + 1.787917324485779, + 1.7853582260894776, + 1.7880721422576904, + 1.7864411516571046, + 1.7777207873916625, + 1.7769269942855834, + 1.777174091796875, + 1.7795673489761352, + 1.773414126586914, + 1.7774351581192016, + 1.7697969200897217, + 1.771698450050354, + 1.768428716278076, + 1.7676336987304688, + 1.765194714126587, + 1.7675122838974, + 1.7682551845932006, + 1.7667323706436158, + 1.7661600426864623, + 1.7620434980010986, + 1.7610514661026, + 1.7625030724716186, + 1.7606126858520508, + 1.760137271118164, + 1.7585180844879151, + 1.7585363976287842, + 1.7592174551010131, + 1.755930234375, + 1.7548912310791016, + 1.7563516518783568, + 1.7501784167099, + 1.7541064535522461, + 1.7512517785263062, + 1.7541993662261963, + 1.7516761654663087, + 1.7516398949813843, + 1.749719626121521, + 1.7497738579559325, + 1.752181930885315, + 1.7519074490356445, + 1.7518238845443725, + 1.7466175811004638, + 1.748845227355957, + 1.746575064430237, + 1.7496823914337158, + 1.7468877982330322, + 1.7439219277191162, + 1.7428745371246337, + 1.746016036453247, + 1.742904917678833, + 1.7454003299713134, + 1.7449493420028686, + 1.744745844039917, + 1.7438948041152955, + 1.7453510579681397, + 1.7438724200439453, + 1.7425698751068115, + 1.742772998123169, + 1.7421956006622314, + 1.7445682699966432, + 1.7442712462615966 + ], + "train_acc": [ + 0.25024, + 0.28724, + 0.30528, + 0.31264, + 0.32216, + 0.32208, + 0.32792, + 0.33064, + 0.3365, + 0.3396, + 0.34, + 0.34242, + 0.34468, + 0.34422, + 0.34752, + 0.34912, + 0.34524, + 0.34978, + 0.34856, + 0.3501, + 0.34806, + 0.35048, + 0.35214, + 0.35416, + 0.35338, + 0.35482, + 0.35432, + 0.35244, + 0.35388, + 0.35836, + 0.35712, + 0.3568, + 0.3591, + 0.36118, + 0.36208, + 0.35824, + 0.36296, + 0.3625, + 0.3616, + 0.3628, + 0.36246, + 0.36298, + 0.3605, + 0.36632, + 0.36574, + 0.36678, + 0.36478, + 0.3649, + 0.3662, + 0.36734, + 0.3665, + 0.36722, + 0.36714, + 0.3702, + 0.36908, + 0.37058, + 0.36998, + 0.36858, + 0.37244, + 0.37104, + 0.3722, + 0.37344, + 0.3726, + 0.37172, + 0.3741, + 0.37406, + 0.3731, + 0.37294, + 0.37284, + 0.37628, + 0.37682, + 0.37742, + 0.37544, + 0.37656, + 0.37632, + 0.37814, + 0.37808, + 0.37642, + 0.37748, + 0.37462, + 0.37968, + 0.37766, + 0.37704, + 0.37846, + 0.37784, + 0.379, + 0.37904, + 0.37854, + 0.38202, + 0.37828, + 0.37712, + 0.37962, + 0.38004, + 0.37898, + 0.3783, + 0.37898, + 0.38082, + 0.38006, + 0.37922, + 0.37846 + ], + "test_acc": [ + 0.3027, + 0.306, + 0.3365, + 0.3382, + 0.3465, + 0.3597, + 0.3579, + 0.3625, + 0.3757, + 0.3702, + 0.3663, + 0.3612, + 0.374, + 0.378, + 0.3778, + 0.3778, + 0.3807, + 0.3691, + 0.3833, + 0.3714, + 0.3788, + 0.3629, + 0.3794, + 0.382, + 0.3843, + 0.3732, + 0.3818, + 0.373, + 0.3819, + 0.3776, + 0.3882, + 0.3926, + 0.385, + 0.3853, + 0.3879, + 0.3927, + 0.3894, + 0.3945, + 0.3903, + 0.3924, + 0.3896, + 0.3941, + 0.3919, + 0.3888, + 0.3863, + 0.3927, + 0.3922, + 0.3903, + 0.3939, + 0.3828, + 0.3963, + 0.3935, + 0.3958, + 0.3971, + 0.4048, + 0.4003, + 0.3934, + 0.3945, + 0.395, + 0.3979, + 0.3908, + 0.3969, + 0.4012, + 0.3998, + 0.3987, + 0.3965, + 0.4025, + 0.3984, + 0.4038, + 0.3982, + 0.4015, + 0.4027, + 0.3997, + 0.4022, + 0.4045, + 0.4031, + 0.4028, + 0.4041, + 0.3998, + 0.4012, + 0.3998, + 0.3979, + 0.4026, + 0.401, + 0.4006, + 0.4048, + 0.4018, + 0.4009, + 0.4025, + 0.4024, + 0.4009, + 0.4036, + 0.4025, + 0.4022, + 0.4026, + 0.401, + 0.4018, + 0.401, + 0.4015, + 0.4016 + ] + }, + "diagnostics": { + "bp_cosine": [ + 0.02052094228565693, + 0.07671980559825897, + -0.10412630438804626, + -0.03588568791747093, + -0.07592153549194336, + 0.9841817617416382 + ], + "perturbation_rho": [ + -0.0499441958963871, + 0.03510870411992073, + 0.012152664363384247, + -0.01593763567507267, + -0.030490349978208542, + 0.01056537963449955 + ], + "nudging": { + "0.001": [ + -1.460895873606205e-06, + -1.8533319234848022e-07, + 8.102506399154663e-08, + 3.9814040064811707e-08, + 7.660128176212311e-08, + -1.0418007150292397e-06 + ], + "0.003": [ + -4.51505184173584e-06, + -7.73230567574501e-07, + 4.012836143374443e-07, + 1.9150320440530777e-07, + 2.3317988961935043e-07, + -3.8853613659739494e-06 + ], + "0.01": [ + -1.504761166870594e-05, + -2.689310349524021e-06, + 1.5775440260767937e-06, + 6.139744073152542e-07, + 9.952345862984657e-07, + -1.3828510418534279e-05 + ] + }, + "hidden_norms_per_layer": [ + 7937.806640625, + 157942.125, + 845318.25, + 1734418.625, + 1908211.875, + 2166763.75, + 1424229.125 + ], + "bp_grad_norms_per_layer": [ + 2.3930177121656016e-05, + 1.1552400565051357e-06, + 6.149262503640784e-07, + 6.002979375807627e-07, + 6.000758503432735e-07, + 5.996230356686283e-07, + 5.751607545789739e-07 + ] + }, + "drift": { + "embed.weight": 54.05447299340841, + "embed.bias": 16.031038115248215, + "blocks.0.ln.weight": 1.2416108593211508, + "blocks.0.w1.weight": 17.922589218376636, + "blocks.0.w1.bias": 14.956067338203475, + "blocks.0.w2.weight": 64.67351730215137, + "blocks.1.ln.weight": 1.1286859632363149, + "blocks.1.w1.weight": 23.71804846947617, + "blocks.1.w1.bias": 17.43068124645083, + "blocks.1.w2.weight": 41.38902946126262, + "blocks.2.ln.weight": 0.7406672335136909, + "blocks.2.w1.weight": 24.88918465627309, + "blocks.2.w1.bias": 25.668393301356875, + "blocks.2.w2.weight": 26.33124026065157, + "blocks.3.ln.weight": 0.5408212886770045, + "blocks.3.w1.weight": 19.746562187924944, + "blocks.3.w1.bias": 21.605426127504696, + "blocks.3.w2.weight": 26.408093498021923, + "blocks.4.ln.weight": 0.7373842468395975, + "blocks.4.w1.weight": 21.153183487980545, + "blocks.4.w1.bias": 21.145538252519305, + "blocks.4.w2.weight": 45.59224517841596, + "blocks.5.ln.weight": 0.8316077429831186, + "blocks.5.w1.weight": 20.11472175424346, + "blocks.5.w1.bias": 17.387776840886495, + "blocks.5.w2.weight": 81.52952222436008, + "out_ln.weight": 0.38061390763416836, + "out_head.weight": 7.016437743402715, + "out_head.bias": 0.6179754221306049 + } + } + }, + "config": { + "dataset": "cifar10", + "d_hidden": 512, + "num_blocks": 6, + "batch_size": 128, + "epochs": 100, + "lr": 0.001, + "lr_fb": 0.001, + "wd": 0.01, + "lam": 0.1, + "K": 4, + "sigma_bridge": 0.05, + "ema_momentum": 0.995, + "term_grad_weight": 1.0, + "seeds": [ + 2 + ], + "gpu": 0, + "output_dir": "results/fa_dfa_d512_L6_seed2", + "methods": [ + "fa", + "dfa" + ], + "random_targets": false, + "penalty_lam": 0.0, + "num_classes": 10 + } +}
\ No newline at end of file |