diff options
Diffstat (limited to 'results/fa_dfa_d512_L6_seed4/results_cifar10.json')
| -rw-r--r-- | results/fa_dfa_d512_L6_seed4/results_cifar10.json | 837 |
1 files changed, 837 insertions, 0 deletions
diff --git a/results/fa_dfa_d512_L6_seed4/results_cifar10.json b/results/fa_dfa_d512_L6_seed4/results_cifar10.json new file mode 100644 index 0000000..0f4b59f --- /dev/null +++ b/results/fa_dfa_d512_L6_seed4/results_cifar10.json @@ -0,0 +1,837 @@ +{ + "4": { + "dfa": { + "log": { + "train_loss": [ + 2.0741304068756103, + 2.052452068710327, + 2.0495207961273194, + 2.0484457649230956, + 2.045468647079468, + 2.0459753023529053, + 2.0450184237670896, + 2.0391295627593995, + 2.0377573050689697, + 2.0382228170394896, + 2.0312761193466184, + 2.0324994396591185, + 2.0278026767730712, + 2.0305476247787477, + 2.0286661254119873, + 2.029820890045166, + 2.0290051824188233, + 2.0265300660705567, + 2.025750561065674, + 2.027355994720459, + 2.0275561265563966, + 2.023750612411499, + 2.025079263343811, + 2.025395411605835, + 2.0240028636932372, + 2.022815622253418, + 2.0226645655822755, + 2.022443874359131, + 2.022390442466736, + 2.023733573875427, + 2.0224531535339354, + 2.0227799210357666, + 2.0218398332214353, + 2.020151731872559, + 2.021256269683838, + 2.0194612783432007, + 2.018333575744629, + 2.0183729624938964, + 2.0209142852020263, + 2.022426343536377, + 2.018248730201721, + 2.0204831772613527, + 2.0188777527618407, + 2.0191313500213623, + 2.0171565660858155, + 2.0189703364562988, + 2.0182820797729493, + 2.0170327036285403, + 2.0180798105239868, + 2.0163722726821898, + 2.0178759913635256, + 2.017920862388611, + 2.015365661468506, + 2.0167020557403563, + 2.0157576428985595, + 2.0196311264801023, + 2.0176754986190795, + 2.017508302345276, + 2.0164641773223875, + 2.016444677734375, + 2.0168316576385497, + 2.0152849128723145, + 2.015149051208496, + 2.0158757721710203, + 2.016044344520569, + 2.0154035729217528, + 2.012803511734009, + 2.0160995071792605, + 2.015346680755615, + 2.0140808686828615, + 2.0151058874893186, + 2.0144621922302246, + 2.0154705238342285, + 2.0149063832092287, + 2.0115139068603516, + 2.0131744523620605, + 2.0158527814483644, + 2.0125201052474977, + 2.0122518685913087, + 2.0132210369873045, + 2.0136120219421385, + 2.0142880089950563, + 2.013032984085083, + 2.011840599975586, + 2.0118305902862548, + 2.012003014450073, + 2.011992978248596, + 2.0092738487243653, + 2.0135212693023683, + 2.010035700378418, + 2.011592872314453, + 2.010264990081787, + 2.0128248121643066, + 2.0130243259048464, + 2.0120003854751585, + 2.011955912322998, + 2.011813560256958, + 2.0142602909851073, + 2.0124067852020264, + 2.0116278426742555 + ], + "train_acc": [ + 0.23226, + 0.2375, + 0.23894, + 0.2411, + 0.24176, + 0.24224, + 0.24692, + 0.24654, + 0.24822, + 0.24668, + 0.25084, + 0.24936, + 0.2519, + 0.24896, + 0.25208, + 0.25194, + 0.2549, + 0.25158, + 0.25292, + 0.25402, + 0.25358, + 0.2561, + 0.25688, + 0.25572, + 0.25782, + 0.25752, + 0.2589, + 0.25694, + 0.2558, + 0.25686, + 0.25668, + 0.259, + 0.2579, + 0.25902, + 0.25906, + 0.25856, + 0.26018, + 0.2611, + 0.25834, + 0.26182, + 0.26058, + 0.25944, + 0.26134, + 0.25952, + 0.26058, + 0.2594, + 0.26156, + 0.26526, + 0.26156, + 0.2652, + 0.26178, + 0.26122, + 0.2618, + 0.26384, + 0.26214, + 0.26386, + 0.26256, + 0.2644, + 0.26262, + 0.2629, + 0.26074, + 0.26266, + 0.2634, + 0.26182, + 0.26266, + 0.26396, + 0.26654, + 0.2633, + 0.2613, + 0.26562, + 0.26356, + 0.26384, + 0.26406, + 0.26388, + 0.26606, + 0.26592, + 0.265, + 0.26578, + 0.268, + 0.2666, + 0.26434, + 0.2666, + 0.26418, + 0.26448, + 0.26558, + 0.2644, + 0.26708, + 0.26714, + 0.2644, + 0.2675, + 0.26548, + 0.26528, + 0.26508, + 0.26738, + 0.26266, + 0.2658, + 0.26572, + 0.26576, + 0.26514, + 0.2658 + ], + "test_acc": [ + 0.2469, + 0.2437, + 0.255, + 0.2587, + 0.2612, + 0.2707, + 0.2697, + 0.2504, + 0.279, + 0.2707, + 0.2664, + 0.2693, + 0.2715, + 0.2832, + 0.2849, + 0.2633, + 0.2682, + 0.2874, + 0.2852, + 0.2671, + 0.273, + 0.2819, + 0.2654, + 0.2763, + 0.279, + 0.2924, + 0.281, + 0.2755, + 0.2706, + 0.2758, + 0.2812, + 0.2801, + 0.2847, + 0.2684, + 0.2679, + 0.2819, + 0.2548, + 0.2726, + 0.2794, + 0.2838, + 0.2834, + 0.2788, + 0.283, + 0.2721, + 0.2782, + 0.2742, + 0.2817, + 0.2752, + 0.2722, + 0.2812, + 0.2797, + 0.2792, + 0.2799, + 0.2747, + 0.2879, + 0.2769, + 0.2842, + 0.2793, + 0.2857, + 0.2797, + 0.2793, + 0.283, + 0.2889, + 0.2789, + 0.2748, + 0.2811, + 0.2814, + 0.2775, + 0.282, + 0.2787, + 0.2862, + 0.2861, + 0.2788, + 0.2763, + 0.272, + 0.2788, + 0.2818, + 0.2839, + 0.2865, + 0.2837, + 0.2898, + 0.2814, + 0.2784, + 0.2823, + 0.2852, + 0.2835, + 0.2848, + 0.2846, + 0.2821, + 0.2809, + 0.2812, + 0.2826, + 0.2818, + 0.2832, + 0.2834, + 0.2832, + 0.2838, + 0.2838, + 0.2838, + 0.2837 + ] + }, + "diagnostics": { + "bp_cosine": [ + 0.337674617767334, + 0.0006247189012356102, + 0.0002945333835668862, + -7.626566366525367e-05, + -0.00035525468410924077, + 4.714205351774581e-05 + ], + "perturbation_rho": [ + 0.005920294672250748, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "nudging": { + "0.001": [ + -3.096647560596466e-07, + 0.0, + 0.0, + 0.0, + 9.313225746154785e-10, + 0.0 + ], + "0.003": [ + -9.313225746154785e-07, + 9.313225746154785e-10, + 0.0, + 0.0, + 9.313225746154785e-10, + 0.0 + ], + "0.01": [ + -2.87545844912529e-06, + 0.0, + 0.0, + 9.313225746154785e-10, + 9.313225746154785e-10, + 0.0 + ] + }, + "hidden_norms_per_layer": [ + 58616.81640625, + 1590750592.0, + 3689531648.0, + 5549292544.0, + 7886680064.0, + 9552689152.0, + 9643235328.0 + ], + "bp_grad_norms_per_layer": [ + 1.978580996819801e-07, + 1.773336893995392e-10, + 1.7791043638304416e-10, + 1.7622008019468893e-10, + 1.761738255279255e-10, + 1.7640035265831244e-10, + 1.7640640337379665e-10 + ] + }, + "drift": { + "embed.weight": 352.51471697650226, + "embed.bias": 269.49001616797665, + "blocks.0.ln.weight": 9.43372656624646, + "blocks.0.w1.weight": 313.54087904771035, + "blocks.0.w1.bias": 290.62855542890503, + "blocks.0.w2.weight": 505.8716937587808, + "blocks.1.ln.weight": 8.808169542950294, + "blocks.1.w1.weight": 358.5357409826361, + "blocks.1.w1.bias": 345.5274778587847, + "blocks.1.w2.weight": 349.75977929168596, + "blocks.2.ln.weight": 9.276124324101515, + "blocks.2.w1.weight": 395.6550378108265, + "blocks.2.w1.bias": 371.0546436106137, + "blocks.2.w2.weight": 375.3367421271525, + "blocks.3.ln.weight": 10.24244368106198, + "blocks.3.w1.weight": 420.44883795775684, + "blocks.3.w1.bias": 397.1721710589656, + "blocks.3.w2.weight": 405.8311891944568, + "blocks.4.ln.weight": 10.577140914534366, + "blocks.4.w1.weight": 436.864273411592, + "blocks.4.w1.bias": 406.9723932351042, + "blocks.4.w2.weight": 404.6446833274338, + "blocks.5.ln.weight": 7.782034028860308, + "blocks.5.w1.weight": 310.2471895048895, + "blocks.5.w1.bias": 297.2855707982679, + "blocks.5.w2.weight": 270.3724536312007, + "out_ln.weight": 0.5880164032165464, + "out_head.weight": 8.450568347443314, + "out_head.bias": 0.7740018910126388 + } + }, + "fa": { + "log": { + "train_loss": [ + 2.038021390991211, + 1.9517871558380127, + 1.923053580093384, + 1.9045403602600097, + 1.8952905172729493, + 1.8863004998397828, + 1.8793055545043946, + 1.868332847251892, + 1.860797550354004, + 1.8578368386077881, + 1.8483906720352172, + 1.8424116570663451, + 1.8290137046051025, + 1.8305858782577515, + 1.823958169631958, + 1.8175954293823242, + 1.8124929406356811, + 1.8102095357513428, + 1.8039513228607178, + 1.8015951740264893, + 1.7953115439605714, + 1.7889242324447632, + 1.7881110052871705, + 1.7843929150390625, + 1.7783852965545655, + 1.777275090942383, + 1.7724220377349853, + 1.7673138269424438, + 1.7659701934814453, + 1.7594620475006104, + 1.753105486755371, + 1.7537677935028075, + 1.7502301782989502, + 1.743755245666504, + 1.7426675580596924, + 1.7381902320098876, + 1.7371277405548096, + 1.7380020862197876, + 1.7344951665878297, + 1.732631077194214, + 1.7297903375244141, + 1.7305091592025756, + 1.72773092628479, + 1.7255595262908936, + 1.7265567044067383, + 1.7226529630279541, + 1.7242008368301391, + 1.7205993241119384, + 1.714460986251831, + 1.7183882989501953, + 1.7157616848373414, + 1.7143050561141968, + 1.711619351425171, + 1.712949155960083, + 1.7116730854797364, + 1.7097789264297485, + 1.7088176770401, + 1.7031783606338502, + 1.705644631690979, + 1.702429769935608, + 1.7007943072891236, + 1.6994391622924805, + 1.698121346435547, + 1.6975506435775758, + 1.6984819930648805, + 1.6961595495224, + 1.6988952493286134, + 1.700139475479126, + 1.698183412246704, + 1.6925607485580445, + 1.6942372484970092, + 1.695046120262146, + 1.6923927303695678, + 1.6946959600448608, + 1.6924981302261353, + 1.688026438217163, + 1.6864081888580322, + 1.6876143859863282, + 1.687906523399353, + 1.6851787896347046, + 1.6870249395370482, + 1.6865479514312745, + 1.6857007721710204, + 1.686013254776001, + 1.685618132247925, + 1.6866127777099609, + 1.682880577697754, + 1.6817467867660523, + 1.6864028040695191, + 1.683715857887268, + 1.6836617621612548, + 1.6802171184921264, + 1.6822573038101196, + 1.6837931204223633, + 1.6831412961196899, + 1.6796661951065064, + 1.682324230041504, + 1.6844535943222045, + 1.6789856439590454, + 1.680855456161499 + ], + "train_acc": [ + 0.2477, + 0.28914, + 0.30422, + 0.30836, + 0.31314, + 0.31906, + 0.3247, + 0.32558, + 0.33112, + 0.33134, + 0.33596, + 0.33794, + 0.34382, + 0.34438, + 0.3429, + 0.3491, + 0.34826, + 0.351, + 0.35252, + 0.3536, + 0.35608, + 0.36104, + 0.35948, + 0.36006, + 0.36364, + 0.36444, + 0.36608, + 0.367, + 0.36846, + 0.36866, + 0.37048, + 0.37416, + 0.37428, + 0.37498, + 0.37714, + 0.37884, + 0.37872, + 0.3782, + 0.381, + 0.38058, + 0.3804, + 0.38016, + 0.38418, + 0.38552, + 0.38428, + 0.3811, + 0.38204, + 0.38364, + 0.38758, + 0.38694, + 0.38588, + 0.38568, + 0.388, + 0.38704, + 0.39158, + 0.38884, + 0.38912, + 0.38936, + 0.39034, + 0.39182, + 0.39316, + 0.3944, + 0.39182, + 0.39272, + 0.39394, + 0.39482, + 0.39348, + 0.39242, + 0.39554, + 0.3963, + 0.39508, + 0.39252, + 0.39634, + 0.39332, + 0.39612, + 0.39644, + 0.39648, + 0.39668, + 0.3985, + 0.3995, + 0.39528, + 0.39904, + 0.39988, + 0.39688, + 0.39954, + 0.39648, + 0.4014, + 0.39972, + 0.39808, + 0.4019, + 0.39968, + 0.40222, + 0.39884, + 0.39954, + 0.39804, + 0.40034, + 0.39982, + 0.39856, + 0.40168, + 0.4006 + ], + "test_acc": [ + 0.2942, + 0.311, + 0.3308, + 0.342, + 0.3419, + 0.3528, + 0.3504, + 0.3462, + 0.3605, + 0.367, + 0.3696, + 0.3674, + 0.3716, + 0.3801, + 0.3769, + 0.3624, + 0.3792, + 0.3785, + 0.3758, + 0.3779, + 0.3853, + 0.3821, + 0.3816, + 0.3891, + 0.3866, + 0.3936, + 0.3896, + 0.3908, + 0.397, + 0.3971, + 0.4013, + 0.4023, + 0.393, + 0.3983, + 0.3977, + 0.3969, + 0.3982, + 0.405, + 0.4014, + 0.4089, + 0.4078, + 0.3985, + 0.4063, + 0.4039, + 0.4008, + 0.4096, + 0.4111, + 0.4059, + 0.4128, + 0.4079, + 0.4066, + 0.4115, + 0.407, + 0.4156, + 0.4156, + 0.4208, + 0.4164, + 0.4206, + 0.4124, + 0.4186, + 0.4142, + 0.4123, + 0.4105, + 0.4153, + 0.4198, + 0.4192, + 0.419, + 0.4198, + 0.4202, + 0.417, + 0.4233, + 0.4224, + 0.4164, + 0.4183, + 0.4193, + 0.4178, + 0.4189, + 0.4232, + 0.4184, + 0.4206, + 0.4223, + 0.4222, + 0.417, + 0.4199, + 0.421, + 0.4202, + 0.4181, + 0.4177, + 0.4214, + 0.4186, + 0.4201, + 0.4189, + 0.4221, + 0.4205, + 0.4194, + 0.42, + 0.4203, + 0.4199, + 0.4196, + 0.4195 + ] + }, + "diagnostics": { + "bp_cosine": [ + 0.015272621065378189, + 0.05551045760512352, + 0.03781959414482117, + -0.09142173081636429, + -0.12766994535923004, + 0.9957026839256287 + ], + "perturbation_rho": [ + -0.044239602982997894, + 0.018639111891388893, + -0.002674652263522148, + -0.024060344323515892, + -0.007919290103018284, + -0.016996072605252266 + ], + "nudging": { + "0.001": [ + -2.239597961306572e-06, + -6.057089194655418e-07, + -1.341104507446289e-07, + 1.417938619852066e-07, + 1.6426201909780502e-07, + -1.8667196854948997e-06 + ], + "0.003": [ + -6.492482498288155e-06, + -1.5690457075834274e-06, + -4.0302984416484833e-07, + 5.105976015329361e-07, + 7.660128176212311e-07, + -6.384681910276413e-06 + ], + "0.01": [ + -2.1643005311489105e-05, + -5.265465006232262e-06, + -1.4334218576550484e-06, + 2.071727067232132e-06, + 2.557528205215931e-06, + -2.2026244550943375e-05 + ] + }, + "hidden_norms_per_layer": [ + 5698.91357421875, + 74089.609375, + 464746.75, + 571451.9375, + 1133723.625, + 1542811.875, + 774708.375 + ], + "bp_grad_norms_per_layer": [ + 3.8776230212533846e-05, + 3.033989969480899e-06, + 9.73796886682976e-07, + 8.570191312173847e-07, + 8.443464594165562e-07, + 8.415062779931759e-07, + 8.336798487107444e-07 + ] + }, + "drift": { + "embed.weight": 39.486431642766554, + "embed.bias": 14.997056918535808, + "blocks.0.ln.weight": 1.193207647495217, + "blocks.0.w1.weight": 15.819034065039398, + "blocks.0.w1.bias": 13.818478530432463, + "blocks.0.w2.weight": 51.89139321984625, + "blocks.1.ln.weight": 1.0430825935788601, + "blocks.1.w1.weight": 19.906462986471904, + "blocks.1.w1.bias": 13.39027258705086, + "blocks.1.w2.weight": 46.62053425901664, + "blocks.2.ln.weight": 0.7864666972877575, + "blocks.2.w1.weight": 18.270196046735624, + "blocks.2.w1.bias": 12.944067249835973, + "blocks.2.w2.weight": 49.53460850460699, + "blocks.3.ln.weight": 0.64141877927046, + "blocks.3.w1.weight": 20.764880942811217, + "blocks.3.w1.bias": 20.679665524082093, + "blocks.3.w2.weight": 25.665386885036355, + "blocks.4.ln.weight": 0.5680829244316266, + "blocks.4.w1.weight": 20.078876458586997, + "blocks.4.w1.bias": 21.20100063860491, + "blocks.4.w2.weight": 30.60469406190784, + "blocks.5.ln.weight": 0.6546824348706847, + "blocks.5.w1.weight": 18.8873912461224, + "blocks.5.w1.bias": 18.79210459184489, + "blocks.5.w2.weight": 45.45712755218628, + "out_ln.weight": 0.3591377698736513, + "out_head.weight": 6.323626106875271, + "out_head.bias": 0.8856670357666573 + } + } + }, + "config": { + "dataset": "cifar10", + "d_hidden": 512, + "num_blocks": 6, + "batch_size": 128, + "epochs": 100, + "lr": 0.001, + "lr_fb": 0.001, + "wd": 0.01, + "lam": 0.1, + "K": 4, + "sigma_bridge": 0.05, + "ema_momentum": 0.995, + "term_grad_weight": 1.0, + "seeds": [ + 4 + ], + "gpu": 0, + "output_dir": "results/fa_dfa_d512_L6_seed4", + "methods": [ + "fa", + "dfa" + ], + "random_targets": false, + "penalty_lam": 0.0, + "num_classes": 10 + } +}
\ No newline at end of file |