{ "42": { "bp": { "log": { "train_loss": [ 3.975249941253662, 3.6423986249542235, 3.4934494227600097, 3.3944045304870607, 3.3128096391296387, 3.249527327423096, 3.202136681060791, 3.1545809954071045, 3.118299830780029, 3.08477955947876, 3.05101420211792, 3.014707311477661, 2.9815688012695314, 2.9542644719696045, 2.917737830657959, 2.8987481777954103, 2.877259644317627, 2.8476891691589357, 2.826910527801514, 2.8041292054748537, 2.7817903074645995, 2.7624645190429686, 2.736394059448242, 2.7153989936065672, 2.6938070545959474, 2.6722896592712404, 2.656808620452881, 2.6342206648254396, 2.6130940935516356, 2.601909016876221, 2.5847591175842286, 2.5616293630218507, 2.540946982421875, 2.528531006851196, 2.508262563323975, 2.494774736251831, 2.471127844619751, 2.455823439178467, 2.443439641647339, 2.419278503417969, 2.409900106124878, 2.3866433473205566, 2.3731185454559327, 2.348314314727783, 2.336296524658203, 2.3228698544311523, 2.299138896865845, 2.2919679093933105, 2.271020979156494, 2.263141948699951, 2.242116179199219, 2.2228171809387205, 2.210650175628662, 2.1947706311798094, 2.1783614316558837, 2.1615723634338377, 2.152301993675232, 2.139739892730713, 2.11690347366333, 2.1134400886535643, 2.0890519944000245, 2.076347621765137, 2.067469055480957, 2.0518165933990478, 2.041287666015625, 2.027727124786377, 2.0111099968719484, 1.9989793057250977, 1.991382496948242, 1.9685791171264648, 1.966265661239624, 1.9562527508544922, 1.942470475769043, 1.9250774355316163, 1.9187585319137572, 1.9093894129943847, 1.9017769010162353, 1.8909222549438476, 1.8789906490707398, 1.87238977394104, 1.876755567703247, 1.8579993398284913, 1.8549057474517823, 1.8437388555145264, 1.83396729637146, 1.8345417359161378, 1.83224958152771, 1.8185578295516969, 1.8260858072280883, 1.8161109666442872, 1.8139908428573608, 1.8139760601043702, 1.8037351627349854, 1.8002028354644775, 1.8035412215423583, 1.8000410940933227, 1.7957222201156615, 1.79567547290802, 1.7953530138397218, 1.7897127409744262 ], "train_acc": [ 0.09828, 0.14442, 0.16788, 0.18412, 0.1995, 0.21148, 0.21652, 0.2278, 0.23422, 0.24242, 0.24772, 0.25402, 0.26126, 0.2639, 0.27258, 0.27642, 0.2808, 0.28564, 0.2893, 0.29352, 0.29922, 0.3014, 0.30794, 0.31222, 0.31722, 0.322, 0.32392, 0.32726, 0.33092, 0.3352, 0.33782, 0.34382, 0.34482, 0.34966, 0.3531, 0.35308, 0.36068, 0.36312, 0.36942, 0.37268, 0.3705, 0.3762, 0.38242, 0.38778, 0.38854, 0.39198, 0.3985, 0.39664, 0.4058, 0.40534, 0.40916, 0.41304, 0.41528, 0.42002, 0.42598, 0.4278, 0.42834, 0.43244, 0.43612, 0.43712, 0.4414, 0.44592, 0.44574, 0.44904, 0.4551, 0.45958, 0.46296, 0.4616, 0.46646, 0.47288, 0.4693, 0.47354, 0.47626, 0.48042, 0.4848, 0.48688, 0.4857, 0.4874, 0.49078, 0.49406, 0.49478, 0.49768, 0.49744, 0.50144, 0.50262, 0.5014, 0.50426, 0.5064, 0.5074, 0.50692, 0.50832, 0.50964, 0.5119, 0.51018, 0.5099, 0.51064, 0.51258, 0.515, 0.51522, 0.5126 ], "test_acc": [ 0.1432, 0.181, 0.1871, 0.212, 0.2185, 0.2122, 0.2368, 0.2466, 0.241, 0.2492, 0.2563, 0.2629, 0.2656, 0.2695, 0.2731, 0.2753, 0.2737, 0.2823, 0.2832, 0.2883, 0.2946, 0.2915, 0.2957, 0.2922, 0.2981, 0.294, 0.2961, 0.301, 0.3015, 0.3078, 0.3127, 0.3097, 0.308, 0.3058, 0.3138, 0.3098, 0.3114, 0.3129, 0.3126, 0.3158, 0.3146, 0.3119, 0.313, 0.3137, 0.3127, 0.3167, 0.3186, 0.3181, 0.3165, 0.316, 0.3222, 0.3188, 0.3228, 0.3173, 0.3178, 0.3174, 0.3217, 0.3173, 0.3196, 0.3223, 0.3181, 0.3186, 0.3196, 0.3161, 0.3202, 0.314, 0.3178, 0.3172, 0.3174, 0.3207, 0.3193, 0.3214, 0.3213, 0.3175, 0.3228, 0.3207, 0.3208, 0.3213, 0.3187, 0.3197, 0.3226, 0.32, 0.3205, 0.3213, 0.3217, 0.3192, 0.3203, 0.3207, 0.3209, 0.3197, 0.3207, 0.3195, 0.3192, 0.3186, 0.3178, 0.3191, 0.3189, 0.3193, 0.3189, 0.3192 ] }, "diagnostics": { "bp_cosine": [ 1.0, 1.0, 1.0, 1.0 ], "perturbation_rho": [ 0.9980340600013733, 0.9982945919036865, 0.9984437227249146, 0.9987074136734009 ], "nudging": { "0.001": [ -0.0022835906129330397, -0.0023387258406728506, -0.002373999450355768, -0.002260879147797823 ], "0.003": [ -0.006846790201961994, -0.007012245710939169, -0.007117925677448511, -0.006779024377465248 ], "0.01": [ -0.022776642814278603, -0.02332628145813942, -0.023679494857788086, -0.02255532145500183 ] }, "hidden_norms_per_layer": [ 236.57855224609375, 217.48927307128906, 197.1617889404297, 187.24839782714844, 191.277587890625 ], "bp_grad_norms_per_layer": [ 0.0010384325869381428, 0.0010872395941987634, 0.0011213896796107292, 0.001103188842535019, 0.0009315301431342959 ] }, "drift": { "embed.weight": 7.433656905137662, "embed.bias": 13.602162006682997, "blocks.0.ln.weight": 0.14788760244846344, "blocks.0.w1.weight": 3.605816897627063, "blocks.0.w1.bias": 5.290238058535016, "blocks.0.w2.weight": 13.310045077395138, "blocks.1.ln.weight": 0.2914122939109802, "blocks.1.w1.weight": 3.997572101972956, "blocks.1.w1.bias": 4.732854205627193, "blocks.1.w2.weight": 14.921397012401421, "blocks.2.ln.weight": 0.35930144786834717, "blocks.2.w1.weight": 4.149029664687284, "blocks.2.w1.bias": 3.8845074849434162, "blocks.2.w2.weight": 16.00171210299565, "blocks.3.ln.weight": 0.5467617511749268, "blocks.3.w1.weight": 4.475958556672767, "blocks.3.w1.bias": 3.7954587976923664, "blocks.3.w2.weight": 17.52879838948756, "out_ln.weight": 0.1443399339914322, "out_head.weight": 2.871805586504566, "out_head.bias": 3.1293217686796893 } }, "dfa": { "log": { "train_loss": [ 4.17356710144043, 4.045213865203857, 4.0394830491638185, 4.052726871032715, 4.062585520324707, 4.077131650848389, 4.081288940353393, 4.082902023620606, 4.082292946929932, 4.084377682189942, 4.082009593811035, 4.078371176147461, 4.0787283967590335, 4.0781014080810545, 4.071053223266602, 4.073739369049072, 4.070565816497803, 4.068224396362305, 4.064294567718506, 4.062663984680176, 4.060789375686645, 4.060499965057373, 4.05942348953247, 4.05340063583374, 4.054450568389893, 4.051036394500732, 4.052292782592773, 4.04923277923584, 4.047891028900146, 4.046583199005127, 4.046011067352295, 4.0444303074646, 4.045907030334472, 4.044839619445801, 4.03859343673706, 4.043664588165283, 4.0400294355773925, 4.035496199493408, 4.038207025909424, 4.035675854187012, 4.037980937347412, 4.038932502746582, 4.036024468383789, 4.037031109313965, 4.034120268630981, 4.036895367584228, 4.037431150054932, 4.033238902282715, 4.036427326049805, 4.03467604888916, 4.03339782157898, 4.0328554144287105, 4.031013775024414, 4.031612381286621, 4.0287201686096195, 4.031132036132813, 4.029230642242432, 4.029863064575196, 4.03147859588623, 4.029825403366089, 4.030359633789063, 4.028107041168213, 4.02976943069458, 4.029035094909668, 4.028509668731689, 4.0304917379760745, 4.030941032562255, 4.026340734710693, 4.027734536590576, 4.024428620147705, 4.025526253967286, 4.025494504394532, 4.026692573547363, 4.0229616456604, 4.026408356018067, 4.024612335662842, 4.023342019500732, 4.024318858337402, 4.023249450683593, 4.023151823806763, 4.026289680709839, 4.023877305831909, 4.023189326248169, 4.022549798278809, 4.024784528045655, 4.022774189758301, 4.024208254547119, 4.023730496368408, 4.02367640625, 4.021903743209839, 4.02390080947876, 4.022859935150146, 4.023393632354736, 4.021766328201294, 4.02325630645752, 4.023518178710938, 4.021166778717041, 4.023976897583008, 4.023609465179443, 4.024444749069214 ], "train_acc": [ 0.06788, 0.08332, 0.08406, 0.0818, 0.08046, 0.08148, 0.07862, 0.07956, 0.08038, 0.07976, 0.08042, 0.0808, 0.08058, 0.08086, 0.08006, 0.08136, 0.08244, 0.08388, 0.08354, 0.084, 0.08448, 0.08518, 0.08594, 0.08624, 0.08568, 0.08764, 0.08492, 0.08604, 0.08814, 0.0876, 0.08836, 0.08956, 0.08702, 0.08854, 0.08976, 0.08954, 0.0895, 0.09006, 0.0894, 0.08964, 0.09064, 0.09034, 0.09288, 0.0913, 0.0914, 0.09116, 0.09094, 0.09192, 0.09044, 0.0909, 0.09252, 0.09214, 0.09332, 0.0931, 0.09364, 0.09176, 0.09246, 0.09302, 0.09276, 0.09334, 0.09404, 0.0942, 0.09392, 0.09358, 0.09336, 0.09392, 0.09252, 0.0949, 0.09444, 0.0958, 0.09542, 0.0949, 0.0954, 0.09438, 0.09338, 0.0958, 0.09376, 0.0959, 0.09448, 0.09388, 0.09458, 0.09422, 0.09542, 0.09522, 0.09468, 0.0951, 0.09564, 0.09586, 0.09536, 0.09624, 0.09588, 0.09594, 0.09642, 0.09588, 0.09496, 0.09496, 0.096, 0.09454, 0.09542, 0.09488 ], "test_acc": [ 0.0775, 0.0794, 0.0808, 0.0828, 0.0742, 0.0795, 0.0788, 0.0855, 0.0809, 0.0819, 0.0767, 0.0826, 0.0854, 0.0752, 0.0798, 0.0796, 0.0813, 0.0771, 0.0823, 0.0759, 0.0814, 0.0772, 0.0826, 0.079, 0.0803, 0.0815, 0.0818, 0.0819, 0.0818, 0.084, 0.0853, 0.0855, 0.0875, 0.0818, 0.0861, 0.0827, 0.0847, 0.0825, 0.0853, 0.0862, 0.0841, 0.0857, 0.0852, 0.0852, 0.0827, 0.0863, 0.0878, 0.0858, 0.0857, 0.0879, 0.0888, 0.088, 0.0878, 0.0871, 0.0878, 0.0845, 0.0863, 0.0852, 0.0872, 0.0879, 0.086, 0.0878, 0.0851, 0.0863, 0.0877, 0.0861, 0.0889, 0.0839, 0.0893, 0.0883, 0.0867, 0.0855, 0.0878, 0.086, 0.0876, 0.0882, 0.0895, 0.0877, 0.0876, 0.0871, 0.088, 0.0876, 0.088, 0.0885, 0.0879, 0.0888, 0.0882, 0.0876, 0.0882, 0.087, 0.0873, 0.0877, 0.0876, 0.0882, 0.0878, 0.0876, 0.0877, 0.0876, 0.0875, 0.0875 ] }, "diagnostics": { "bp_cosine": [ 0.14061881601810455, 0.0022556069307029247, -0.009000124409794807, -0.013949227519333363 ], "perturbation_rho": [ 0.04333949834108353, 0.0, 0.0, 0.0 ], "nudging": { "0.001": [ -5.513429641723633e-07, 0.0, 0.0, -3.725290298461914e-09 ], "0.003": [ -1.7480924725532532e-06, -1.4901161193847656e-08, 0.0, -1.862645149230957e-09 ], "0.01": [ -5.760230123996735e-06, 2.3283064365386963e-08, -1.862645149230957e-09, 0.0 ] }, "hidden_norms_per_layer": [ 20187.3125, 13252251.0, 93144488.0, 163952208.0, 202900624.0 ], "bp_grad_norms_per_layer": [ 1.3867399957234738e-06, 9.786202070927175e-09, 9.394233835280374e-09, 9.39475075512064e-09, 9.382485899323001e-09 ] }, "drift": { "embed.weight": 164.5243882671684, "embed.bias": 86.31975598233767, "blocks.0.ln.weight": 9.94965648651123, "blocks.0.w1.weight": 116.60864500939675, "blocks.0.w1.bias": 119.5896283256837, "blocks.0.w2.weight": 139.89574047116318, "blocks.1.ln.weight": 6.952559471130371, "blocks.1.w1.weight": 171.01066211955538, "blocks.1.w1.bias": 184.66872276388008, "blocks.1.w2.weight": 113.70390206644275, "blocks.2.ln.weight": 8.037435531616211, "blocks.2.w1.weight": 204.0534914689403, "blocks.2.w1.bias": 205.1741516813065, "blocks.2.w2.weight": 112.9529550385935, "blocks.3.ln.weight": 7.489717483520508, "blocks.3.w1.weight": 199.12470243900225, "blocks.3.w1.bias": 198.81992912624156, "blocks.3.w2.weight": 104.7339169635497, "out_ln.weight": 1.6446622610092163, "out_head.weight": 8.633984180685534, "out_head.bias": 0.8233496742420285 } }, "fa": { "log": { "train_loss": [ 4.184186522827148, 4.179929753265381, 4.089846964416504, 4.048904882202148, 4.019415323638916, 4.003474291839599, 3.9891197368621825, 3.9800468362426757, 3.9685846406555174, 3.9551162309265138, 3.942166231536865, 3.9271056993865967, 3.915979294891357, 3.9063652964782714, 3.8933532473754884, 3.892304434890747, 3.8851386541748045, 3.880706764526367, 3.8791183966064455, 3.874532577972412, 3.870268273162842, 3.8705162049865725, 3.8667305128479006, 3.8626690478515626, 3.8593408808898926, 3.8570751000213623, 3.858237083206177, 3.8517847180938722, 3.8476831745910642, 3.8457262559509275, 3.840909554901123, 3.835939132537842, 3.834127527542114, 3.830621910324097, 3.8192956964874267, 3.8194126902770997, 3.8129332183074953, 3.803470793991089, 3.8037892980194092, 3.7975124457550047, 3.7967339707946777, 3.7944636122131348, 3.786077890625, 3.7846675917053223, 3.7791504066467283, 3.7762106622314455, 3.777709127731323, 3.7638775648498535, 3.765332045440674, 3.7623159846496583, 3.761364775314331, 3.7529254064178468, 3.749479592971802, 3.744997635345459, 3.7406992932128906, 3.74050118598938, 3.740082041091919, 3.7403511964416505, 3.7391156189727783, 3.7338412855529786, 3.733470021286011, 3.726227327346802, 3.7261299742889404, 3.722200173187256, 3.7222453344726563, 3.7236415620422365, 3.720610160598755, 3.71873546585083, 3.7171892735290526, 3.710947210006714, 3.7083427098846435, 3.7072678149414062, 3.70656751663208, 3.704509799041748, 3.7028028456115725, 3.7015775603485106, 3.7001070789337156, 3.6981460167694093, 3.696165197067261, 3.6909815367889403, 3.6975155518341065, 3.693085638427734, 3.6924814643096924, 3.6891826426696777, 3.688821280517578, 3.689004655685425, 3.6925391107177736, 3.6897733489990236, 3.687680895843506, 3.6844489430236815, 3.688553215789795, 3.6839564624023438, 3.685328346557617, 3.6844889741516114, 3.682617470626831, 3.685823777923584, 3.6817508575439453, 3.6835061328125, 3.683719411773682, 3.685913360671997 ], "train_acc": [ 0.06392, 0.06052, 0.07254, 0.0799, 0.0863, 0.08926, 0.09172, 0.09366, 0.0967, 0.09784, 0.10026, 0.10226, 0.10344, 0.10226, 0.10548, 0.10614, 0.10576, 0.1094, 0.10852, 0.11026, 0.1109, 0.1113, 0.11464, 0.11204, 0.114, 0.11382, 0.11484, 0.11308, 0.11652, 0.116, 0.11544, 0.11934, 0.1164, 0.11756, 0.12118, 0.12098, 0.1208, 0.1226, 0.12196, 0.12346, 0.1241, 0.12418, 0.12376, 0.12556, 0.12682, 0.12578, 0.12646, 0.12802, 0.12834, 0.12896, 0.1285, 0.13022, 0.13064, 0.13266, 0.132, 0.13304, 0.13302, 0.13274, 0.13234, 0.13406, 0.1352, 0.1352, 0.13452, 0.13546, 0.1378, 0.1353, 0.13576, 0.13548, 0.13744, 0.1404, 0.13826, 0.13848, 0.14008, 0.13914, 0.13812, 0.14022, 0.14048, 0.14066, 0.14008, 0.1426, 0.1408, 0.1407, 0.1429, 0.14204, 0.14294, 0.14272, 0.14102, 0.14226, 0.14414, 0.1436, 0.14408, 0.14312, 0.1434, 0.14338, 0.14488, 0.14448, 0.14394, 0.1426, 0.14502, 0.14398 ], "test_acc": [ 0.0598, 0.0694, 0.069, 0.0805, 0.086, 0.0911, 0.0953, 0.0986, 0.0954, 0.0949, 0.0993, 0.1051, 0.1069, 0.1004, 0.1066, 0.11, 0.1087, 0.1039, 0.1151, 0.1101, 0.1136, 0.1096, 0.1118, 0.1138, 0.1155, 0.1151, 0.1171, 0.1197, 0.1188, 0.1212, 0.1182, 0.1194, 0.12, 0.122, 0.1259, 0.1204, 0.1205, 0.1259, 0.1231, 0.1247, 0.1263, 0.1294, 0.1233, 0.1265, 0.1299, 0.1267, 0.1297, 0.1338, 0.1276, 0.1332, 0.1337, 0.1302, 0.1399, 0.1354, 0.1322, 0.1382, 0.1372, 0.1363, 0.1369, 0.1397, 0.1357, 0.1394, 0.144, 0.1395, 0.1448, 0.1373, 0.1435, 0.1417, 0.1398, 0.14, 0.1409, 0.1409, 0.1408, 0.141, 0.145, 0.1425, 0.1425, 0.1452, 0.1449, 0.1457, 0.1432, 0.1449, 0.1446, 0.1486, 0.1429, 0.1477, 0.1458, 0.146, 0.1449, 0.1455, 0.1442, 0.1453, 0.1458, 0.1462, 0.1468, 0.1456, 0.1462, 0.1464, 0.1465, 0.1464 ] }, "diagnostics": { "bp_cosine": [ 0.032749101519584656, -0.02806573174893856, -0.13188865780830383, 0.9981366395950317 ], "perturbation_rho": [ 0.023128217086195946, -0.033153705298900604, -0.03114410489797592, -0.04209538549184799 ], "nudging": { "0.001": [ -3.4319236874580383e-07, 3.664754331111908e-07, 2.3958273231983185e-07, -1.4158431440591812e-06 ], "0.003": [ -1.9101426005363464e-06, 6.353948265314102e-07, 1.5594996511936188e-06, -8.973991498351097e-06 ], "0.01": [ -6.851274520158768e-06, 8.617062121629715e-07, 6.175599992275238e-06, -3.897189162671566e-05 ] }, "hidden_norms_per_layer": [ 10709.3916015625, 183619.875, 1852831.0, 3034211.25, 169794.0625 ], "bp_grad_norms_per_layer": [ 1.2192158465040848e-05, 2.159640871468582e-06, 1.9386970961932093e-06, 1.93888718058588e-06, 1.940315996762365e-06 ] }, "drift": { "embed.weight": 97.51059845469358, "embed.bias": 45.31500470139968, "blocks.0.ln.weight": 2.126800775527954, "blocks.0.w1.weight": 24.113425166766486, "blocks.0.w1.bias": 25.534244327420822, "blocks.0.w2.weight": 69.8748433766199, "blocks.1.ln.weight": 1.5964641571044922, "blocks.1.w1.weight": 35.519098992915914, "blocks.1.w1.bias": 42.27561830539123, "blocks.1.w2.weight": 61.05750225415427, "blocks.2.ln.weight": 1.4540088176727295, "blocks.2.w1.weight": 41.92037705190106, "blocks.2.w1.bias": 47.8882125375433, "blocks.2.w2.weight": 47.86373297818362, "blocks.3.ln.weight": 1.6165372133255005, "blocks.3.w1.weight": 47.10451321088829, "blocks.3.w1.bias": 53.07025310766248, "blocks.3.w2.weight": 48.00579573400754, "out_ln.weight": 0.49529579281806946, "out_head.weight": 4.173857948425521, "out_head.bias": 3.4238911704789996 } } }, "config": { "dataset": "cifar100", "d_hidden": 256, "num_blocks": 4, "batch_size": 128, "epochs": 100, "lr": 0.001, "lr_fb": 0.001, "wd": 0.01, "lam": 0.1, "K": 4, "sigma_bridge": 0.05, "ema_momentum": 0.995, "term_grad_weight": 1.0, "seeds": [ 42 ], "gpu": 0, "output_dir": "results/cifar100_d256_L4_s42", "methods": [ "bp", "fa", "dfa" ], "random_targets": false, "penalty_lam": 0.0, "num_classes": 100 } }