{ "123": { "bp": { "log": { "train_loss": [ 3.967920153274536, 3.643245764923096, 3.486677552947998, 3.389170970993042, 3.3176012518310545, 3.2524516577148437, 3.196259712524414, 3.152468095245361, 3.116606701889038, 3.0753799269104003, 3.0412558711242674, 3.0103372289276122, 2.9851040869140624, 2.9536216430664064, 2.919733713989258, 2.8960993688964844, 2.872693371734619, 2.84370941696167, 2.8186388218688965, 2.802537234954834, 2.775104519882202, 2.752585594329834, 2.7324084952545165, 2.7153604712677004, 2.6842776251220704, 2.667392734527588, 2.6493602058410644, 2.6304399774932863, 2.6058559870910645, 2.5874294744110107, 2.5774721023559573, 2.560277436981201, 2.5345041821289063, 2.5185300437927247, 2.5040577131652832, 2.4822867348480226, 2.4557650662994384, 2.4456021406555175, 2.4341802147674563, 2.4113426206207276, 2.401808072052002, 2.3768489891052247, 2.3607859144592287, 2.3446667034912108, 2.329873125, 2.3144486641693116, 2.297909416656494, 2.2798938914489746, 2.266936881713867, 2.2467652272033694, 2.2354507537841797, 2.2205448300933837, 2.2017585570526124, 2.1915560935974123, 2.1722356983947755, 2.1580420698547362, 2.138712609024048, 2.1268783713531496, 2.1147646588897704, 2.1034324825286865, 2.088131596069336, 2.0740757128143312, 2.049090367126465, 2.0376090378570555, 2.0276603718566895, 2.0167199254989625, 2.0046445736694336, 1.9935844805145264, 1.9779361727905274, 1.9663786093521118, 1.9583420275497436, 1.94067984375, 1.929230265197754, 1.9221224881362915, 1.912958982810974, 1.9034557625579833, 1.8937049017715455, 1.8790358535003662, 1.876692020301819, 1.8658557400512696, 1.8551803729629517, 1.8528358583831788, 1.8547788270568848, 1.83579787399292, 1.825839135055542, 1.8247737094116212, 1.8170814221191407, 1.8188595218658448, 1.815419641456604, 1.8142681406402588, 1.8094764706802369, 1.7942750823593139, 1.7980619449615478, 1.7933076245498658, 1.7883917538452148, 1.7867490985870362, 1.7903605484771727, 1.79024905128479, 1.7870432864379884, 1.7758308526611328 ], "train_acc": [ 0.0986, 0.14448, 0.17068, 0.18832, 0.20038, 0.21186, 0.22002, 0.22894, 0.23506, 0.24404, 0.24964, 0.25374, 0.25782, 0.26606, 0.2711, 0.27704, 0.28214, 0.2849, 0.29088, 0.2938, 0.29992, 0.3054, 0.3096, 0.30988, 0.3165, 0.3225, 0.32436, 0.32574, 0.33332, 0.33762, 0.33924, 0.343, 0.34878, 0.3505, 0.35344, 0.35974, 0.36384, 0.36348, 0.3704, 0.36994, 0.37362, 0.38144, 0.38266, 0.3852, 0.3905, 0.3918, 0.39726, 0.40046, 0.40264, 0.40842, 0.40998, 0.41106, 0.4151, 0.4193, 0.4234, 0.42538, 0.43214, 0.43326, 0.43584, 0.4373, 0.44392, 0.4447, 0.44954, 0.45406, 0.45662, 0.45958, 0.46094, 0.46528, 0.46588, 0.47114, 0.47258, 0.47772, 0.47956, 0.48164, 0.48602, 0.48696, 0.48908, 0.49296, 0.49344, 0.49132, 0.49576, 0.49762, 0.49882, 0.49984, 0.50594, 0.5048, 0.50622, 0.50504, 0.50664, 0.50984, 0.50942, 0.5116, 0.5117, 0.51286, 0.51336, 0.51602, 0.5143, 0.51562, 0.51416, 0.51578 ], "test_acc": [ 0.1439, 0.1786, 0.2007, 0.208, 0.2165, 0.2315, 0.2325, 0.2385, 0.2483, 0.2501, 0.254, 0.2585, 0.2634, 0.2702, 0.2749, 0.2758, 0.2809, 0.2848, 0.2814, 0.2812, 0.2857, 0.2915, 0.2921, 0.2923, 0.2939, 0.295, 0.302, 0.2965, 0.3039, 0.3021, 0.3035, 0.3055, 0.3095, 0.3052, 0.3129, 0.3112, 0.3097, 0.308, 0.3123, 0.3104, 0.3135, 0.3145, 0.3105, 0.3173, 0.3165, 0.3154, 0.315, 0.3172, 0.3154, 0.316, 0.3159, 0.3184, 0.316, 0.3219, 0.3177, 0.3206, 0.3164, 0.3151, 0.3233, 0.3209, 0.3224, 0.3239, 0.3212, 0.318, 0.3248, 0.3255, 0.324, 0.3217, 0.323, 0.3212, 0.323, 0.3249, 0.3196, 0.3215, 0.3217, 0.3204, 0.3211, 0.3238, 0.3237, 0.3208, 0.3243, 0.3225, 0.3216, 0.3233, 0.3208, 0.3198, 0.3185, 0.319, 0.3196, 0.3201, 0.3197, 0.3214, 0.3217, 0.3219, 0.3217, 0.3221, 0.3217, 0.3217, 0.3218, 0.3218 ] }, "diagnostics": { "bp_cosine": [ 1.0, 1.0, 1.0, 1.0 ], "perturbation_rho": [ 0.9976038932800293, 0.9983073472976685, 0.9986611604690552, 0.9987543821334839 ], "nudging": { "0.001": [ -0.002336513716727495, -0.0024050232023000717, -0.002418666612356901, -0.0022877217270433903 ], "0.003": [ -0.007005014922469854, -0.007211083546280861, -0.007251948118209839, -0.006859402172267437 ], "0.01": [ -0.023301422595977783, -0.02398984506726265, -0.024125652387738228, -0.022821901366114616 ] }, "hidden_norms_per_layer": [ 228.35731506347656, 216.70431518554688, 197.1549835205078, 189.50289916992188, 193.0804901123047 ], "bp_grad_norms_per_layer": [ 0.0011750052217394114, 0.0012150296242907643, 0.00123639649245888, 0.0011825052788481116, 0.0009545879438519478 ] }, "drift": { "embed.weight": 7.3996956607158415, "embed.bias": 13.278713687472328, "blocks.0.ln.weight": 0.15684440732002258, "blocks.0.w1.weight": 3.6258136635529388, "blocks.0.w1.bias": 5.57490288506498, "blocks.0.w2.weight": 13.211600071846687, "blocks.1.ln.weight": 0.26902759075164795, "blocks.1.w1.weight": 3.9457858088997937, "blocks.1.w1.bias": 4.8656985397285295, "blocks.1.w2.weight": 14.784374975769294, "blocks.2.ln.weight": 0.37451720237731934, "blocks.2.w1.weight": 4.11706460399454, "blocks.2.w1.bias": 4.146142198756796, "blocks.2.w2.weight": 15.840220385160595, "blocks.3.ln.weight": 0.5431011319160461, "blocks.3.w1.weight": 4.464837126822249, "blocks.3.w1.bias": 3.6144859875754407, "blocks.3.w2.weight": 17.545783297025896, "out_ln.weight": 0.14564843475818634, "out_head.weight": 2.8817944671871123, "out_head.bias": 3.056171448565604 } }, "dfa": { "log": { "train_loss": [ 4.179048243713379, 4.067725786132812, 4.071110060424805, 4.091065592956543, 4.10388401260376, 4.1050505090332035, 4.10725640045166, 4.107924006652832, 4.103042182159424, 4.101259585266114, 4.0994670211791995, 4.092466098937988, 4.091690199432373, 4.088890562286377, 4.083471063919068, 4.082734613113403, 4.079294251174927, 4.074109273834228, 4.072821339111328, 4.072047854766845, 4.068868335876465, 4.064674437255859, 4.064519798278808, 4.062208421783447, 4.059947501296997, 4.0568214564514165, 4.0578119854736325, 4.05678455657959, 4.054043687744141, 4.04934491607666, 4.049620868835449, 4.048787186126709, 4.041555746765137, 4.046662612380981, 4.043773061065674, 4.0444277174377445, 4.043180988922119, 4.040115342483521, 4.041283304138184, 4.040324337005615, 4.03773109161377, 4.037001483154297, 4.036031613235473, 4.035188417053223, 4.035520594024658, 4.031777966766358, 4.036043049621582, 4.0324571522521975, 4.032010763549804, 4.032135240783691, 4.028632532043457, 4.028738806915284, 4.029276781768798, 4.030177063598633, 4.027558575515747, 4.0295089608764645, 4.02790505935669, 4.0268916735076905, 4.025881996612549, 4.024340693664551, 4.024784558410644, 4.022587103195191, 4.023770975189209, 4.024697297668457, 4.023757594070434, 4.025282996749878, 4.020828542404175, 4.02296846206665, 4.021536037902832, 4.021342518310547, 4.023781976318359, 4.020269456634521, 4.019074986038208, 4.020880366287232, 4.02205199508667, 4.019182727050781, 4.019642067718506, 4.019440497131348, 4.018397001342773, 4.020665705871582, 4.020597035064697, 4.019291881713867, 4.019497079620361, 4.018186092681884, 4.018166368408203, 4.017542911529541, 4.018283336334228, 4.015659646606445, 4.0151326177978515, 4.017795926361084, 4.016853940124512, 4.018234666442871, 4.015345659790039, 4.017251117095947, 4.018545875244141, 4.017435375213623, 4.015963760681152, 4.015863068695069, 4.015675266418457, 4.018068486938477 ], "train_acc": [ 0.06442, 0.07788, 0.07748, 0.07344, 0.07334, 0.0722, 0.07326, 0.07414, 0.07376, 0.07382, 0.07452, 0.07688, 0.07534, 0.07782, 0.08016, 0.07796, 0.08084, 0.0805, 0.0801, 0.08082, 0.08118, 0.08206, 0.08406, 0.08258, 0.08372, 0.08414, 0.08464, 0.08358, 0.0859, 0.08652, 0.08582, 0.08718, 0.08834, 0.08784, 0.08652, 0.08678, 0.08714, 0.08734, 0.0885, 0.08662, 0.08978, 0.08958, 0.08816, 0.08956, 0.08942, 0.09074, 0.08972, 0.08778, 0.08952, 0.08974, 0.09166, 0.08992, 0.0922, 0.09098, 0.09182, 0.092, 0.09068, 0.09088, 0.09264, 0.09206, 0.09184, 0.09252, 0.09102, 0.09206, 0.09348, 0.09252, 0.09214, 0.09232, 0.09276, 0.09236, 0.0915, 0.09298, 0.09236, 0.0928, 0.09326, 0.09264, 0.09282, 0.0924, 0.0927, 0.09328, 0.09474, 0.09338, 0.09336, 0.09342, 0.09358, 0.09408, 0.09404, 0.09384, 0.09388, 0.09484, 0.09448, 0.09396, 0.09364, 0.09346, 0.09286, 0.09446, 0.0944, 0.094, 0.0935, 0.09318 ], "test_acc": [ 0.0808, 0.0776, 0.0759, 0.0747, 0.0703, 0.0777, 0.0773, 0.0701, 0.0792, 0.0764, 0.0753, 0.0743, 0.0772, 0.0824, 0.0807, 0.081, 0.0784, 0.0813, 0.0774, 0.0803, 0.0826, 0.0865, 0.0793, 0.0798, 0.0821, 0.082, 0.0822, 0.0812, 0.0812, 0.0845, 0.0808, 0.087, 0.0849, 0.0823, 0.0861, 0.0851, 0.0873, 0.0833, 0.0826, 0.0855, 0.0843, 0.0861, 0.0904, 0.0866, 0.0882, 0.0861, 0.0875, 0.0871, 0.0843, 0.0852, 0.0874, 0.0892, 0.0868, 0.0887, 0.0882, 0.0874, 0.0816, 0.0865, 0.0861, 0.0856, 0.0874, 0.0869, 0.0905, 0.0844, 0.0838, 0.0898, 0.0881, 0.0865, 0.0891, 0.0868, 0.0861, 0.0874, 0.0909, 0.0889, 0.0856, 0.0875, 0.088, 0.0864, 0.0889, 0.0867, 0.0852, 0.0869, 0.0879, 0.0875, 0.0872, 0.0883, 0.0897, 0.0879, 0.0875, 0.0875, 0.0868, 0.087, 0.0869, 0.0868, 0.0875, 0.0873, 0.0871, 0.0872, 0.0872, 0.0872 ] }, "diagnostics": { "bp_cosine": [ 0.14216047525405884, -0.000992744229733944, -0.008401205763220787, -0.018219897523522377 ], "perturbation_rho": [ 0.053201157599687576, 0.0, 0.0, 0.0 ], "nudging": { "0.001": [ -6.109476089477539e-07, 3.725290298461914e-09, 0.0, 0.0 ], "0.003": [ -1.6260892152786255e-06, 3.725290298461914e-09, 3.725290298461914e-09, 0.0 ], "0.01": [ -5.261041224002838e-06, -1.4901161193847656e-08, 5.587935447692871e-09, 5.587935447692871e-09 ] }, "hidden_norms_per_layer": [ 19441.8984375, 14779116.0, 143599776.0, 226312160.0, 280974592.0 ], "bp_grad_norms_per_layer": [ 1.4361885405378416e-06, 8.184267485944474e-09, 7.640669430486469e-09, 7.643201627161034e-09, 7.633985887878225e-09 ] }, "drift": { "embed.weight": 162.24755066692174, "embed.bias": 87.51431369480697, "blocks.0.ln.weight": 10.42286205291748, "blocks.0.w1.weight": 122.86975821758014, "blocks.0.w1.bias": 127.51968896872295, "blocks.0.w2.weight": 144.60590878787062, "blocks.1.ln.weight": 7.615029811859131, "blocks.1.w1.weight": 201.6705191292493, "blocks.1.w1.bias": 211.79976689727667, "blocks.1.w2.weight": 125.73570047362797, "blocks.2.ln.weight": 8.678234100341797, "blocks.2.w1.weight": 226.59101485856874, "blocks.2.w1.bias": 218.5035230014578, "blocks.2.w2.weight": 132.07798132121874, "blocks.3.ln.weight": 8.401058197021484, "blocks.3.w1.weight": 221.76508002622688, "blocks.3.w1.bias": 204.23780375800553, "blocks.3.w2.weight": 123.66583688447245, "out_ln.weight": 1.673746943473816, "out_head.weight": 9.550233758399107, "out_head.bias": 0.9096587370527971 } }, "fa": { "log": { "train_loss": [ 4.1970798970031735, 4.170009743804932, 4.130948940124512, 4.094561860961914, 4.050638759155273, 4.024049156494141, 4.013653193511963, 4.014947728729248, 4.032731681671143, 4.047689598388672, 4.044174816589355, 4.0340768909454345, 4.026809699783326, 4.0191610552215575, 4.009859062652588, 4.004589066696167, 4.000014752273559, 3.9934563516235353, 3.989729327392578, 3.9867174030303953, 3.980116393737793, 3.979052162322998, 3.976570917816162, 3.972892049102783, 3.9706854679870607, 3.964727850341797, 3.9669337309265136, 3.962656950378418, 3.959514749221802, 3.950425816116333, 3.9518183033752443, 3.944525026168823, 3.9370513851928712, 3.9401658685302734, 3.9340728694152833, 3.933683126220703, 3.929854642791748, 3.9251790898132324, 3.925471938934326, 3.920428736419678, 3.9194086865997315, 3.916180799026489, 3.910477230758667, 3.9091271575164797, 3.9079003392791747, 3.9009140887451172, 3.903854856109619, 3.896608190765381, 3.8983821444702147, 3.8914879888153076, 3.8876359399414064, 3.8866743579101564, 3.886013005371094, 3.8823793061065675, 3.8820696644592285, 3.8807897300720215, 3.8792161915588377, 3.873366918258667, 3.872225737609863, 3.867956961669922, 3.8688921507263183, 3.864798822555542, 3.8635835083770753, 3.8616594306182863, 3.86274955078125, 3.8615631605529783, 3.855813476715088, 3.8575434510040285, 3.8537498898315428, 3.8516512804412844, 3.853169044036865, 3.848847638626099, 3.8501204053497315, 3.8475019567871094, 3.8478130152130126, 3.841899245147705, 3.8445189476776123, 3.8427941259002685, 3.841190276489258, 3.8432645402526857, 3.840385712738037, 3.841228730545044, 3.8406887775421144, 3.838410483932495, 3.8366693685913087, 3.8328694233703615, 3.8348467766571046, 3.8281507495880125, 3.8312854344177247, 3.8341920923614503, 3.832209959793091, 3.832971982574463, 3.827496012649536, 3.832459149017334, 3.8320746698760986, 3.833449503631592, 3.8311014671325685, 3.828391033859253, 3.8309260621643064, 3.832998709869385 ], "train_acc": [ 0.06318, 0.06262, 0.06896, 0.07402, 0.08474, 0.0894, 0.09134, 0.0919, 0.08782, 0.08538, 0.08668, 0.0868, 0.08842, 0.0918, 0.09218, 0.09394, 0.09588, 0.09596, 0.09472, 0.09676, 0.09782, 0.09964, 0.10034, 0.09964, 0.09956, 0.10158, 0.10016, 0.10198, 0.10194, 0.1036, 0.10318, 0.10394, 0.1047, 0.1039, 0.10536, 0.10528, 0.10612, 0.10692, 0.10644, 0.10704, 0.10844, 0.1082, 0.10744, 0.10862, 0.10826, 0.11176, 0.10998, 0.10964, 0.11052, 0.1107, 0.11298, 0.11352, 0.11398, 0.11322, 0.11342, 0.11344, 0.11324, 0.11226, 0.11314, 0.11472, 0.11338, 0.11524, 0.11458, 0.11598, 0.11536, 0.11664, 0.11622, 0.11714, 0.1169, 0.11656, 0.11594, 0.11838, 0.1178, 0.11898, 0.1183, 0.11852, 0.11904, 0.11914, 0.1193, 0.11876, 0.11858, 0.12086, 0.11934, 0.11944, 0.12044, 0.1214, 0.12184, 0.12008, 0.12106, 0.12018, 0.12146, 0.12004, 0.12064, 0.11986, 0.12106, 0.12248, 0.12166, 0.12014, 0.12266, 0.12278 ], "test_acc": [ 0.0708, 0.0659, 0.0746, 0.0752, 0.0902, 0.0946, 0.0937, 0.0824, 0.0936, 0.0847, 0.095, 0.084, 0.0884, 0.0959, 0.0927, 0.0944, 0.101, 0.0972, 0.098, 0.0997, 0.095, 0.1032, 0.0977, 0.0949, 0.1003, 0.1004, 0.0969, 0.101, 0.1019, 0.1037, 0.0979, 0.103, 0.1047, 0.1026, 0.1092, 0.1071, 0.1062, 0.1024, 0.1051, 0.1068, 0.1088, 0.105, 0.108, 0.1068, 0.1092, 0.1112, 0.1096, 0.1084, 0.108, 0.1091, 0.1102, 0.1141, 0.1109, 0.1138, 0.1085, 0.1143, 0.1107, 0.1135, 0.1153, 0.1135, 0.1181, 0.1159, 0.1171, 0.1126, 0.1165, 0.1193, 0.1206, 0.113, 0.1139, 0.1156, 0.1152, 0.1201, 0.1239, 0.1171, 0.1187, 0.1197, 0.1187, 0.1212, 0.1187, 0.1182, 0.1187, 0.1203, 0.1195, 0.1212, 0.122, 0.1207, 0.1192, 0.1205, 0.1205, 0.1215, 0.12, 0.1201, 0.1203, 0.1203, 0.1213, 0.1208, 0.1209, 0.1209, 0.1209, 0.1208 ] }, "diagnostics": { "bp_cosine": [ 0.017588060349225998, -0.016497433185577393, -0.053346455097198486, 0.9986427426338196 ], "perturbation_rho": [ 0.02255980297923088, 0.005564957857131958, -0.049259480088949203, -0.011541965417563915 ], "nudging": { "0.001": [ 1.280568540096283e-07, 1.62515789270401e-07, 7.82310962677002e-08, -5.41098415851593e-07 ], "0.003": [ 1.0710209608078003e-07, 2.7334317564964294e-07, 1.1641532182693481e-07, -2.648215740919113e-06 ], "0.01": [ 3.57162207365036e-07, 2.039596438407898e-07, 8.307397365570068e-07, -1.1221971362829208e-05 ] }, "hidden_norms_per_layer": [ 19381.54296875, 312615.90625, 2321965.25, 3640082.0, 692179.875 ], "bp_grad_norms_per_layer": [ 5.805031378258718e-06, 6.129388907538669e-07, 5.996270147079485e-07, 6.020890737090667e-07, 6.002287022965902e-07 ] }, "drift": { "embed.weight": 160.52406007787414, "embed.bias": 24.62414456524661, "blocks.0.ln.weight": 2.7208244800567627, "blocks.0.w1.weight": 31.32931103808957, "blocks.0.w1.bias": 24.578464288381767, "blocks.0.w2.weight": 76.69986787657376, "blocks.1.ln.weight": 2.1396796703338623, "blocks.1.w1.weight": 41.923375948001016, "blocks.1.w1.bias": 42.687481681423954, "blocks.1.w2.weight": 41.924463722510936, "blocks.2.ln.weight": 1.8985017538070679, "blocks.2.w1.weight": 49.16152213344094, "blocks.2.w1.bias": 49.20711111001434, "blocks.2.w2.weight": 40.28985166850181, "blocks.3.ln.weight": 1.8008719682693481, "blocks.3.w1.weight": 48.175959456780525, "blocks.3.w1.bias": 46.53396040604286, "blocks.3.w2.weight": 50.22310178120013, "out_ln.weight": 0.6654373407363892, "out_head.weight": 4.396233728367674, "out_head.bias": 1.1577362249288412 } } }, "config": { "dataset": "cifar100", "d_hidden": 256, "num_blocks": 4, "batch_size": 128, "epochs": 100, "lr": 0.001, "lr_fb": 0.001, "wd": 0.01, "lam": 0.1, "K": 4, "sigma_bridge": 0.05, "ema_momentum": 0.995, "term_grad_weight": 1.0, "seeds": [ 123 ], "gpu": 0, "output_dir": "results/cifar100_d256_L4_s123", "methods": [ "bp", "fa", "dfa" ], "random_targets": false, "penalty_lam": 0.0, "num_classes": 100 } }