diff options
Diffstat (limited to 'results/fa_dfa_d256_L12_seed2/results_cifar10.json')
| -rw-r--r-- | results/fa_dfa_d256_L12_seed2/results_cifar10.json | 969 |
1 files changed, 969 insertions, 0 deletions
diff --git a/results/fa_dfa_d256_L12_seed2/results_cifar10.json b/results/fa_dfa_d256_L12_seed2/results_cifar10.json new file mode 100644 index 0000000..70776c8 --- /dev/null +++ b/results/fa_dfa_d256_L12_seed2/results_cifar10.json @@ -0,0 +1,969 @@ +{ + "2": { + "dfa": { + "log": { + "train_loss": [ + 2.092766728363037, + 2.071445608520508, + 2.0701047048187258, + 2.0665665516662597, + 2.067467219543457, + 2.066609907684326, + 2.067391862182617, + 2.0658849488830566, + 2.065659608001709, + 2.065375587425232, + 2.0646166355133055, + 2.0635058017349244, + 2.0630363352966308, + 2.0627129718780517, + 2.0648969506835937, + 2.063057222671509, + 2.062092007293701, + 2.062603257293701, + 2.0662800875091554, + 2.0649482633972167, + 2.0633998834609986, + 2.0644692920684813, + 2.0658038328552246, + 2.064523613204956, + 2.0647737124633787, + 2.063466505584717, + 2.0641796745300294, + 2.0648309812164305, + 2.06493783164978, + 2.0642703956604005, + 2.065596410675049, + 2.067113346405029, + 2.0670792920684815, + 2.0674756241607666, + 2.066420622558594, + 2.066541123046875, + 2.0665447259521486, + 2.0689855900573733, + 2.067198378753662, + 2.0686578797149657, + 2.06837053024292, + 2.0679433206939697, + 2.0698270501708986, + 2.0687507022094724, + 2.070133085708618, + 2.068685834007263, + 2.0692544694519044, + 2.069098585357666, + 2.070015923538208, + 2.0695283253479, + 2.0700033166503906, + 2.0695154525756836, + 2.0704460035705567, + 2.0702316856384275, + 2.0706022947692873, + 2.0712034313201904, + 2.070552589263916, + 2.071217707977295, + 2.0711937672424314, + 2.072246285095215, + 2.068459527130127, + 2.0709887282562254, + 2.0723592150115966, + 2.072640259552002, + 2.0726871685791015, + 2.069176601486206, + 2.071328667678833, + 2.0708692224121092, + 2.0729969272613524, + 2.0725777490997315, + 2.0719549550628664, + 2.0703202209472655, + 2.0716604712677, + 2.0724998860168458, + 2.072380848312378, + 2.0722954035949708, + 2.070641780166626, + 2.071795623474121, + 2.0716914239120485, + 2.073136379394531, + 2.0709913427734374, + 2.0712071760559083, + 2.071455654296875, + 2.0722044621276856, + 2.071295954055786, + 2.0727719202041626, + 2.0717286488342284, + 2.0731436529541014, + 2.0714281242370607, + 2.07157251701355, + 2.0711801865386965, + 2.071615061340332, + 2.0700036166381834, + 2.0721589122009276, + 2.071688577957153, + 2.0693919525909426, + 2.0720266328430177, + 2.0719666347503662, + 2.0707905860900877, + 2.072465877342224 + ], + "train_acc": [ + 0.23206, + 0.2349, + 0.23432, + 0.2351, + 0.2368, + 0.23936, + 0.23746, + 0.24068, + 0.24038, + 0.24096, + 0.2413, + 0.24336, + 0.2423, + 0.24238, + 0.24486, + 0.24402, + 0.24642, + 0.24514, + 0.24274, + 0.24326, + 0.24602, + 0.24686, + 0.24686, + 0.24712, + 0.24644, + 0.24618, + 0.24624, + 0.24394, + 0.2468, + 0.24656, + 0.24538, + 0.24386, + 0.24564, + 0.24706, + 0.24646, + 0.24484, + 0.24802, + 0.24608, + 0.24632, + 0.2467, + 0.24684, + 0.2485, + 0.24542, + 0.24522, + 0.2466, + 0.24708, + 0.24736, + 0.24888, + 0.2499, + 0.24734, + 0.24814, + 0.24822, + 0.24724, + 0.24642, + 0.24566, + 0.2475, + 0.24706, + 0.2471, + 0.24564, + 0.2467, + 0.24904, + 0.2474, + 0.24816, + 0.24836, + 0.24672, + 0.24812, + 0.24726, + 0.2477, + 0.24868, + 0.24794, + 0.2471, + 0.24962, + 0.24896, + 0.24758, + 0.2479, + 0.24738, + 0.24834, + 0.24676, + 0.2483, + 0.24708, + 0.24694, + 0.24966, + 0.25074, + 0.2455, + 0.24898, + 0.2485, + 0.2489, + 0.24698, + 0.24926, + 0.248, + 0.2504, + 0.25158, + 0.25008, + 0.24946, + 0.24912, + 0.2502, + 0.24898, + 0.25056, + 0.24956, + 0.24938 + ], + "test_acc": [ + 0.2527, + 0.2607, + 0.246, + 0.2601, + 0.2571, + 0.266, + 0.2599, + 0.2551, + 0.2513, + 0.2552, + 0.2562, + 0.2613, + 0.2675, + 0.2656, + 0.264, + 0.2367, + 0.2691, + 0.2628, + 0.2582, + 0.2619, + 0.2729, + 0.2684, + 0.2705, + 0.2654, + 0.2614, + 0.2692, + 0.2514, + 0.2663, + 0.2645, + 0.2706, + 0.2657, + 0.2484, + 0.264, + 0.2659, + 0.2636, + 0.2718, + 0.2601, + 0.2585, + 0.2664, + 0.2562, + 0.2649, + 0.2623, + 0.2622, + 0.2599, + 0.26, + 0.2643, + 0.2635, + 0.2689, + 0.2733, + 0.2606, + 0.2726, + 0.2692, + 0.2663, + 0.264, + 0.2636, + 0.2583, + 0.2676, + 0.2779, + 0.2617, + 0.2661, + 0.2716, + 0.2707, + 0.266, + 0.2684, + 0.2702, + 0.2573, + 0.2637, + 0.2675, + 0.2738, + 0.2544, + 0.2636, + 0.2666, + 0.2671, + 0.2687, + 0.2668, + 0.2666, + 0.2612, + 0.2664, + 0.2675, + 0.2643, + 0.2657, + 0.2671, + 0.2624, + 0.2684, + 0.2624, + 0.2684, + 0.2646, + 0.2645, + 0.2655, + 0.263, + 0.2648, + 0.2677, + 0.2644, + 0.2658, + 0.2661, + 0.2656, + 0.2649, + 0.2652, + 0.2652, + 0.2652 + ] + }, + "diagnostics": { + "bp_cosine": [ + 0.21441566944122314, + 0.0008995254174806178, + -0.0011490017641335726, + -0.0006673308089375496, + -0.0004349752562120557, + 0.001973408740013838, + 0.00042850073077715933, + 0.0003071604296565056, + 0.0011147534241899848, + -0.0019351006485521793, + -0.00017125890008173883, + 0.0004633249482139945 + ], + "perturbation_rho": [ + 0.03552835434675217, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "nudging": { + "0.001": [ + -8.102506399154663e-08, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + -2.7939677238464355e-09 + ], + "0.003": [ + -3.5390257835388184e-07, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + -2.7939677238464355e-09 + ], + "0.01": [ + -1.2461096048355103e-06, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + -2.7939677238464355e-09 + ] + }, + "hidden_norms_per_layer": [ + 45168.5625, + 768591872.0, + 1624721024.0, + 1961752960.0, + 2100152448.0, + 2500510976.0, + 2508241920.0, + 2941273344.0, + 2989846784.0, + 3150896128.0, + 3321745152.0, + 3535885312.0, + 3636161792.0 + ], + "bp_grad_norms_per_layer": [ + 1.8181714267484494e-07, + 3.9466230283835557e-10, + 3.8982980732349404e-10, + 3.898551481640311e-10, + 3.8986006090091507e-10, + 3.900082479191269e-10, + 3.8990691231255425e-10, + 3.899590372835604e-10, + 3.900538503298634e-10, + 3.901238221359904e-10, + 3.9011002761490943e-10, + 3.9018840936044796e-10, + 3.9032757581658473e-10 + ] + }, + "drift": { + "embed.weight": 408.6565633657492, + "embed.bias": 423.28110485460303, + "blocks.0.ln.weight": 9.70258617401123, + "blocks.0.w1.weight": 286.6816708847807, + "blocks.0.w1.bias": 311.9898470727492, + "blocks.0.w2.weight": 590.5617039721983, + "blocks.1.ln.weight": 11.42900562286377, + "blocks.1.w1.weight": 339.1322667459177, + "blocks.1.w1.bias": 319.191979974844, + "blocks.1.w2.weight": 485.54212783414897, + "blocks.2.ln.weight": 10.751296043395996, + "blocks.2.w1.weight": 306.6921002240311, + "blocks.2.w1.bias": 286.8824089191201, + "blocks.2.w2.weight": 419.397779279592, + "blocks.3.ln.weight": 10.36766242980957, + "blocks.3.w1.weight": 282.4814974926866, + "blocks.3.w1.bias": 252.8190034898104, + "blocks.3.w2.weight": 394.269135440916, + "blocks.4.ln.weight": 11.355030059814453, + "blocks.4.w1.weight": 326.6622613339727, + "blocks.4.w1.bias": 314.3979807413481, + "blocks.4.w2.weight": 461.04714607850275, + "blocks.5.ln.weight": 8.093010902404785, + "blocks.5.w1.weight": 217.27913735619703, + "blocks.5.w1.bias": 207.68042804194928, + "blocks.5.w2.weight": 294.75329385151446, + "blocks.6.ln.weight": 11.64451789855957, + "blocks.6.w1.weight": 331.7248686164095, + "blocks.6.w1.bias": 309.00249567397475, + "blocks.6.w2.weight": 484.57904574635427, + "blocks.7.ln.weight": 8.409276008605957, + "blocks.7.w1.weight": 225.75220538114135, + "blocks.7.w1.bias": 216.53163331133942, + "blocks.7.w2.weight": 324.7052199531864, + "blocks.8.ln.weight": 11.350098609924316, + "blocks.8.w1.weight": 320.93864855988267, + "blocks.8.w1.bias": 306.19484707322147, + "blocks.8.w2.weight": 442.72910719653026, + "blocks.9.ln.weight": 10.944558143615723, + "blocks.9.w1.weight": 307.8993593428455, + "blocks.9.w1.bias": 290.93916631623483, + "blocks.9.w2.weight": 440.2539306532767, + "blocks.10.ln.weight": 11.270280838012695, + "blocks.10.w1.weight": 321.55439380753864, + "blocks.10.w1.bias": 288.9418448710429, + "blocks.10.w2.weight": 411.81523324589523, + "blocks.11.ln.weight": 10.046154022216797, + "blocks.11.w1.weight": 286.57908879426424, + "blocks.11.w1.bias": 275.6362336173934, + "blocks.11.w2.weight": 399.62017806369147, + "out_ln.weight": 0.7484539747238159, + "out_head.weight": 8.125483521933623, + "out_head.bias": 0.7273602975063448 + } + }, + "fa": { + "log": { + "train_loss": [ + 2.074700547027588, + 2.030654427947998, + 2.0057160476684572, + 1.9860789415740967, + 1.9725520542144774, + 1.95931587890625, + 1.9556365052032472, + 1.949812787322998, + 1.9462810146331788, + 1.9393242764282226, + 1.9346750591278077, + 1.9271999840545655, + 1.9232344121932983, + 1.9189868188095094, + 1.919980531692505, + 1.9102826860046387, + 1.90958540763855, + 1.906280760154724, + 1.9068798965835572, + 1.9018908991241454, + 1.8965862115859986, + 1.894734903831482, + 1.893279719581604, + 1.8903550232315063, + 1.8864814365005493, + 1.884921064682007, + 1.8844925833129882, + 1.8856516895294189, + 1.881670213623047, + 1.880837113380432, + 1.8822157500457763, + 1.882532099876404, + 1.882689362220764, + 1.881834898109436, + 1.8771058824920654, + 1.8768038847351074, + 1.876141697921753, + 1.877026067199707, + 1.8729603647613526, + 1.8720484650039673, + 1.8726009467315674, + 1.8684697211456298, + 1.871248878250122, + 1.869133426246643, + 1.8653471280288696, + 1.8631344338226319, + 1.8654646643066406, + 1.8650356465911866, + 1.8616812967300416, + 1.8621146585464476, + 1.8621270999908448, + 1.8586566192626952, + 1.8607052938842774, + 1.8588444815063476, + 1.8568199838256836, + 1.855796114845276, + 1.8530415719604492, + 1.8559222234725952, + 1.856116162338257, + 1.8548921677017212, + 1.8508316849517823, + 1.8514919400787353, + 1.8525736228179932, + 1.8520959496307372, + 1.8482627039337158, + 1.8451711013412475, + 1.8483959712982179, + 1.8483288528442383, + 1.848829554977417, + 1.848747034034729, + 1.8459476345443726, + 1.8471334496307372, + 1.8432773351669312, + 1.8460039371490478, + 1.8457718017196656, + 1.8440695235824585, + 1.8412491543579101, + 1.8426079097747803, + 1.8468621598052979, + 1.8427482269668578, + 1.8419597222137452, + 1.841674754562378, + 1.8427531800079346, + 1.8430379685211182, + 1.8435487987899781, + 1.842195815963745, + 1.842669207496643, + 1.8409197840118408, + 1.8410899520492554, + 1.8386338932037354, + 1.8406484191894532, + 1.8388528701400757, + 1.837176209716797, + 1.8422198328399657, + 1.8413942455673218, + 1.839041402053833, + 1.8415731252288818, + 1.839198327407837, + 1.8407912665557862, + 1.8404628273773194 + ], + "train_acc": [ + 0.23152, + 0.2483, + 0.26406, + 0.27226, + 0.27666, + 0.2867, + 0.28482, + 0.2895, + 0.29026, + 0.29298, + 0.29752, + 0.30064, + 0.30484, + 0.30198, + 0.30454, + 0.30912, + 0.31148, + 0.3106, + 0.3109, + 0.31142, + 0.317, + 0.31652, + 0.31412, + 0.31586, + 0.32138, + 0.31964, + 0.32098, + 0.32112, + 0.32356, + 0.32264, + 0.32256, + 0.32364, + 0.32494, + 0.32508, + 0.32778, + 0.32594, + 0.32872, + 0.32838, + 0.32896, + 0.32934, + 0.32812, + 0.32972, + 0.33028, + 0.33102, + 0.33146, + 0.33284, + 0.33266, + 0.33498, + 0.3342, + 0.33536, + 0.33232, + 0.33478, + 0.33422, + 0.33506, + 0.33846, + 0.33662, + 0.33606, + 0.33612, + 0.33684, + 0.3362, + 0.3385, + 0.33634, + 0.33684, + 0.33968, + 0.33902, + 0.34212, + 0.34068, + 0.33904, + 0.33856, + 0.34094, + 0.34132, + 0.34188, + 0.3424, + 0.34188, + 0.34086, + 0.34386, + 0.34486, + 0.34176, + 0.3404, + 0.34224, + 0.34428, + 0.34282, + 0.34242, + 0.3411, + 0.3426, + 0.34362, + 0.34128, + 0.34484, + 0.34452, + 0.34312, + 0.34328, + 0.34296, + 0.34432, + 0.34434, + 0.34316, + 0.3433, + 0.34516, + 0.346, + 0.34178, + 0.3444 + ], + "test_acc": [ + 0.2629, + 0.2729, + 0.2809, + 0.3082, + 0.3046, + 0.3106, + 0.3059, + 0.3072, + 0.3226, + 0.3051, + 0.3139, + 0.3342, + 0.336, + 0.3323, + 0.3386, + 0.3217, + 0.3373, + 0.3327, + 0.3293, + 0.3421, + 0.346, + 0.3439, + 0.3489, + 0.3488, + 0.3561, + 0.3493, + 0.3318, + 0.3436, + 0.3441, + 0.3531, + 0.3485, + 0.3409, + 0.3518, + 0.3509, + 0.3502, + 0.36, + 0.3513, + 0.3503, + 0.3492, + 0.3425, + 0.3538, + 0.3402, + 0.3518, + 0.3541, + 0.3519, + 0.3499, + 0.3639, + 0.3504, + 0.3575, + 0.3545, + 0.3569, + 0.3559, + 0.3541, + 0.36, + 0.3586, + 0.3625, + 0.3608, + 0.3595, + 0.3629, + 0.3649, + 0.3603, + 0.3653, + 0.3651, + 0.3619, + 0.3644, + 0.3655, + 0.3616, + 0.3621, + 0.3631, + 0.361, + 0.3605, + 0.3647, + 0.3669, + 0.366, + 0.3644, + 0.3677, + 0.3653, + 0.3664, + 0.3679, + 0.3688, + 0.3658, + 0.3666, + 0.3664, + 0.368, + 0.3659, + 0.3647, + 0.3667, + 0.368, + 0.3688, + 0.367, + 0.3683, + 0.3688, + 0.3674, + 0.3685, + 0.3683, + 0.3682, + 0.3679, + 0.368, + 0.3683, + 0.3684 + ] + }, + "diagnostics": { + "bp_cosine": [ + -0.014710275456309319, + 0.02226361073553562, + 0.03605305403470993, + 0.003516306634992361, + 0.012390266172587872, + 0.058464229106903076, + -0.04347992688417435, + 0.0054677389562129974, + 0.015231117606163025, + -0.03650633990764618, + -0.04501262307167053, + 0.9956947565078735 + ], + "perturbation_rho": [ + 0.008943558670580387, + 0.00040163123048841953, + -0.040518321096897125, + -0.022643186151981354, + 0.044356103986501694, + 0.02536097541451454, + 0.022935548797249794, + 0.028215371072292328, + 0.011796066537499428, + 0.016897639259696007, + -0.004378834739327431, + -0.04016704112291336 + ], + "nudging": { + "0.001": [ + 5.587935447692871e-09, + -9.569339454174042e-08, + -4.7963112592697144e-08, + -2.2351741790771484e-08, + -4.6566128730773926e-09, + -6.28642737865448e-08, + 7.450580596923828e-09, + -3.771856427192688e-08, + -4.516914486885071e-08, + 5.587935447692871e-09, + 1.6298145055770874e-08, + -6.00004568696022e-07 + ], + "0.003": [ + 3.2922253012657166e-07, + -1.3760291039943695e-07, + -1.1292286217212677e-07, + -9.778887033462524e-09, + -7.450580596923828e-09, + -1.5995465219020844e-07, + 8.055940270423889e-08, + -1.862645149230957e-08, + -9.313225746154785e-08, + 9.778887033462524e-09, + 0.0, + -1.941109076142311e-06 + ], + "0.01": [ + 1.0672956705093384e-06, + -3.3783726394176483e-07, + -4.216562956571579e-07, + -1.5529803931713104e-07, + -9.639188647270203e-08, + -5.022156983613968e-07, + 2.952292561531067e-07, + -8.288770914077759e-08, + -1.0244548320770264e-07, + 2.5960616767406464e-07, + 3.287568688392639e-07, + -6.931368261575699e-06 + ] + }, + "hidden_norms_per_layer": [ + 11081.2958984375, + 118196.625, + 614115.75, + 828878.8125, + 921725.125, + 1069404.875, + 1237270.375, + 1402473.625, + 1720954.375, + 1997529.875, + 2216433.75, + 2325002.5, + 1993140.375 + ], + "bp_grad_norms_per_layer": [ + 1.0706523426051717e-05, + 6.52372534659662e-07, + 4.187676552191988e-07, + 4.1873394707181433e-07, + 4.1315911403216887e-07, + 4.1367323433405545e-07, + 4.1193311517417897e-07, + 4.11331342320409e-07, + 4.107659776764194e-07, + 4.1129530359285127e-07, + 4.1165918673868873e-07, + 4.1159788111144735e-07, + 4.073814920957375e-07 + ] + }, + "drift": { + "embed.weight": 98.4177462274615, + "embed.bias": 23.69940251347622, + "blocks.0.ln.weight": 2.1789700984954834, + "blocks.0.w1.weight": 24.29409235730548, + "blocks.0.w1.bias": 18.31784316907853, + "blocks.0.w2.weight": 101.8385022981204, + "blocks.1.ln.weight": 1.3870518207550049, + "blocks.1.w1.weight": 26.373394752918255, + "blocks.1.w1.bias": 22.570698680348862, + "blocks.1.w2.weight": 47.64154250315352, + "blocks.2.ln.weight": 0.9734257459640503, + "blocks.2.w1.weight": 21.148009831845542, + "blocks.2.w1.bias": 20.856615521275852, + "blocks.2.w2.weight": 53.70072333386875, + "blocks.3.ln.weight": 1.1795594692230225, + "blocks.3.w1.weight": 20.73167225109056, + "blocks.3.w1.bias": 18.579235259886254, + "blocks.3.w2.weight": 86.87336634029717, + "blocks.4.ln.weight": 0.997854471206665, + "blocks.4.w1.weight": 21.602122373726505, + "blocks.4.w1.bias": 22.47848497309394, + "blocks.4.w2.weight": 53.78872222769277, + "blocks.5.ln.weight": 0.9774206280708313, + "blocks.5.w1.weight": 22.79774822645063, + "blocks.5.w1.bias": 24.21613131547018, + "blocks.5.w2.weight": 58.26315367277549, + "blocks.6.ln.weight": 1.0200481414794922, + "blocks.6.w1.weight": 26.093657863707982, + "blocks.6.w1.bias": 26.27916630293517, + "blocks.6.w2.weight": 33.89854797735531, + "blocks.7.ln.weight": 1.0476559400558472, + "blocks.7.w1.weight": 28.70756132891856, + "blocks.7.w1.bias": 30.025930965175473, + "blocks.7.w2.weight": 39.54073820690228, + "blocks.8.ln.weight": 1.030435562133789, + "blocks.8.w1.weight": 29.48263158312378, + "blocks.8.w1.bias": 31.083890237870737, + "blocks.8.w2.weight": 37.91543643893619, + "blocks.9.ln.weight": 1.0992196798324585, + "blocks.9.w1.weight": 29.805154447874546, + "blocks.9.w1.bias": 31.101382421095806, + "blocks.9.w2.weight": 34.2461727464101, + "blocks.10.ln.weight": 1.1228426694869995, + "blocks.10.w1.weight": 25.197803524439806, + "blocks.10.w1.bias": 24.54186072317423, + "blocks.10.w2.weight": 46.988036920078784, + "blocks.11.ln.weight": 0.9333347082138062, + "blocks.11.w1.weight": 19.997632331729342, + "blocks.11.w1.bias": 16.848829693670883, + "blocks.11.w2.weight": 117.94127434264082, + "out_ln.weight": 0.5421217679977417, + "out_head.weight": 6.825100238203774, + "out_head.bias": 0.5928900621441576 + } + } + }, + "config": { + "dataset": "cifar10", + "d_hidden": 256, + "num_blocks": 12, + "batch_size": 128, + "epochs": 100, + "lr": 0.001, + "lr_fb": 0.001, + "wd": 0.01, + "lam": 0.1, + "K": 4, + "sigma_bridge": 0.05, + "ema_momentum": 0.995, + "term_grad_weight": 1.0, + "seeds": [ + 2 + ], + "gpu": 0, + "output_dir": "results/fa_dfa_d256_L12_seed2", + "methods": [ + "fa", + "dfa" + ], + "random_targets": false, + "penalty_lam": 0.0, + "num_classes": 10 + } +}
\ No newline at end of file |