1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
|
{
"s_eT_tgw1.0_wr0.2": {
"test_acc": 0.5605,
"mean_bp_cosine": 0.46006373316049576,
"mean_rho": 0.5312944203615189,
"mean_nudge": -0.03811332583427429,
"bp_cosine_per_layer": [
0.18216726183891296,
0.5338927507400513,
0.5597370862960815,
0.5644578337669373
],
"rho_per_layer": [
0.2443590760231018,
0.5769795775413513,
0.6636242270469666,
0.6402148008346558
],
"final_value_loss": 0.39113953742980956,
"s_type": "eT",
"term_grad_weight": 1.0,
"warmup_ratio": 0.2
},
"s_deltaL_tgw1.0_wr0.2": {
"test_acc": 0.442,
"mean_bp_cosine": 0.5583586767315865,
"mean_rho": 0.49782785028219223,
"mean_nudge": -0.051266513764858246,
"bp_cosine_per_layer": [
0.2746746242046356,
0.6255925893783569,
0.6661019921302795,
0.6670655012130737
],
"rho_per_layer": [
0.27920976281166077,
0.5733566284179688,
0.594862163066864,
0.5438828468322754
],
"final_value_loss": 0.47215704221725463,
"s_type": "deltaL",
"term_grad_weight": 1.0,
"warmup_ratio": 0.2
},
"s_eT_hL_tgw1.0_wr0.2": {
"test_acc": 0.559,
"mean_bp_cosine": 0.0016190913447644562,
"mean_rho": 0.016410242416895926,
"mean_nudge": -0.00021975510026095435,
"bp_cosine_per_layer": [
-0.004644347354769707,
0.0010294992243871093,
-0.0010337075218558311,
0.011124921031296253
],
"rho_per_layer": [
0.046507176011800766,
0.007014757487922907,
-0.006595376878976822,
0.018714413046836853
],
"final_value_loss": 0.8668150995254517,
"s_type": "eT_hL",
"term_grad_weight": 1.0,
"warmup_ratio": 0.2
},
"s_deltaL_hL_tgw1.0_wr0.2": {
"test_acc": 0.5635,
"mean_bp_cosine": 0.018434561206959188,
"mean_rho": 0.026365829166024923,
"mean_nudge": -0.0017427445782232098,
"bp_cosine_per_layer": [
0.013983936980366707,
0.02003202959895134,
0.00282791955396533,
0.036894358694553375
],
"rho_per_layer": [
-0.006572568789124489,
0.049806319177150726,
0.020004384219646454,
0.042225182056427
],
"final_value_loss": 0.7973948855400086,
"s_type": "deltaL_hL",
"term_grad_weight": 1.0,
"warmup_ratio": 0.2
}
}
|
