1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
|
{
"protocol": "Replayed view truth, evaluation-only. Each cell is the within-node cross-modal Spearman of relation fields at the true view correspondence, against a shuffled-view control.",
"nodes": 5000,
"baseline_isolated_cosine": {
"matched_mean": 0.1279621422290802,
"shuffled_mean": -0.002400045981630683,
"gap_z": 46.60123062133789
},
"channels": {
"text_attn_g0 x vision_attn_g0": {
"matched_mean": 0.0246360395103693,
"shuffled_mean": 0.0018186916131526232,
"gap_z": 7.828590393066406
},
"text_attn_g0 x vision_attn_g1": {
"matched_mean": 0.02522178180515766,
"shuffled_mean": 0.0023200020659714937,
"gap_z": 7.974187850952148
},
"text_attn_g0 x vision_attn_g2": {
"matched_mean": 0.026846596971154213,
"shuffled_mean": 0.003111843252554536,
"gap_z": 8.169827461242676
},
"text_attn_g0 x vision_ctx_cos": {
"matched_mean": 0.03297057002782822,
"shuffled_mean": 0.0029256935231387615,
"gap_z": 9.99555492401123
},
"text_attn_g1 x vision_attn_g0": {
"matched_mean": 0.058189209550619125,
"shuffled_mean": 0.0014253206318244338,
"gap_z": 19.106473922729492
},
"text_attn_g1 x vision_attn_g1": {
"matched_mean": 0.062455616891384125,
"shuffled_mean": 0.001968336058780551,
"gap_z": 20.717960357666016
},
"text_attn_g1 x vision_attn_g2": {
"matched_mean": 0.06540140509605408,
"shuffled_mean": 0.002817628439515829,
"gap_z": 21.213720321655273
},
"text_attn_g1 x vision_ctx_cos": {
"matched_mean": 0.07714134454727173,
"shuffled_mean": 0.0024559840094298124,
"gap_z": 24.414701461791992
},
"text_attn_g2 x vision_attn_g0": {
"matched_mean": 0.05762112885713577,
"shuffled_mean": 0.00122644763905555,
"gap_z": 18.899877548217773
},
"text_attn_g2 x vision_attn_g1": {
"matched_mean": 0.062174536287784576,
"shuffled_mean": 0.0014159244019538164,
"gap_z": 20.693180084228516
},
"text_attn_g2 x vision_attn_g2": {
"matched_mean": 0.06511640548706055,
"shuffled_mean": 0.002295177197083831,
"gap_z": 21.171533584594727
},
"text_attn_g2 x vision_ctx_cos": {
"matched_mean": 0.07819025963544846,
"shuffled_mean": 0.0018244234379380941,
"gap_z": 24.823591232299805
},
"text_attn_g3 x vision_attn_g0": {
"matched_mean": 0.048348233103752136,
"shuffled_mean": 0.0008306392701342702,
"gap_z": 15.629465103149414
},
"text_attn_g3 x vision_attn_g1": {
"matched_mean": 0.052185527980327606,
"shuffled_mean": 0.001778527395799756,
"gap_z": 16.855741500854492
},
"text_attn_g3 x vision_attn_g2": {
"matched_mean": 0.0553436279296875,
"shuffled_mean": 0.0025363026652485132,
"gap_z": 17.472745895385742
},
"text_attn_g3 x vision_ctx_cos": {
"matched_mean": 0.0647592768073082,
"shuffled_mean": 0.0026518579106777906,
"gap_z": 19.79732322692871
},
"text_ctx_cos x vision_attn_g0": {
"matched_mean": 0.15180939435958862,
"shuffled_mean": -0.001719625317491591,
"gap_z": 51.155216217041016
},
"text_ctx_cos x vision_attn_g1": {
"matched_mean": 0.17549680173397064,
"shuffled_mean": -0.0016138383653014898,
"gap_z": 59.05224609375
},
"text_ctx_cos x vision_attn_g2": {
"matched_mean": 0.18720850348472595,
"shuffled_mean": -0.00174369674641639,
"gap_z": 62.099239349365234
},
"text_ctx_cos x vision_ctx_cos": {
"matched_mean": 0.21646080911159515,
"shuffled_mean": -0.004605674650520086,
"gap_z": 69.60621643066406
}
}
}
|
