Retire the correlation gate: the shared spectrum governs recovery

A controlled truncation refutes the project's central go/no-go rule.
Projecting the recovering synthetic fields to rank r holds the field
correlation at 0.902-0.929 while recovery moves 6.2% -> 12.9% -> 95.6%
across ranks 4, 8, 16. A field past the supposed 0.9 threshold recovers
13%, so correlation neither predicts nor forbids recovery and the width
of the shared spectrum is what moves it.

The gate becomes a joint condition on correlation and shared width,
measured by principal angles against a scene-shuffled null. Neither
suffices alone: 18 shared directions at 0.508 fails, 11 at 0.902 fails.

With the old gate retired, natural data was finally searched: 0.0000
against 0.0039 chance. The old verdict was right, its reasoning was not.

Also closes route D by measurement. rho_IT ~ sqrt(4 log N / N) rises as N
falls, and at N = 16 through 96 the deepest state a strong searcher
reaches is deeper than the truth in 3/3 replicates at every size.

Free gains: eigenvalue-weighted projection over a wide basis with
128-dim text vectors takes the correlation 0.656 -> 0.716 and shared
width 10 -> 16. Hubness refuted as an inflation hypothesis.

Moving the per-image segmentation eigendecomposition onto the GPU cut
batch time from 130s to 1.9s.

Co-Authored-By: Claude <noreply@anthropic.com>

1 files changed, 265 insertions, 0 deletions

diff --git a/artifacts/vg_5k/projection_sweep.json b/artifacts/vg_5k/projection_sweep.json
new file mode 100644
index 0000000..8881383
--- /dev/null
+++ b/artifacts/vg_5k/projection_sweep.json
@@ -0,0 +1,265 @@
+{
+  "protocol": "Projection fitted per modality on scenes outside the evaluated set; hidden pairs read only for the correlation. Baseline is linear, keep=8, shrinkage=0.05.",
+  "evaluated": 256,
+  "rows": [
+    {
+      "variant": "linear",
+      "keep": 4,
+      "shrinkage": 0.01,
+      "correlation": 0.6453137317062534
+    },
+    {
+      "variant": "linear",
+      "keep": 8,
+      "shrinkage": 0.01,
+      "correlation": 0.6495421074385569
+    },
+    {
+      "variant": "linear",
+      "keep": 16,
+      "shrinkage": 0.01,
+      "correlation": 0.6470767138606525
+    },
+    {
+      "variant": "linear",
+      "keep": 32,
+      "shrinkage": 0.01,
+      "correlation": 0.6227407963928071
+    },
+    {
+      "variant": "linear",
+      "keep": 64,
+      "shrinkage": 0.01,
+      "correlation": 0.6142722294788403
+    },
+    {
+      "variant": "linear",
+      "keep": 4,
+      "shrinkage": 0.05,
+      "correlation": 0.65249653291618
+    },
+    {
+      "variant": "linear",
+      "keep": 8,
+      "shrinkage": 0.05,
+      "correlation": 0.6558829559111284
+    },
+    {
+      "variant": "linear",
+      "keep": 16,
+      "shrinkage": 0.05,
+      "correlation": 0.6519028422068834
+    },
+    {
+      "variant": "linear",
+      "keep": 32,
+      "shrinkage": 0.05,
+      "correlation": 0.6274081754272933
+    },
+    {
+      "variant": "linear",
+      "keep": 64,
+      "shrinkage": 0.05,
+      "correlation": 0.619760201230621
+    },
+    {
+      "variant": "linear",
+      "keep": 4,
+      "shrinkage": 0.2,
+      "correlation": 0.6607265117297335
+    },
+    {
+      "variant": "linear",
+      "keep": 8,
+      "shrinkage": 0.2,
+      "correlation": 0.6582353562445471
+    },
+    {
+      "variant": "linear",
+      "keep": 16,
+      "shrinkage": 0.2,
+      "correlation": 0.6513068306415126
+    },
+    {
+      "variant": "linear",
+      "keep": 32,
+      "shrinkage": 0.2,
+      "correlation": 0.6292167478629883
+    },
+    {
+      "variant": "linear",
+      "keep": 64,
+      "shrinkage": 0.2,
+      "correlation": 0.6230802049271942
+    },
+    {
+      "variant": "linear",
+      "keep": 4,
+      "shrinkage": 1.0,
+      "correlation": 0.6578716141349961
+    },
+    {
+      "variant": "linear",
+      "keep": 8,
+      "shrinkage": 1.0,
+      "correlation": 0.6405385423647052
+    },
+    {
+      "variant": "linear",
+      "keep": 16,
+      "shrinkage": 1.0,
+      "correlation": 0.6280429181450786
+    },
+    {
+      "variant": "linear",
+      "keep": 32,
+      "shrinkage": 1.0,
+      "correlation": 0.6142054425413382
+    },
+    {
+      "variant": "linear",
+      "keep": 64,
+      "shrinkage": 1.0,
+      "correlation": 0.6106156014213242
+    },
+    {
+      "variant": "weighted^0.5",
+      "keep": 8,
+      "shrinkage": 0.05,
+      "correlation": 0.6698246762363366
+    },
+    {
+      "variant": "weighted^1.0",
+      "keep": 8,
+      "shrinkage": 0.05,
+      "correlation": 0.665060874094268
+    },
+    {
+      "variant": "weighted^0.5",
+      "keep": 16,
+      "shrinkage": 0.05,
+      "correlation": 0.6826263927928936
+    },
+    {
+      "variant": "weighted^1.0",
+      "keep": 16,
+      "shrinkage": 0.05,
+      "correlation": 0.6721336214645782
+    },
+    {
+      "variant": "weighted^0.5",
+      "keep": 32,
+      "shrinkage": 0.05,
+      "correlation": 0.687375151713245
+    },
+    {
+      "variant": "weighted^1.0",
+      "keep": 32,
+      "shrinkage": 0.05,
+      "correlation": 0.6753247222620263
+    },
+    {
+      "variant": "weighted^0.5",
+      "keep": 64,
+      "shrinkage": 0.05,
+      "correlation": 0.6922159706308013
+    },
+    {
+      "variant": "weighted^1.0",
+      "keep": 64,
+      "shrinkage": 0.05,
+      "correlation": 0.6767939950719634
+    },
+    {
+      "variant": "weighted^0.5",
+      "keep": 8,
+      "shrinkage": 0.2,
+      "correlation": 0.672827939323971
+    },
+    {
+      "variant": "weighted^1.0",
+      "keep": 8,
+      "shrinkage": 0.2,
+      "correlation": 0.6691737665681982
+    },
+    {
+      "variant": "weighted^0.5",
+      "keep": 16,
+      "shrinkage": 0.2,
+      "correlation": 0.6834609415667857
+    },
+    {
+      "variant": "weighted^1.0",
+      "keep": 16,
+      "shrinkage": 0.2,
+      "correlation": 0.675193020267635
+    },
+    {
+      "variant": "weighted^0.5",
+      "keep": 32,
+      "shrinkage": 0.2,
+      "correlation": 0.6883825134950975
+    },
+    {
+      "variant": "weighted^1.0",
+      "keep": 32,
+      "shrinkage": 0.2,
+      "correlation": 0.6780073690253073
+    },
+    {
+      "variant": "weighted^0.5",
+      "keep": 64,
+      "shrinkage": 0.2,
+      "correlation": 0.6918038515455318
+    },
+    {
+      "variant": "weighted^1.0",
+      "keep": 64,
+      "shrinkage": 0.2,
+      "correlation": 0.6788283392518062
+    },
+    {
+      "variant": "kernel g=0.25",
+      "keep": 8,
+      "shrinkage": 0.05,
+      "correlation": 0.6239363564555721
+    },
+    {
+      "variant": "kernel g=0.25",
+      "keep": 16,
+      "shrinkage": 0.05,
+      "correlation": 0.6199483306751332
+    },
+    {
+      "variant": "kernel g=0.25",
+      "keep": 32,
+      "shrinkage": 0.05,
+      "correlation": 0.6043865046351463
+    },
+    {
+      "variant": "kernel g=1.0",
+      "keep": 8,
+      "shrinkage": 0.05,
+      "correlation": 0.5588913870951268
+    },
+    {
+      "variant": "kernel g=1.0",
+      "keep": 16,
+      "shrinkage": 0.05,
+      "correlation": 0.5472099552797389
+    },
+    {
+      "variant": "kernel g=1.0",
+      "keep": 32,
+      "shrinkage": 0.05,
+      "correlation": 0.5251472992094228
+    }
+  ],
+  "best": {
+    "variant": "weighted^0.5",
+    "keep": 64,
+    "shrinkage": 0.05,
+    "correlation": 0.6922159706308013
+  },
+  "recovery_threshold": 0.9
+}
\ No newline at end of file