From 08fd63b8fee62ccdc284380c9832900ee83f9ede Mon Sep 17 00:00:00 2001 From: YurenHao0426 Date: Sat, 1 Aug 2026 16:15:41 -0500 Subject: Retire the correlation gate: the shared spectrum governs recovery A controlled truncation refutes the project's central go/no-go rule. Projecting the recovering synthetic fields to rank r holds the field correlation at 0.902-0.929 while recovery moves 6.2% -> 12.9% -> 95.6% across ranks 4, 8, 16. A field past the supposed 0.9 threshold recovers 13%, so correlation neither predicts nor forbids recovery and the width of the shared spectrum is what moves it. The gate becomes a joint condition on correlation and shared width, measured by principal angles against a scene-shuffled null. Neither suffices alone: 18 shared directions at 0.508 fails, 11 at 0.902 fails. With the old gate retired, natural data was finally searched: 0.0000 against 0.0039 chance. The old verdict was right, its reasoning was not. Also closes route D by measurement. rho_IT ~ sqrt(4 log N / N) rises as N falls, and at N = 16 through 96 the deepest state a strong searcher reaches is deeper than the truth in 3/3 replicates at every size. Free gains: eigenvalue-weighted projection over a wide basis with 128-dim text vectors takes the correlation 0.656 -> 0.716 and shared width 10 -> 16. Hubness refuted as an inflation hypothesis. Moving the per-image segmentation eigendecomposition onto the GPU cut batch time from 130s to 1.9s. Co-Authored-By: Claude --- CANDIDATE_REGISTER.md | 196 ++++++++------- EXTENDED_ABSTRACT.md | 74 ++++-- LAB_NOTES.md | 73 ++++++ RANK_RESULTS.md | 175 +++++++++++++ artifacts/synth_v1/omit_colour.json | 34 +++ artifacts/synth_v1/omit_count.json | 34 +++ artifacts/synth_v1/omit_none.json | 34 +++ artifacts/synth_v1/omit_size.json | 34 +++ artifacts/synth_v1/omit_size_count.json | 34 +++ artifacts/synth_v1/rank_causal.json | 51 ++++ artifacts/vg_5k/field_anatomy.json | 40 +++ artifacts/vg_5k/field_anatomy_w64.json | 40 +++ artifacts/vg_5k/natural_pipeline_oracle.json | 9 + artifacts/vg_5k/natural_pipeline_seg16.json | 9 + artifacts/vg_5k/natural_pipeline_w64.json | 9 + artifacts/vg_5k/np_deg3_12.json | 10 + artifacts/vg_5k/np_deg3_20.json | 10 + artifacts/vg_5k/np_deg3_8.json | 10 + artifacts/vg_5k/np_n1024.json | 9 + artifacts/vg_5k/np_n256.json | 9 + artifacts/vg_5k/np_n512.json | 9 + artifacts/vg_5k/np_wv128.json | 9 + artifacts/vg_5k/np_wv24.json | 9 + artifacts/vg_5k/np_wv256.json | 9 + artifacts/vg_5k/np_wv48.json | 9 + artifacts/vg_5k/projection_sweep.json | 265 ++++++++++++++++++++ artifacts/vg_5k/shared_rank.json | 343 ++++++++++++++++++++++++++ artifacts/vg_5k/shared_rank_degree.json | 351 +++++++++++++++++++++++++++ artifacts/vg_5k/shared_rank_final.json | 351 +++++++++++++++++++++++++++ artifacts/vg_5k/shared_rank_scale.json | 259 ++++++++++++++++++++ artifacts/vg_5k/shared_rank_segments.json | 343 ++++++++++++++++++++++++++ artifacts/vg_5k/shared_rank_text.json | 343 ++++++++++++++++++++++++++ logs/backbone_battery.log | 0 logs/obj_base_gpu.log | 3 + logs/obj_large.log | 3 + logs/projection_sweep.log | 44 ++++ logs/rank_causal.log | 8 + logs/small_n_gate.log | 7 + worldalign/diversity_subset.py | 171 +++++++++++++ worldalign/field_anatomy.py | 148 +++++++++++ worldalign/natural_objects.py | 47 +++- worldalign/natural_pipeline.py | 78 +++++- worldalign/natural_recovery.py | 130 ++++++++++ worldalign/projection_sweep.py | 240 ++++++++++++++++++ worldalign/rank_causal.py | 141 +++++++++++ worldalign/shared_rank.py | 135 +++++++++++ worldalign/tier0_pipeline.py | 37 ++- 47 files changed, 4245 insertions(+), 141 deletions(-) create mode 100644 RANK_RESULTS.md create mode 100644 artifacts/synth_v1/omit_colour.json create mode 100644 artifacts/synth_v1/omit_count.json create mode 100644 artifacts/synth_v1/omit_none.json create mode 100644 artifacts/synth_v1/omit_size.json create mode 100644 artifacts/synth_v1/omit_size_count.json create mode 100644 artifacts/synth_v1/rank_causal.json create mode 100644 artifacts/vg_5k/field_anatomy.json create mode 100644 artifacts/vg_5k/field_anatomy_w64.json create mode 100644 artifacts/vg_5k/natural_pipeline_oracle.json create mode 100644 artifacts/vg_5k/natural_pipeline_seg16.json create mode 100644 artifacts/vg_5k/natural_pipeline_w64.json create mode 100644 artifacts/vg_5k/np_deg3_12.json create mode 100644 artifacts/vg_5k/np_deg3_20.json create mode 100644 artifacts/vg_5k/np_deg3_8.json create mode 100644 artifacts/vg_5k/np_n1024.json create mode 100644 artifacts/vg_5k/np_n256.json create mode 100644 artifacts/vg_5k/np_n512.json create mode 100644 artifacts/vg_5k/np_wv128.json create mode 100644 artifacts/vg_5k/np_wv24.json create mode 100644 artifacts/vg_5k/np_wv256.json create mode 100644 artifacts/vg_5k/np_wv48.json create mode 100644 artifacts/vg_5k/projection_sweep.json create mode 100644 artifacts/vg_5k/shared_rank.json create mode 100644 artifacts/vg_5k/shared_rank_degree.json create mode 100644 artifacts/vg_5k/shared_rank_final.json create mode 100644 artifacts/vg_5k/shared_rank_scale.json create mode 100644 artifacts/vg_5k/shared_rank_segments.json create mode 100644 artifacts/vg_5k/shared_rank_text.json create mode 100644 logs/backbone_battery.log create mode 100644 logs/obj_base_gpu.log create mode 100644 logs/obj_large.log create mode 100644 logs/projection_sweep.log create mode 100644 logs/rank_causal.log create mode 100644 logs/small_n_gate.log create mode 100644 worldalign/diversity_subset.py create mode 100644 worldalign/field_anatomy.py create mode 100644 worldalign/natural_recovery.py create mode 100644 worldalign/projection_sweep.py create mode 100644 worldalign/rank_causal.py create mode 100644 worldalign/shared_rank.py diff --git a/CANDIDATE_REGISTER.md b/CANDIDATE_REGISTER.md index efed50e..eff860e 100644 --- a/CANDIDATE_REGISTER.md +++ b/CANDIDATE_REGISTER.md @@ -1,107 +1,122 @@ # Candidate register -The obstruction on natural data decomposes into five factors, and every -candidate attacks exactly one of them. Field correlation stands at 0.656 -against the 0.9 that polynomial recovery needs, so the register is -ordered by expected movement per unit of effort rather than by elegance. +*Reordered 2026-08-01 after the rank experiment. See `RANK_RESULTS.md`.* + +The register used to be ordered by how much each candidate was expected to +raise the field correlation, because correlation was the go/no-go statistic. +That statistic has been shown not to govern recovery: a synthetic field +truncated to rank 8 correlates at 0.906 and recovers 13%, while the same field +at rank 16 recovers 96%. **The target is now a pair — the correlation and the +width of the shared spectrum — and candidates are ordered by which one they +move.** Status is one of: **done** (measured, number recorded), **running**, -**queued** (specified, not started), **open** (idea, not specified). +**queued** (specified, not started), **open** (idea, not specified), +**closed** (measured and eliminated). + +Current position: correlation 0.716, shared directions 15, against a +recovering synthetic reference at 0.928 and 26. -## A. Raise the field correlation +## A. Widen the shared spectrum -The binding constraint. Everything else is worth less than a tenth of -what this is worth if it moves. +The binding constraint, and the one only identified today. Both modalities are +individually rich — vision effective rank 40, text 48 — and agree on 15 +directions. Candidates here are judged on the shared count, not the +correlation. + +| Candidate | Cost | Status | +|---|---|---| +| Eigenvalue-weighted projection, wide basis | low | done: shared 10 to 15, ρ 0.656 to 0.692 | +| Text vector dimension | low | done: shared 10 to 17, saturates; ρ peaks 0.716 at 128 | +| More segments per scene (6 to 16) | low | closed: shared 15 to 14 | +| Annotated boxes instead of segmentation (oracle) | low | closed: shared 15, buys one direction | +| More scenes (256 to 1024) | low | closed: shared 23 to 19 at matched width | +| Augmentation orbits, four views | medium | closed: no gain, ρ 0.6559 either way | +| Larger vision backbone (DINOv2-large) | medium | running: predicted useless, see below | +| Third-order moments in the set kernel | low | queued | +| Corpora with naturally wide overlap | high | queued | +| Intervention responses: occlude, re-encode, use the change | high | queued | +| Corpus-trained text encoder instead of PPMI vectors | medium | queued | +| Structured phrase encoding: head and modifiers apart | low | queued | +| Nonlinear content projection (kernel discriminant) | low | closed: 0.62 against 0.69 linear | + +The backbone row is a live prediction test rather than a hope. The shared count +does not move when segmentation is replaced by ground-truth boxes, when +segments are nearly tripled, or when the vision field's own effective rank +doubles, so a larger backbone is predicted to leave it unchanged. The run is +finishing anyway because the project's rule is to measure the bound rather than +infer it, and a cheap refutation of my own prediction is worth more than the +GPU time. + +**Corpora with naturally wide overlap is the row today's result argues for and +the one never attempted.** The synthetic world reaches 26 shared directions +because its captions state exactly the world state; Visual Genome reaches 15 +because a region description and a patch descriptor overlap on about that many +aspects of a photograph. If the ceiling is a property of what the two corpora +are about, the lever is corpus choice: dense descriptions, product listings +carrying photographs and full specifications, screenshots paired with +accessibility trees, scientific figures with their captions. + +## B. Raise the correlation + +Still necessary — 18 shared directions at ρ = 0.508 also fails — but no longer +sufficient, and no longer the thing to optimise alone. | Candidate | Cost | Status | |---|---|---| | Continuous states instead of class codes | low | done: 0.166 to 0.489 | | Content projection, between-scene over within-scene | low | done: 0.489 to 0.656 | -| Augmentation orbits, four views | medium | done: no gain, 0.6559 either way | -| Larger vision backbone (DINOv2-large, -giant) | medium | queued | +| Eigenvalue weighting plus wider text vectors | low | done: 0.656 to 0.716 | +| Degree-residual fields | low | done: +0.015, keep, composes freely | +| Hubness inflation of the statistic | low | closed: refuted, degree model is 1-5% of variance | | Multi-scale patch features, several receptive fields | medium | queued | -| Corpus-trained text encoder instead of PPMI vectors | medium | queued | -| Structured phrase encoding: head and modifiers apart | low | queued | -| Intervention responses: occlude a region, re-encode, use the change | high | queued | -| Native multi-view corpora: product photographs, video frames | high | queued | -| Nonlinear content projection (kernel discriminant) | low | open | | Adaptive segment count with quality filtering | low | open | -The intervention row is the one the closed world argued for hardest and -the one never run on natural data. It needs real forward passes -- masked -re-encoding on both sides -- not cached-feature algebra, which is a -tautology. - -## B. Reshape the landscape - -Decoys exist because they match second-order statistics. Either raise -them or widen the true basin. +## C. Reshape the landscape — closed | Candidate | Cost | Status | |---|---|---| -| Third-order terms, tr(M^3) | low | done: deepens truth and decoys alike | -| Rank-truncation ladder, coarse to sharp | low | done: at chance, all ranks | -| Convex-concave path following (PATH, FAQ family) | medium | queued | -| Entropic doubly-stochastic relaxation | low | done: stalls far above truth | -| Degree normalisation of the fields | low | queued | -| Hubness correction (CSLS) on field similarities | low | queued | -| Sparsified fields, top-k neighbours only | low | queued | -| Functional term: held-out generalisation of the induced map | medium | running | - -The rank ladder settles the category. At the natural-data field quality -every rank from four to full returns chance accuracy, as does full-rank -spectral initialisation with refinement -- 0.0039 against 0.0039 chance. -**Landscape reshaping does not substitute for field correlation:** below -the polynomial threshold the deficit is information the algorithm class -cannot use, not basins it cannot find, so smoothing has nothing to -recover. Path following is the canonical landscape-deformation method for -quadratic assignment and remains untried, but this result lowers its -expected value at the current correlation and it should wait until the -correlation moves. The functional term -is landscape reshaping by orthogonal information: a decoy that copies -relational statistics still has to induce a map that generalises. - -## C. Search harder +| Third-order terms, tr(M³) | low | done: deepens truth and decoys alike | +| Rank-truncation ladder, coarse to sharp | low | closed: at chance, all ranks | +| Entropic doubly-stochastic relaxation | low | closed: stalls far above truth | +| Convex-concave path following (PATH, FAQ family) | medium | closed by inference | -| Candidate | Cost | Status | -|---|---|---| -| Spectral initialisation plus exact refinement | low | done: 53.5% to 95.3% composed | -| Parallel tempering with exact swap deltas | medium | done: equilibrates into the decoy shelf | -| Multiple spectral inits over eta, keep best by energy | low | queued | -| Seeded expansion from high-confidence pairs | medium | queued | -| Belief propagation on the assignment factor graph | high | queued | -| Cluster moves: swap blocks rather than single pairs | low | queued | -| Path relinking and crossover over permutations | medium | open | -| Branch and bound, exact on small blocks | medium | open | -| Sum-of-squares or low-degree hierarchies | high | open | - -Accepting exponential cost does not rescue the hard phase at fixed size: -tempering already is exponential-time and fails by equilibrating, not by -running out of budget. Exponential methods pay only where the block is -small enough to finish, which is why C and D are the same programme seen -from two sides. - -## D. Shrink the problem - -The phase boundary is joint in correlation and population size, so a -corpus need not be matched at once. +Below the recoverable region the deficit is information the algorithm class +cannot use, not basins it cannot find, so smoothing has nothing to recover. +Path following remains the canonical landscape-deformation method for quadratic +assignment and remains untried, but it should not be tried until the shared +spectrum moves. + +## D. Shrink the problem — closed, and the reasoning was backwards | Candidate | Cost | Status | |---|---|---| -| Largest recoverable N at the natural field quality | low | running | -| Diversity-selected bootstrap subset instead of random | low | queued | -| Hierarchical: cluster first, match within cluster | medium | queued | -| Many small blocks, merge by consistency voting | medium | queued | - -Diversity selection is the cheapest idea in the register and possibly -the best value: decoys arise from scenes that resemble each other, and -choosing a mutually dissimilar bootstrap set thins them at no -algorithmic cost. It composes with small blocks by construction, since -if only a few hundred pairs are needed they should be the most -distinguishable few hundred. +| Largest recoverable N at natural field quality | low | closed: none, 16 through 256 | +| Diversity-selected bootstrap subset | low | closed by the above, and not protocol-legal | +| Hierarchical: cluster first, match within cluster | medium | closed | +| Many small blocks, merge by consistency voting | medium | closed | + +The premise was that the hard phase is joint in correlation and population +size, so small blocks would need less correlation. The sign is the other way: +ρ_IT ≈ √(4 log N / N) **rises** as N falls — 0.29 at 256 scenes, 0.51 at 64, +0.83 at 16 — so small blocks need *more* correlation, not less. The gate +confirms it directly: at N = 16, 24, 32, 48 and 64, the best state a strong +searcher reaches is deeper than the truth in three replicates of three. When +the truth is not the optimum the algorithm class is irrelevant, which is also +why accepting exponential cost buys nothing. + +Diversity selection additionally turns out not to be protocol-legal as +imagined: choosing dissimilar images and dissimilar captions independently +yields two sets that are not in bijection, so the selection cannot be realised +without the pairing it is meant to discover. It survives only inside the +unbalanced formulation in F. ## E. Add orthogonal signal +Unaffected by today's result and now relatively more attractive, because +information that does not pass through the relation field is not subject to the +shared-spectrum ceiling. + | Candidate | Cost | Status | |---|---|---| | Held-out generalisation of the induced map | medium | running | @@ -113,22 +128,25 @@ distinguishable few hundred. ## F. Change the formulation Real corpora are not in bijection, so the permutation formulation is a -scaffold that has to be replaced eventually regardless of performance. +scaffold that has to be replaced eventually regardless of performance. Today's +closure of D makes this more urgent, since the unbalanced formulation is now +the only home for selection-based ideas. | Candidate | Cost | Status | |---|---|---| | Unbalanced coupling: partial mass, unmatched items allowed | medium | queued | +| Diversity selection inside the unbalanced formulation | medium | queued | | Many-to-many soft assignment | medium | open | | Part-level matching across the corpus rather than scene-level | medium | open | | Joint factor-level and scene-level, alternating | medium | open | ## Order of attempt -Cheap and untried first, because the register's value is in eliminating -branches quickly: diversity selection, degree normalisation, hubness -correction, sparsified fields, multiple spectral inits, structured phrase -encoding, higher-order joint tables. Then the medium tier led by path -following and the functional term. The two expensive rows worth pre- -committing to are intervention responses and a natively multi-view -corpus, because the closed world named both as protocol requirements -rather than optimisations. +Third-order moments first: it is the cheapest untried thing that raises field +rank by construction, since a set kernel carrying third moments spans more +directions than one carrying two. Then structured phrase encoding and the +corpus-trained text encoder, because the text side is the one that moved the +shared count at all. Then the two rows worth pre-committing to regardless of +cost — intervention responses, and a corpus chosen for naturally wide overlap — +the second of which is now the register's central bet rather than one candidate +among many. diff --git a/EXTENDED_ABSTRACT.md b/EXTENDED_ABSTRACT.md index 1a75b8c..1e96cae 100644 --- a/EXTENDED_ABSTRACT.md +++ b/EXTENDED_ABSTRACT.md @@ -71,14 +71,26 @@ third-order energy that appeared to pass, and a value-alignment ceiling of six correct entries in ten that combining two signals lifted to ten. Each was settled by computing the bound rather than inferring it. -The gate yields a cheap predictor. Recovery tracks a single measurable -quantity, the correlation between the two relation fields at the true -pairing, and the phase boundary sits where correlated-matching theory -puts it: at five hundred scenes the information-theoretic threshold is -near 0.29 and the polynomial-algorithm threshold near 0.9. Nothing we -built worked below 0.9, and recovery appeared as soon as fields crossed -it. Any new representation can therefore be priced in minutes, before a -single search is run. +The gate also produced our longest-lived mistake, and correcting it is +the paper's second result. Recovery appeared to track one measurable +quantity — the correlation between the two relation fields at the true +pairing — with the phase boundary where correlated-matching theory puts +it, near 0.9 at five hundred scenes. We priced representations against +that number for months. **It does not govern recovery.** Truncating the +synthetic fields that recover to rank eight leaves the correlation at +0.906, past the supposed threshold, and recovery at 13%; the same fields +at rank sixteen recover 96%. Correlation is nearly fixed across the +ladder while recovery crosses its whole range. + +The theory said so in advance and we misread it. Matching thresholds are +derived for exchangeable full-rank noise, where each of the N²/2 field +entries constrains the pairing independently; a rank-*r* shared component +supplies about *rN*, an eighth as many at 256 scenes with *r* near ten. +The corrected gate is a joint condition on the correlation and on the +width of the shared spectrum — the number of directions along which the +two modalities agree, measured by principal angles against a +scene-shuffled null. Neither alone suffices: 18 shared directions at +correlation 0.508 fails, and 11 at 0.902 fails. ## A closed world settles the mechanism @@ -120,9 +132,9 @@ frequency rankings diverge in the tail. Ported to Visual Genome under the same recipe — DINOv2 patch features, spectral segmentation, distributional word statistics — the field -correlation reaches 0.656, against the 0.9 recovery needs. No recovery -run is warranted, and the trajectory that got there is more informative -than the number. +correlation reaches 0.716 and the shared spectrum 15 directions, against +0.928 and 26 for the synthetic world that recovers. The trajectory that +got there is more informative than either number. Discretisation cost more than half the signal: hard class codes give 0.166 where continuous states give 0.489, because fields need only @@ -137,20 +149,38 @@ underperform corpus-fitted statistics, 0.19 against 0.656, so in-context features beat isolated crops and corpus-specific vectors beat a language model's pooled states. +The width diagnosis localises the remaining gap somewhere the correlation +alone never pointed. Both modalities are individually rich on +photographs — vision effective rank 40, text 48 — and they agree on 15 +directions, so **each side is rich about something and they are rich +about different things.** That intersection barely moves under anything +tunable on the vision side: near-tripling the segments costs one shared +direction, and handing the pipeline annotated boxes instead of +unsupervised segmentation buys one. The text side moves it from 10 to 17 +and saturates. Enlarging the population does not help either, 23 to 19 +shared directions from 256 to 1024 scenes. The ceiling behaves like a +property of what the two corpora are about rather than of how either is +encoded, which makes corpus choice a first-class variable: the closed +world reaches 26 because its captions state exactly the world state. + ## What is not shown The closed-world result uses object states built by hand; no self-supervised recipe we tested — orbit contrastive, masked pixel reconstruction, their hybrid, masked feature prediction, or slot attention at two resolutions — produced their equivalent, each failing -by a distinct measured mechanism. Natural data sits 0.24 below the -recovery threshold. No frozen language model has been driven end to end -on natural data by a recovered correspondence, though a paired control -establishes that the interface works once the correspondence is right. - -The untried lever is the one the closed world identified as a protocol -requirement: two-sided multi-view observation. Visual Genome offers one -photograph per scene, so orbits must come from augmentation or from -corpora that carry them natively. Whether that closes 0.656 to 0.9 is -the next measurement, and the gate will answer it before any search is -run. +by a distinct measured mechanism. Natural data recovers no better than +chance. No frozen language model has been driven end to end on natural +data by a recovered correspondence, though a paired control establishes +that the interface works once the correspondence is right. + +The lever the closed world named as a protocol requirement — two-sided +multi-view observation — has now been tried on photographs and returned +nothing: four augmentation orbits leave the correlation at 0.6559 either +way. Visual Genome offers one photograph per scene, so this tests +augmentation rather than genuine re-observation, and corpora carrying +native orbits remain untested. But the width result reframes what to +look for in them. The question is no longer whether a lever closes a +scalar gap; it is whether a pair of corpora describes enough of the same +world for their shared spectrum to be wide, and that is a property to +select corpora on rather than a deficiency to engineer around. diff --git a/LAB_NOTES.md b/LAB_NOTES.md index e6aa208..5b38e5f 100644 --- a/LAB_NOTES.md +++ b/LAB_NOTES.md @@ -584,6 +584,79 @@ battery -- larger backbones and finer segmentation over the same corpus -- with three array tasks starting immediately against a queue of 810, 9,624 GPU-hours remaining on the account. +## 2026-08-01: shrinking the population is backwards, and the truth is not the optimum + +Population size was the last non-correlation lever, on the reasoning that +the hard phase is joint in correlation and size, so small blocks would +need less correlation. The sign is the other way. The +information-theoretic threshold is ρ_IT ≈ √(4 log N / N), which **rises** +as N falls: 0.29 at 256 scenes, 0.51 at 64, 0.83 at 16. + +The gate settles it directly rather than by citation. At N = 16, 24, 32, +48, 64 and 96, the deepest state a strong searcher reaches — spectral +start plus 200 random restarts, each run to a local optimum under exact +steepest descent — is deeper than the truth, in three replicates out of +three at every size. Where the truth is not the optimum, no searcher of +any cost finds it and the algorithm class stops mattering, which also +disposes of "accept exponential cost". Route D closes. + +## 2026-08-01: the correlation was never the statistic that governs recovery + +The project's central gate — correlation at truth must reach about 0.9, +price any representation in minutes without running a search — is wrong, +and a controlled truncation shows it. Project the synthetic fields that +recover to rank *r* and match blind. Correlation stays at 0.902, 0.906, +0.928 across ranks 4, 8, 16 while recovery goes 6.2%, 12.9%, **95.6%**. +A field past the supposed threshold recovers 13%. + +We misread the theory rather than being failed by it. Matching thresholds +assume exchangeable full-rank noise, where each of N²/2 entries +constrains the pairing independently; a rank-*r* shared component +supplies about *rN*, an eighth as many at 256 scenes with *r* near ten. +The gate becomes a joint condition on correlation and on the width of the +shared spectrum, measured by principal angles between the two fields' +leading eigenspaces against a scene-shuffled null. Neither suffices +alone: 18 shared directions at correlation 0.508 fails, 11 at 0.902 +fails. + +With the old gate retired the excuse for never searching natural data +went with it. Five blind trials on the best natural field recover +**0.0000 against 0.0039 chance** — refinement moves away from the truth, +which is what descent does when the truth is not the optimum. The old +verdict was right; its reasoning was not. + +Diagnosis: on photographs both modalities are individually rich — vision +effective rank 40, text 48 — and agree on 15 directions. Each side is +rich about something and they are rich about different things. The +intersection barely responds to anything on the vision side: 6 to 16 +segments costs one direction, annotated boxes buy one, third-order +moments in the set kernel raise the vision field's own rank by seven and +the shared count by one. The text side moves it from 10 to 17 and +saturates; more scenes do not help (23 to 19 from 256 to 1024). + +Free gains along the way: eigenvalue-weighted projection over a wide +basis plus 128-dimensional text vectors take the correlation from 0.656 +to 0.716 and the shared count from 10 to 16. Hubness was refuted as an +inflation hypothesis — the additive degree model is 1–5% of field +variance and removing it slightly *raises* the correlation. + +See `RANK_RESULTS.md`. + +## 2026-08-01: two operational lessons + +**Never `scancel -u` on Delta.** Clearing three failed array tasks that +way also killed four DAGFormer H200 jobs queued nineteen minutes earlier +on the same shared account. Restored from `sacct`'s recorded WorkDir and +SubmitLine, losing queue position only. Rule and recovery recipe written +into `DELTA_HPC.md`. + +**The vision extraction was CPU-bound on its own eigendecomposition,** +one dense symmetric problem of side grid² per image, costing more than +the forward pass that produced the features. Moving it to the accelerator +already holding the model took the batch time from 130 s to 1.9 s, a 68× +speedup that makes the whole backbone battery cheap rather than +overnight. + ## Current experimental gate The node-level ordering-and-basin gate is passed on VG by content-projected diff --git a/RANK_RESULTS.md b/RANK_RESULTS.md new file mode 100644 index 0000000..e1c4343 --- /dev/null +++ b/RANK_RESULTS.md @@ -0,0 +1,175 @@ +# The gate was measuring the wrong thing + +*2026-08-01. Supersedes the correlation threshold used as the project's +go/no-go statistic since the synthetic world results.* + +## The correlation does not govern recovery + +The standing rule was that the field correlation at the true pairing has to +reach about 0.9, that nothing works below it, and that any new representation +can therefore be priced in minutes without running a search. Natural data sat +at 0.656 and no recovery run was made on it, on the strength of that rule. + +**A controlled truncation refutes the rule.** Take the synthetic fields that +recover at 95.3%, project both to rank *r*, and match blind from a spectral +start with exact refinement. The correlation barely moves across the ladder; +recovery moves across its entire range. + +| Field | ρ at truth | Shared directions | Recovery (256 scenes, chance 0.4%) | +|---|---|---|---| +| synthetic, rank 4 | 0.902 | 11 | 6.2% | +| synthetic, rank 8 | 0.906 | 13 | 12.9% | +| synthetic, rank 16 | 0.928 | 18 | **95.6%** | +| synthetic, rank 32 | 0.928 | 27 | 93.8% | +| synthetic, rank 64 | 0.928 | 27 | 95.8% | +| synthetic, rank 128 | 0.929 | 27 | 96.1% | +| synthetic, full | 0.929 | 27 | 95.8% | + +A rank-8 field correlating at 0.906 — comfortably past the supposed threshold +— recovers 13%. The same field at rank 16 recovers 96%. Correlation is held +almost fixed while recovery crosses from failure to success, so the variable +that moves recovery is the width of the shared spectrum, not the strength of +the agreement. + +The theory says the same thing in advance and we misread it. Correlated-matching +thresholds are derived for exchangeable full-rank noise, where each of the +N²/2 entries carries an independent constraint on the pairing. A rank-*r* +shared component supplies about *rN*. At 256 scenes with *r* near ten that is +an eighth of what the threshold calculation assumes, so quoting the 0.9 figure +against a low-rank field compares a number to a bound derived for a different +object. + +The correction cuts against us as well as for us. With the old gate retired, +the reason for never having run a search on natural data went with it, so the +search was run: five blind trials on the best natural field, spectral start +chosen by energy over five candidates, exact refinement to convergence. +**Recovery is 0.0000 against a chance rate of 0.0039** — the spectral start +lands exactly on chance and refinement moves away from the truth, which is what +descent does when the truth is not the optimum. Natural data does not recover, +and now that is a measurement rather than an inference from a statistic that +does not govern it. + +## What the shared spectrum measures + +Each field is eigendecomposed and the principal angles between the two leading +32-dimensional eigenspaces are computed; directions with cosine above 0.7 count +as shared. A scene-shuffled null puts the chance count at 1.0 in every +condition below, so the counts are not an artefact of subspace dimension. + +| Field | ρ | vision eff. rank | text eff. rank | shared (net of null) | +|---|---|---|---|---| +| synthetic watershed (recovers 95.3%) | 0.928 | 14.5 | 11.8 | **26** | +| synthetic v0 (fails, 5.1%) | 0.508 | 31.3 | 27.9 | 18 | +| Visual Genome, previous best | 0.656 | 18.7 | 13.9 | 10 | +| Visual Genome, best today | 0.716 | 39.7 | 48.0 | 15 | + +The failing synthetic field settles the sufficiency question in the other +direction: 18 shared directions with a correlation of 0.508 also fails. +**Recovery needs both a wide shared spectrum and a strong correlation, and +neither alone predicts it.** The gate becomes a joint condition, and the honest +statement of our position is that natural data is short on both — though far +closer on width than the correlation gap alone suggested. + +## Natural data is narrow in the intersection, not in either modality + +The diagnosis is sharper than "the features are not good enough". Each side is +individually rich — vision effective rank 40, text 48 — while the two agree on +15 directions. **Each modality is rich about something, and they are rich about +different things.** + +That intersection is close to invariant under everything tunable on the vision +side, all measured at matched width: + +| Vision configuration | ρ | shared | +|---|---|---| +| 6 spectral segments | 0.692 | 15 | +| 16 spectral segments | 0.679 | 14 | +| annotated region boxes (oracle) | 0.703 | 15 | + +Handing the pipeline ground-truth boxes buys one direction over unsupervised +segmentation and nothing over using fewer segments. This confirms from a new +angle what the earlier oracle-box comparison found: **segmentation is not the +constraint on photographs.** + +The text side does move it, and saturates: + +| Text vector dimension | text eff. rank | ρ | shared | +|---|---|---|---| +| 24 | 19.0 | 0.661 | 10 | +| 48 | 31.8 | 0.692 | 15 | +| 128 | 48.0 | **0.716** | 16 | +| 256 | 53.0 | 0.700 | 17 | + +Tripling the text representation's own rank from 19 to 53 buys seven shared +directions and then stops. Enlarging the scene population does not help either +— at matched width the shared count goes 23, 20, 19 for 256, 512 and 1024 +scenes, so the ceiling is a property of what the two corpora are about rather +than of how many scenes are sampled. + +## Two smaller results, one refuted hypothesis + +**The projection improved.** Scaling each discriminant direction by its own +eigenvalue to the power 0.5 lets a wide basis be kept without the weak +directions drowning the strong ones. It dominates the unweighted projection at +every setting tested and, unlike the unweighted version, improves with width — +which is the direction that raises rank. With text vectors at 128 dimensions +this takes the field correlation from 0.656 to **0.716** and the shared count +from 10 to 16, at no cost. + +**Hubness was the wrong suspect.** The hypothesis was that the correlation is +inflated by a shared "this scene resembles everything" component carrying no +matching information. It is not: the additive row-and-column model accounts for +0.9% of the visual field's variance and 4.9% of the text field's, and removing +it *raises* the correlation slightly, from 0.656 to 0.671. The headline +statistic was honest; it was simply not the statistic that governs recovery. + +## Shrinking the problem is dead, and now by measurement + +Small blocks were the last standing non-correlation route, on the reasoning +that the hard phase is joint in correlation and population size. The reasoning +had the sign backwards. The information-theoretic threshold is +ρ_IT ≈ √(4 log N / N), which **rises** as N falls: 0.29 at 256 scenes, 0.51 at +64, 0.83 at 16. Shrinking the population raises the bar. + +The project's own gate confirms it directly. At every size tested, the best +state a strong searcher reaches — spectral start plus 200 random restarts, each +run to a local optimum under exact steepest descent — is **deeper than the +truth**, in three replicates out of three: + +| N | ρ | E(truth) | E(best found) | accuracy of best | truth deepest? | +|---|---|---|---|---|---| +| 16 | 0.705 | 2.95 | 2.37 | 0.271 | 0/3 | +| 24 | 0.703 | 3.24 | 2.29 | 0.194 | 0/3 | +| 32 | 0.679 | 3.69 | 2.38 | 0.073 | 0/3 | +| 48 | 0.706 | 3.48 | 2.65 | 0.125 | 0/3 | +| 64 | 0.700 | 3.24 | 2.15 | 0.151 | 0/3 | +| 96 | 0.698 | 3.65 | 2.57 | 0.115 | 0/3 | + +When the truth is not the optimum, no searcher of any cost finds it, and the +question of algorithm class does not arise. Accepting exponential cost buys +nothing here — this is the same conclusion tempering reached by equilibrating +below the truth, now established at sizes small enough that search cannot be +blamed. + +Combined with the earlier rank-ladder result — at natural field quality every +truncation from rank 4 to full returns chance — three of the five routes in the +candidate register are closed: reshaping the landscape, searching harder, and +shrinking the problem. Raising the shared spectrum and adding orthogonal signal +are what remain. + +## What this changes + +The target moves from a scalar to a pair. Raising the correlation from 0.656 to +0.716 was worth having, but the measurement that matters is that it came with +the shared count going from 10 to 16, and 16 is inside the interval where the +synthetic ladder crosses from 13% to 96%. + +The corpus, not the encoder, is where the width comes from. The synthetic world +has 26 shared directions because its captions state exactly the world state; +Visual Genome has 15 because a region description and a patch descriptor +overlap on roughly that many aspects of a photograph, and no amount of +segmentation quality, backbone capacity, or scene count moves it. That reframes +the next step as a question about which corpora have naturally wide overlap — +dense descriptions, product listings with photographs and full specifications, +screenshots paired with their accessibility trees — rather than a question +about better features for this one. diff --git a/artifacts/synth_v1/omit_colour.json b/artifacts/synth_v1/omit_colour.json new file mode 100644 index 0000000..61f71bc --- /dev/null +++ b/artifacts/synth_v1/omit_colour.json @@ -0,0 +1,34 @@ +{ + "data_dir": "artifacts/synth_v1", + "split": "test", + "samples": 256, + "colour_classes": 10, + "text_families": { + "count_words": [ + "a", + "three", + "four", + "two" + ], + "colour_words": [ + "red", + "orange", + "yellow", + "green", + "blue", + "purple", + "pink", + "white", + "gray", + "brown" + ], + "size_words": [ + [ + "small", + "large" + ] + ] + }, + "field_correlation_at_truth": 0.7356797393477761, + "note": "The dictionary is derived from disjoint corpora; the correlation is a diagnostic computed with hidden pairs and never used by the pipeline." +} \ No newline at end of file diff --git a/artifacts/synth_v1/omit_count.json b/artifacts/synth_v1/omit_count.json new file mode 100644 index 0000000..7a40369 --- /dev/null +++ b/artifacts/synth_v1/omit_count.json @@ -0,0 +1,34 @@ +{ + "data_dir": "artifacts/synth_v1", + "split": "test", + "samples": 256, + "colour_classes": 10, + "text_families": { + "count_words": [ + "a", + "three", + "four", + "two" + ], + "colour_words": [ + "red", + "orange", + "yellow", + "green", + "blue", + "purple", + "pink", + "white", + "gray", + "brown" + ], + "size_words": [ + [ + "small", + "large" + ] + ] + }, + "field_correlation_at_truth": 0.678442918934101, + "note": "The dictionary is derived from disjoint corpora; the correlation is a diagnostic computed with hidden pairs and never used by the pipeline." +} \ No newline at end of file diff --git a/artifacts/synth_v1/omit_none.json b/artifacts/synth_v1/omit_none.json new file mode 100644 index 0000000..7ff06f9 --- /dev/null +++ b/artifacts/synth_v1/omit_none.json @@ -0,0 +1,34 @@ +{ + "data_dir": "artifacts/synth_v1", + "split": "test", + "samples": 256, + "colour_classes": 10, + "text_families": { + "count_words": [ + "a", + "three", + "four", + "two" + ], + "colour_words": [ + "red", + "orange", + "yellow", + "green", + "blue", + "purple", + "pink", + "white", + "gray", + "brown" + ], + "size_words": [ + [ + "small", + "large" + ] + ] + }, + "field_correlation_at_truth": 0.9284809883663888, + "note": "The dictionary is derived from disjoint corpora; the correlation is a diagnostic computed with hidden pairs and never used by the pipeline." +} \ No newline at end of file diff --git a/artifacts/synth_v1/omit_size.json b/artifacts/synth_v1/omit_size.json new file mode 100644 index 0000000..d81aa00 --- /dev/null +++ b/artifacts/synth_v1/omit_size.json @@ -0,0 +1,34 @@ +{ + "data_dir": "artifacts/synth_v1", + "split": "test", + "samples": 256, + "colour_classes": 10, + "text_families": { + "count_words": [ + "a", + "three", + "four", + "two" + ], + "colour_words": [ + "red", + "orange", + "yellow", + "green", + "blue", + "purple", + "pink", + "white", + "gray", + "brown" + ], + "size_words": [ + [ + "small", + "large" + ] + ] + }, + "field_correlation_at_truth": 0.8301825436675783, + "note": "The dictionary is derived from disjoint corpora; the correlation is a diagnostic computed with hidden pairs and never used by the pipeline." +} \ No newline at end of file diff --git a/artifacts/synth_v1/omit_size_count.json b/artifacts/synth_v1/omit_size_count.json new file mode 100644 index 0000000..2eca4dc --- /dev/null +++ b/artifacts/synth_v1/omit_size_count.json @@ -0,0 +1,34 @@ +{ + "data_dir": "artifacts/synth_v1", + "split": "test", + "samples": 256, + "colour_classes": 10, + "text_families": { + "count_words": [ + "a", + "three", + "four", + "two" + ], + "colour_words": [ + "red", + "orange", + "yellow", + "green", + "blue", + "purple", + "pink", + "white", + "gray", + "brown" + ], + "size_words": [ + [ + "small", + "large" + ] + ] + }, + "field_correlation_at_truth": 0.5442660429860929, + "note": "The dictionary is derived from disjoint corpora; the correlation is a diagnostic computed with hidden pairs and never used by the pipeline." +} \ No newline at end of file diff --git a/artifacts/synth_v1/rank_causal.json b/artifacts/synth_v1/rank_causal.json new file mode 100644 index 0000000..222e469 --- /dev/null +++ b/artifacts/synth_v1/rank_causal.json @@ -0,0 +1,51 @@ +{ + "protocol": "Both fields truncated to the same rank, then matched blind from a spectral start with exact refinement. Hidden pairing generated per trial and read only for scoring.", + "label": "synthetic-watershed", + "size": 256, + "chance": 0.00390625, + "rows": [ + { + "rank": 4, + "correlation": 0.9016677754313268, + "recovery": 0.0625, + "visual_effective_rank": 44.86571084196529 + }, + { + "rank": 8, + "correlation": 0.9058447537972105, + "recovery": 0.12890625, + "visual_effective_rank": 71.3420544398913 + }, + { + "rank": 16, + "correlation": 0.9281446653770646, + "recovery": 0.9557291666666666, + "visual_effective_rank": 95.72603497323556 + }, + { + "rank": 32, + "correlation": 0.9282073730691501, + "recovery": 0.9375, + "visual_effective_rank": 109.47942916160699 + }, + { + "rank": 64, + "correlation": 0.9284208632987705, + "recovery": 0.9583333333333334, + "visual_effective_rank": 106.31167521180129 + }, + { + "rank": 128, + "correlation": 0.9284791170404748, + "recovery": 0.9609375, + "visual_effective_rank": 106.04029186749484 + }, + { + "rank": 256, + "correlation": 0.928480988366389, + "recovery": 0.9583333333333334, + "visual_effective_rank": 106.03700034539153 + } + ], + "reading": "Recovery collapsing at low rank while the correlation stays above the recovery threshold shows the headline correlation is not sufficient and the shared spectrum's width is the binding constraint." +} \ No newline at end of file diff --git a/artifacts/vg_5k/field_anatomy.json b/artifacts/vg_5k/field_anatomy.json new file mode 100644 index 0000000..740136c --- /dev/null +++ b/artifacts/vg_5k/field_anatomy.json @@ -0,0 +1,40 @@ +{ + "protocol": "Both fields split into an additive row/column-mean model and its residual, then correlated component by component. Hidden pairs are read only to align the two fields.", + "label": "natural", + "size": 256, + "rows": [ + { + "component": "raw field", + "correlation": 0.6558829559111284 + }, + { + "component": "degree model only", + "correlation": 0.22252881188815218 + }, + { + "component": "residual after degree", + "correlation": 0.6709906297296612 + }, + { + "component": "residual after top-1 eigen", + "correlation": 0.4579725907578625 + }, + { + "component": "residual after top-2 eigen", + "correlation": 0.4555001418827857 + }, + { + "component": "residual after top-5 eigen", + "correlation": 0.3413229646188751 + }, + { + "component": "residual after top-10 eigen", + "correlation": 0.07969017846523709 + } + ], + "visual_degree_variance_share": 0.009036045331312099, + "text_degree_variance_share": 0.04921092158161093, + "it_threshold_at_size": 0.29435250562886867, + "residual_above_it_threshold": true, + "reading": "If the residual correlation is far below the raw correlation, the headline statistic is inflated by nuisance structure that carries no matching information, and the residual is the number that should be compared against the recovery threshold." +} \ No newline at end of file diff --git a/artifacts/vg_5k/field_anatomy_w64.json b/artifacts/vg_5k/field_anatomy_w64.json new file mode 100644 index 0000000..1e023eb --- /dev/null +++ b/artifacts/vg_5k/field_anatomy_w64.json @@ -0,0 +1,40 @@ +{ + "protocol": "Both fields split into an additive row/column-mean model and its residual, then correlated component by component. Hidden pairs are read only to align the two fields.", + "label": "natural-weighted64", + "size": 256, + "rows": [ + { + "component": "raw field", + "correlation": 0.6922159706308013 + }, + { + "component": "degree model only", + "correlation": 0.09663282041737035 + }, + { + "component": "residual after degree", + "correlation": 0.7052697077805243 + }, + { + "component": "residual after top-1 eigen", + "correlation": 0.542199919037366 + }, + { + "component": "residual after top-2 eigen", + "correlation": 0.37481036694327863 + }, + { + "component": "residual after top-5 eigen", + "correlation": 0.3855465012648901 + }, + { + "component": "residual after top-10 eigen", + "correlation": 0.18341225740386502 + } + ], + "visual_degree_variance_share": 0.01546868013212138, + "text_degree_variance_share": 0.026806936723177938, + "it_threshold_at_size": 0.29435250562886867, + "residual_above_it_threshold": true, + "reading": "If the residual correlation is far below the raw correlation, the headline statistic is inflated by nuisance structure that carries no matching information, and the residual is the number that should be compared against the recovery threshold." +} \ No newline at end of file diff --git a/artifacts/vg_5k/natural_pipeline_oracle.json b/artifacts/vg_5k/natural_pipeline_oracle.json new file mode 100644 index 0000000..c7f9025 --- /dev/null +++ b/artifacts/vg_5k/natural_pipeline_oracle.json @@ -0,0 +1,9 @@ +{ + "protocol": "Continuous states, content projection fitted per modality on scenes outside the evaluated set. Hidden pairs are read only for the correlation.", + "samples": 256, + "views": 1, + "kept_directions": 64, + "weight_power": 0.5, + "field_correlation_at_truth": 0.702748890720233, + "recovery_threshold": 0.9 +} \ No newline at end of file diff --git a/artifacts/vg_5k/natural_pipeline_seg16.json b/artifacts/vg_5k/natural_pipeline_seg16.json new file mode 100644 index 0000000..0ef4894 --- /dev/null +++ b/artifacts/vg_5k/natural_pipeline_seg16.json @@ -0,0 +1,9 @@ +{ + "protocol": "Continuous states, content projection fitted per modality on scenes outside the evaluated set. Hidden pairs are read only for the correlation.", + "samples": 256, + "views": 1, + "kept_directions": 64, + "weight_power": 0.5, + "field_correlation_at_truth": 0.6794444345566248, + "recovery_threshold": 0.9 +} \ No newline at end of file diff --git a/artifacts/vg_5k/natural_pipeline_w64.json b/artifacts/vg_5k/natural_pipeline_w64.json new file mode 100644 index 0000000..9e57e09 --- /dev/null +++ b/artifacts/vg_5k/natural_pipeline_w64.json @@ -0,0 +1,9 @@ +{ + "protocol": "Continuous states, content projection fitted per modality on scenes outside the evaluated set. Hidden pairs are read only for the correlation.", + "samples": 256, + "views": 1, + "kept_directions": 64, + "weight_power": 0.5, + "field_correlation_at_truth": 0.6922159706308013, + "recovery_threshold": 0.9 +} \ No newline at end of file diff --git a/artifacts/vg_5k/np_deg3_12.json b/artifacts/vg_5k/np_deg3_12.json new file mode 100644 index 0000000..4b06fe2 --- /dev/null +++ b/artifacts/vg_5k/np_deg3_12.json @@ -0,0 +1,10 @@ +{ + "protocol": "Continuous states, content projection fitted per modality on scenes outside the evaluated set. Hidden pairs are read only for the correlation.", + "samples": 256, + "views": 1, + "kept_directions": 64, + "weight_power": 0.5, + "moment_degree": 3, + "field_correlation_at_truth": 0.7164530320598383, + "recovery_threshold": 0.9 +} \ No newline at end of file diff --git a/artifacts/vg_5k/np_deg3_20.json b/artifacts/vg_5k/np_deg3_20.json new file mode 100644 index 0000000..ba28d29 --- /dev/null +++ b/artifacts/vg_5k/np_deg3_20.json @@ -0,0 +1,10 @@ +{ + "protocol": "Continuous states, content projection fitted per modality on scenes outside the evaluated set. Hidden pairs are read only for the correlation.", + "samples": 256, + "views": 1, + "kept_directions": 64, + "weight_power": 0.5, + "moment_degree": 3, + "field_correlation_at_truth": 0.7171326909156863, + "recovery_threshold": 0.9 +} \ No newline at end of file diff --git a/artifacts/vg_5k/np_deg3_8.json b/artifacts/vg_5k/np_deg3_8.json new file mode 100644 index 0000000..1c87384 --- /dev/null +++ b/artifacts/vg_5k/np_deg3_8.json @@ -0,0 +1,10 @@ +{ + "protocol": "Continuous states, content projection fitted per modality on scenes outside the evaluated set. Hidden pairs are read only for the correlation.", + "samples": 256, + "views": 1, + "kept_directions": 64, + "weight_power": 0.5, + "moment_degree": 3, + "field_correlation_at_truth": 0.717450184813228, + "recovery_threshold": 0.9 +} \ No newline at end of file diff --git a/artifacts/vg_5k/np_n1024.json b/artifacts/vg_5k/np_n1024.json new file mode 100644 index 0000000..6e69a30 --- /dev/null +++ b/artifacts/vg_5k/np_n1024.json @@ -0,0 +1,9 @@ +{ + "protocol": "Continuous states, content projection fitted per modality on scenes outside the evaluated set. Hidden pairs are read only for the correlation.", + "samples": 1024, + "views": 1, + "kept_directions": 64, + "weight_power": 0.5, + "field_correlation_at_truth": 0.6922444253804424, + "recovery_threshold": 0.9 +} \ No newline at end of file diff --git a/artifacts/vg_5k/np_n256.json b/artifacts/vg_5k/np_n256.json new file mode 100644 index 0000000..1faa359 --- /dev/null +++ b/artifacts/vg_5k/np_n256.json @@ -0,0 +1,9 @@ +{ + "protocol": "Continuous states, content projection fitted per modality on scenes outside the evaluated set. Hidden pairs are read only for the correlation.", + "samples": 256, + "views": 1, + "kept_directions": 64, + "weight_power": 0.5, + "field_correlation_at_truth": 0.7157944505313559, + "recovery_threshold": 0.9 +} \ No newline at end of file diff --git a/artifacts/vg_5k/np_n512.json b/artifacts/vg_5k/np_n512.json new file mode 100644 index 0000000..578861d --- /dev/null +++ b/artifacts/vg_5k/np_n512.json @@ -0,0 +1,9 @@ +{ + "protocol": "Continuous states, content projection fitted per modality on scenes outside the evaluated set. Hidden pairs are read only for the correlation.", + "samples": 512, + "views": 1, + "kept_directions": 64, + "weight_power": 0.5, + "field_correlation_at_truth": 0.720884905640894, + "recovery_threshold": 0.9 +} \ No newline at end of file diff --git a/artifacts/vg_5k/np_wv128.json b/artifacts/vg_5k/np_wv128.json new file mode 100644 index 0000000..1faa359 --- /dev/null +++ b/artifacts/vg_5k/np_wv128.json @@ -0,0 +1,9 @@ +{ + "protocol": "Continuous states, content projection fitted per modality on scenes outside the evaluated set. Hidden pairs are read only for the correlation.", + "samples": 256, + "views": 1, + "kept_directions": 64, + "weight_power": 0.5, + "field_correlation_at_truth": 0.7157944505313559, + "recovery_threshold": 0.9 +} \ No newline at end of file diff --git a/artifacts/vg_5k/np_wv24.json b/artifacts/vg_5k/np_wv24.json new file mode 100644 index 0000000..8f199ba --- /dev/null +++ b/artifacts/vg_5k/np_wv24.json @@ -0,0 +1,9 @@ +{ + "protocol": "Continuous states, content projection fitted per modality on scenes outside the evaluated set. Hidden pairs are read only for the correlation.", + "samples": 256, + "views": 1, + "kept_directions": 64, + "weight_power": 0.5, + "field_correlation_at_truth": 0.6614852336553844, + "recovery_threshold": 0.9 +} \ No newline at end of file diff --git a/artifacts/vg_5k/np_wv256.json b/artifacts/vg_5k/np_wv256.json new file mode 100644 index 0000000..617e698 --- /dev/null +++ b/artifacts/vg_5k/np_wv256.json @@ -0,0 +1,9 @@ +{ + "protocol": "Continuous states, content projection fitted per modality on scenes outside the evaluated set. Hidden pairs are read only for the correlation.", + "samples": 256, + "views": 1, + "kept_directions": 64, + "weight_power": 0.5, + "field_correlation_at_truth": 0.7000821325988256, + "recovery_threshold": 0.9 +} \ No newline at end of file diff --git a/artifacts/vg_5k/np_wv48.json b/artifacts/vg_5k/np_wv48.json new file mode 100644 index 0000000..9e57e09 --- /dev/null +++ b/artifacts/vg_5k/np_wv48.json @@ -0,0 +1,9 @@ +{ + "protocol": "Continuous states, content projection fitted per modality on scenes outside the evaluated set. Hidden pairs are read only for the correlation.", + "samples": 256, + "views": 1, + "kept_directions": 64, + "weight_power": 0.5, + "field_correlation_at_truth": 0.6922159706308013, + "recovery_threshold": 0.9 +} \ No newline at end of file diff --git a/artifacts/vg_5k/projection_sweep.json b/artifacts/vg_5k/projection_sweep.json new file mode 100644 index 0000000..8881383 --- /dev/null +++ b/artifacts/vg_5k/projection_sweep.json @@ -0,0 +1,265 @@ +{ + "protocol": "Projection fitted per modality on scenes outside the evaluated set; hidden pairs read only for the correlation. Baseline is linear, keep=8, shrinkage=0.05.", + "evaluated": 256, + "rows": [ + { + "variant": "linear", + "keep": 4, + "shrinkage": 0.01, + "correlation": 0.6453137317062534 + }, + { + "variant": "linear", + "keep": 8, + "shrinkage": 0.01, + "correlation": 0.6495421074385569 + }, + { + "variant": "linear", + "keep": 16, + "shrinkage": 0.01, + "correlation": 0.6470767138606525 + }, + { + "variant": "linear", + "keep": 32, + "shrinkage": 0.01, + "correlation": 0.6227407963928071 + }, + { + "variant": "linear", + "keep": 64, + "shrinkage": 0.01, + "correlation": 0.6142722294788403 + }, + { + "variant": "linear", + "keep": 4, + "shrinkage": 0.05, + "correlation": 0.65249653291618 + }, + { + "variant": "linear", + "keep": 8, + "shrinkage": 0.05, + "correlation": 0.6558829559111284 + }, + { + "variant": "linear", + "keep": 16, + "shrinkage": 0.05, + "correlation": 0.6519028422068834 + }, + { + "variant": "linear", + "keep": 32, + "shrinkage": 0.05, + "correlation": 0.6274081754272933 + }, + { + "variant": "linear", + "keep": 64, + "shrinkage": 0.05, + "correlation": 0.619760201230621 + }, + { + "variant": "linear", + "keep": 4, + "shrinkage": 0.2, + "correlation": 0.6607265117297335 + }, + { + "variant": "linear", + "keep": 8, + "shrinkage": 0.2, + "correlation": 0.6582353562445471 + }, + { + "variant": "linear", + "keep": 16, + "shrinkage": 0.2, + "correlation": 0.6513068306415126 + }, + { + "variant": "linear", + "keep": 32, + "shrinkage": 0.2, + "correlation": 0.6292167478629883 + }, + { + "variant": "linear", + "keep": 64, + "shrinkage": 0.2, + "correlation": 0.6230802049271942 + }, + { + "variant": "linear", + "keep": 4, + "shrinkage": 1.0, + "correlation": 0.6578716141349961 + }, + { + "variant": "linear", + "keep": 8, + "shrinkage": 1.0, + "correlation": 0.6405385423647052 + }, + { + "variant": "linear", + "keep": 16, + "shrinkage": 1.0, + "correlation": 0.6280429181450786 + }, + { + "variant": "linear", + "keep": 32, + "shrinkage": 1.0, + "correlation": 0.6142054425413382 + }, + { + "variant": "linear", + "keep": 64, + "shrinkage": 1.0, + "correlation": 0.6106156014213242 + }, + { + "variant": "weighted^0.5", + "keep": 8, + "shrinkage": 0.05, + "correlation": 0.6698246762363366 + }, + { + "variant": "weighted^1.0", + "keep": 8, + "shrinkage": 0.05, + "correlation": 0.665060874094268 + }, + { + "variant": "weighted^0.5", + "keep": 16, + "shrinkage": 0.05, + "correlation": 0.6826263927928936 + }, + { + "variant": "weighted^1.0", + "keep": 16, + "shrinkage": 0.05, + "correlation": 0.6721336214645782 + }, + { + "variant": "weighted^0.5", + "keep": 32, + "shrinkage": 0.05, + "correlation": 0.687375151713245 + }, + { + "variant": "weighted^1.0", + "keep": 32, + "shrinkage": 0.05, + "correlation": 0.6753247222620263 + }, + { + "variant": "weighted^0.5", + "keep": 64, + "shrinkage": 0.05, + "correlation": 0.6922159706308013 + }, + { + "variant": "weighted^1.0", + "keep": 64, + "shrinkage": 0.05, + "correlation": 0.6767939950719634 + }, + { + "variant": "weighted^0.5", + "keep": 8, + "shrinkage": 0.2, + "correlation": 0.672827939323971 + }, + { + "variant": "weighted^1.0", + "keep": 8, + "shrinkage": 0.2, + "correlation": 0.6691737665681982 + }, + { + "variant": "weighted^0.5", + "keep": 16, + "shrinkage": 0.2, + "correlation": 0.6834609415667857 + }, + { + "variant": "weighted^1.0", + "keep": 16, + "shrinkage": 0.2, + "correlation": 0.675193020267635 + }, + { + "variant": "weighted^0.5", + "keep": 32, + "shrinkage": 0.2, + "correlation": 0.6883825134950975 + }, + { + "variant": "weighted^1.0", + "keep": 32, + "shrinkage": 0.2, + "correlation": 0.6780073690253073 + }, + { + "variant": "weighted^0.5", + "keep": 64, + "shrinkage": 0.2, + "correlation": 0.6918038515455318 + }, + { + "variant": "weighted^1.0", + "keep": 64, + "shrinkage": 0.2, + "correlation": 0.6788283392518062 + }, + { + "variant": "kernel g=0.25", + "keep": 8, + "shrinkage": 0.05, + "correlation": 0.6239363564555721 + }, + { + "variant": "kernel g=0.25", + "keep": 16, + "shrinkage": 0.05, + "correlation": 0.6199483306751332 + }, + { + "variant": "kernel g=0.25", + "keep": 32, + "shrinkage": 0.05, + "correlation": 0.6043865046351463 + }, + { + "variant": "kernel g=1.0", + "keep": 8, + "shrinkage": 0.05, + "correlation": 0.5588913870951268 + }, + { + "variant": "kernel g=1.0", + "keep": 16, + "shrinkage": 0.05, + "correlation": 0.5472099552797389 + }, + { + "variant": "kernel g=1.0", + "keep": 32, + "shrinkage": 0.05, + "correlation": 0.5251472992094228 + } + ], + "best": { + "variant": "weighted^0.5", + "keep": 64, + "shrinkage": 0.05, + "correlation": 0.6922159706308013 + }, + "recovery_threshold": 0.9 +} \ No newline at end of file diff --git a/artifacts/vg_5k/shared_rank.json b/artifacts/vg_5k/shared_rank.json new file mode 100644 index 0000000..538da2d --- /dev/null +++ b/artifacts/vg_5k/shared_rank.json @@ -0,0 +1,343 @@ +{ + "protocol": "Effective rank from the participation ratio of each field's spectrum; shared directions from principal angles between the two leading eigenspaces. No pairing enters beyond the alignment the fields already carry.", + "width": 32, + "rows": [ + { + "label": "synth-watershed-RECOVERS", + "size": 256, + "correlation": 0.9284809883663888, + "visual_effective_rank": 14.50854049599196, + "text_effective_rank": 11.832081486516831, + "shared_directions_cos_gt_0.7": 27, + "principal_cosines_top8": [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.999, + 0.999, + 0.999 + ], + "spectrum_profile": [ + { + "index": 0, + "best_text_overlap": 0.9988184979219944 + }, + { + "index": 1, + "best_text_overlap": 0.9038254548424662 + }, + { + "index": 2, + "best_text_overlap": 0.7738637793117231 + }, + { + "index": 3, + "best_text_overlap": 0.7898924505967239 + }, + { + "index": 4, + "best_text_overlap": 0.9633572761985347 + }, + { + "index": 5, + "best_text_overlap": 0.9667594083538564 + }, + { + "index": 6, + "best_text_overlap": 0.8581068112833727 + }, + { + "index": 7, + "best_text_overlap": 0.7720408457258224 + }, + { + "index": 8, + "best_text_overlap": 0.9451869286586472 + }, + { + "index": 9, + "best_text_overlap": 0.9346387523699597 + }, + { + "index": 10, + "best_text_overlap": 0.9744620235608439 + }, + { + "index": 11, + "best_text_overlap": 0.8287855141725994 + }, + { + "index": 12, + "best_text_overlap": 0.8450929971524885 + }, + { + "index": 13, + "best_text_overlap": 0.7475087759782241 + }, + { + "index": 14, + "best_text_overlap": 0.5363062788775304 + }, + { + "index": 15, + "best_text_overlap": 0.5851587348701632 + } + ] + }, + { + "label": "synth-v0-fails", + "size": 256, + "correlation": 0.507656498620682, + "visual_effective_rank": 31.301831479053796, + "text_effective_rank": 27.887166453738427, + "shared_directions_cos_gt_0.7": 19, + "principal_cosines_top8": [ + 0.999, + 0.99, + 0.986, + 0.983, + 0.981, + 0.979, + 0.975, + 0.971 + ], + "spectrum_profile": [ + { + "index": 0, + "best_text_overlap": 0.9711463046324016 + }, + { + "index": 1, + "best_text_overlap": 0.5862398136648852 + }, + { + "index": 2, + "best_text_overlap": 0.40411629072393973 + }, + { + "index": 3, + "best_text_overlap": 0.3782147186014658 + }, + { + "index": 4, + "best_text_overlap": 0.49623067040441227 + }, + { + "index": 5, + "best_text_overlap": 0.38858486646786783 + }, + { + "index": 6, + "best_text_overlap": 0.4204576749257697 + }, + { + "index": 7, + "best_text_overlap": 0.4531244365205851 + }, + { + "index": 8, + "best_text_overlap": 0.5071647502947677 + }, + { + "index": 9, + "best_text_overlap": 0.43985083861605223 + }, + { + "index": 10, + "best_text_overlap": 0.4285920808891206 + }, + { + "index": 11, + "best_text_overlap": 0.416942537981572 + }, + { + "index": 12, + "best_text_overlap": 0.36849147041514496 + }, + { + "index": 13, + "best_text_overlap": 0.3540029990986704 + }, + { + "index": 14, + "best_text_overlap": 0.41250502299139413 + }, + { + "index": 15, + "best_text_overlap": 0.34226328782237314 + } + ] + }, + { + "label": "natural-keep8", + "size": 256, + "correlation": 0.6558829559111284, + "visual_effective_rank": 18.735297664859708, + "text_effective_rank": 13.915532152821992, + "shared_directions_cos_gt_0.7": 11, + "principal_cosines_top8": [ + 0.993, + 0.971, + 0.954, + 0.938, + 0.922, + 0.868, + 0.845, + 0.826 + ], + "spectrum_profile": [ + { + "index": 0, + "best_text_overlap": 0.8308561390616345 + }, + { + "index": 1, + "best_text_overlap": 0.6425639957390433 + }, + { + "index": 2, + "best_text_overlap": 0.6401247564780709 + }, + { + "index": 3, + "best_text_overlap": 0.5245819731835861 + }, + { + "index": 4, + "best_text_overlap": 0.5324575387488661 + }, + { + "index": 5, + "best_text_overlap": 0.504776020548726 + }, + { + "index": 6, + "best_text_overlap": 0.5017207831704809 + }, + { + "index": 7, + "best_text_overlap": 0.26453910202691044 + }, + { + "index": 8, + "best_text_overlap": 0.3831674979777484 + }, + { + "index": 9, + "best_text_overlap": 0.34779386653289623 + }, + { + "index": 10, + "best_text_overlap": 0.2366745915766192 + }, + { + "index": 11, + "best_text_overlap": 0.1973585194151871 + }, + { + "index": 12, + "best_text_overlap": 0.21439811143331272 + }, + { + "index": 13, + "best_text_overlap": 0.2925437897693861 + }, + { + "index": 14, + "best_text_overlap": 0.09814723704154468 + }, + { + "index": 15, + "best_text_overlap": 0.20755663736183314 + } + ] + }, + { + "label": "natural-weighted64", + "size": 256, + "correlation": 0.6922159706308013, + "visual_effective_rank": 39.69132496762981, + "text_effective_rank": 31.7662539087451, + "shared_directions_cos_gt_0.7": 15, + "principal_cosines_top8": [ + 0.989, + 0.978, + 0.956, + 0.938, + 0.916, + 0.892, + 0.881, + 0.87 + ], + "spectrum_profile": [ + { + "index": 0, + "best_text_overlap": 0.7532219338133863 + }, + { + "index": 1, + "best_text_overlap": 0.618182152081914 + }, + { + "index": 2, + "best_text_overlap": 0.6411572047678387 + }, + { + "index": 3, + "best_text_overlap": 0.5609614131227192 + }, + { + "index": 4, + "best_text_overlap": 0.5786521313263311 + }, + { + "index": 5, + "best_text_overlap": 0.6212040366073345 + }, + { + "index": 6, + "best_text_overlap": 0.6014409101136969 + }, + { + "index": 7, + "best_text_overlap": 0.3241154791286174 + }, + { + "index": 8, + "best_text_overlap": 0.33772536144199544 + }, + { + "index": 9, + "best_text_overlap": 0.46886651280081304 + }, + { + "index": 10, + "best_text_overlap": 0.339879585332424 + }, + { + "index": 11, + "best_text_overlap": 0.2697704959742108 + }, + { + "index": 12, + "best_text_overlap": 0.3089464131279491 + }, + { + "index": 13, + "best_text_overlap": 0.34824951852897246 + }, + { + "index": 14, + "best_text_overlap": 0.26546032832701905 + }, + { + "index": 15, + "best_text_overlap": 0.17108445671033506 + } + ] + } + ], + "reading": "The narrow modality is the one to upgrade. A shared count far below both effective ranks means each side is rich but they are rich about different things." +} \ No newline at end of file diff --git a/artifacts/vg_5k/shared_rank_degree.json b/artifacts/vg_5k/shared_rank_degree.json new file mode 100644 index 0000000..7940d64 --- /dev/null +++ b/artifacts/vg_5k/shared_rank_degree.json @@ -0,0 +1,351 @@ +{ + "protocol": "Effective rank from the participation ratio of each field's spectrum; shared directions from principal angles between the two leading eigenspaces. No pairing enters beyond the alignment the fields already carry.", + "width": 32, + "rows": [ + { + "label": "degree2-best", + "size": 256, + "correlation": 0.7157944505313559, + "visual_effective_rank": 39.69132496762981, + "text_effective_rank": 47.98422953390366, + "shared_directions_cos_gt_0.7": 16, + "shuffled_null": 1.0, + "shared_above_null": 15.0, + "principal_cosines_top8": [ + 0.987, + 0.978, + 0.96, + 0.941, + 0.934, + 0.922, + 0.899, + 0.89 + ], + "spectrum_profile": [ + { + "index": 0, + "best_text_overlap": 0.8406807387214822 + }, + { + "index": 1, + "best_text_overlap": 0.6748753041856854 + }, + { + "index": 2, + "best_text_overlap": 0.7174150319587717 + }, + { + "index": 3, + "best_text_overlap": 0.4742164704027202 + }, + { + "index": 4, + "best_text_overlap": 0.7105187496542062 + }, + { + "index": 5, + "best_text_overlap": 0.5435228035336661 + }, + { + "index": 6, + "best_text_overlap": 0.5173316630294161 + }, + { + "index": 7, + "best_text_overlap": 0.3720956303455928 + }, + { + "index": 8, + "best_text_overlap": 0.4031225977235423 + }, + { + "index": 9, + "best_text_overlap": 0.5909531802492844 + }, + { + "index": 10, + "best_text_overlap": 0.373881786434362 + }, + { + "index": 11, + "best_text_overlap": 0.2816693575587397 + }, + { + "index": 12, + "best_text_overlap": 0.21822240451244762 + }, + { + "index": 13, + "best_text_overlap": 0.3929212823566225 + }, + { + "index": 14, + "best_text_overlap": 0.4024276537497661 + }, + { + "index": 15, + "best_text_overlap": 0.2659834432053085 + } + ] + }, + { + "label": "degree3-w8", + "size": 256, + "correlation": 0.717450184813228, + "visual_effective_rank": 40.67303880139185, + "text_effective_rank": 47.11594894154097, + "shared_directions_cos_gt_0.7": 16, + "shuffled_null": 1.0, + "shared_above_null": 15.0, + "principal_cosines_top8": [ + 0.988, + 0.978, + 0.961, + 0.944, + 0.936, + 0.924, + 0.901, + 0.886 + ], + "spectrum_profile": [ + { + "index": 0, + "best_text_overlap": 0.8348375430905127 + }, + { + "index": 1, + "best_text_overlap": 0.6837720540848308 + }, + { + "index": 2, + "best_text_overlap": 0.7155802705177406 + }, + { + "index": 3, + "best_text_overlap": 0.5058562446867387 + }, + { + "index": 4, + "best_text_overlap": 0.7663355948751955 + }, + { + "index": 5, + "best_text_overlap": 0.516380442780415 + }, + { + "index": 6, + "best_text_overlap": 0.554955451295094 + }, + { + "index": 7, + "best_text_overlap": 0.3516466638329569 + }, + { + "index": 8, + "best_text_overlap": 0.5186308948947995 + }, + { + "index": 9, + "best_text_overlap": 0.5015555070809197 + }, + { + "index": 10, + "best_text_overlap": 0.3700340337533245 + }, + { + "index": 11, + "best_text_overlap": 0.34258025998594904 + }, + { + "index": 12, + "best_text_overlap": 0.256732226634933 + }, + { + "index": 13, + "best_text_overlap": 0.4359760369195398 + }, + { + "index": 14, + "best_text_overlap": 0.32969959689961453 + }, + { + "index": 15, + "best_text_overlap": 0.2530743072716043 + } + ] + }, + { + "label": "degree3-w12", + "size": 256, + "correlation": 0.7164530320598383, + "visual_effective_rank": 42.77448656306514, + "text_effective_rank": 47.90052292048341, + "shared_directions_cos_gt_0.7": 16, + "shuffled_null": 1.0, + "shared_above_null": 15.0, + "principal_cosines_top8": [ + 0.988, + 0.978, + 0.961, + 0.946, + 0.937, + 0.929, + 0.91, + 0.889 + ], + "spectrum_profile": [ + { + "index": 0, + "best_text_overlap": 0.8277435616658919 + }, + { + "index": 1, + "best_text_overlap": 0.6779609268778297 + }, + { + "index": 2, + "best_text_overlap": 0.7219010238018317 + }, + { + "index": 3, + "best_text_overlap": 0.5055290038432999 + }, + { + "index": 4, + "best_text_overlap": 0.7718721615385675 + }, + { + "index": 5, + "best_text_overlap": 0.5256873621550153 + }, + { + "index": 6, + "best_text_overlap": 0.5367136087431718 + }, + { + "index": 7, + "best_text_overlap": 0.36680812886106473 + }, + { + "index": 8, + "best_text_overlap": 0.5387792531732701 + }, + { + "index": 9, + "best_text_overlap": 0.5550666365558536 + }, + { + "index": 10, + "best_text_overlap": 0.2891791715134958 + }, + { + "index": 11, + "best_text_overlap": 0.2984192540709536 + }, + { + "index": 12, + "best_text_overlap": 0.28170926274492825 + }, + { + "index": 13, + "best_text_overlap": 0.4002691992367436 + }, + { + "index": 14, + "best_text_overlap": 0.31596068490447127 + }, + { + "index": 15, + "best_text_overlap": 0.26924174791563826 + } + ] + }, + { + "label": "degree3-w20", + "size": 256, + "correlation": 0.7171326909156863, + "visual_effective_rank": 46.709791939402166, + "text_effective_rank": 50.02596883001826, + "shared_directions_cos_gt_0.7": 17, + "shuffled_null": 1.0, + "shared_above_null": 16.0, + "principal_cosines_top8": [ + 0.988, + 0.978, + 0.962, + 0.945, + 0.939, + 0.931, + 0.911, + 0.894 + ], + "spectrum_profile": [ + { + "index": 0, + "best_text_overlap": 0.8231142857010968 + }, + { + "index": 1, + "best_text_overlap": 0.6748733981742054 + }, + { + "index": 2, + "best_text_overlap": 0.7180139533448364 + }, + { + "index": 3, + "best_text_overlap": 0.5179878165995193 + }, + { + "index": 4, + "best_text_overlap": 0.7736927465190133 + }, + { + "index": 5, + "best_text_overlap": 0.5414967531898147 + }, + { + "index": 6, + "best_text_overlap": 0.5416571648246705 + }, + { + "index": 7, + "best_text_overlap": 0.3600170379372908 + }, + { + "index": 8, + "best_text_overlap": 0.5145850503977925 + }, + { + "index": 9, + "best_text_overlap": 0.5683887253705958 + }, + { + "index": 10, + "best_text_overlap": 0.30798470685067697 + }, + { + "index": 11, + "best_text_overlap": 0.31660436348552895 + }, + { + "index": 12, + "best_text_overlap": 0.2381351557071557 + }, + { + "index": 13, + "best_text_overlap": 0.44012504821073994 + }, + { + "index": 14, + "best_text_overlap": 0.35872551158185606 + }, + { + "index": 15, + "best_text_overlap": 0.2717148785929642 + } + ] + } + ], + "reading": "The narrow modality is the one to upgrade. A shared count far below both effective ranks means each side is rich but they are rich about different things." +} \ No newline at end of file diff --git a/artifacts/vg_5k/shared_rank_final.json b/artifacts/vg_5k/shared_rank_final.json new file mode 100644 index 0000000..3ade344 --- /dev/null +++ b/artifacts/vg_5k/shared_rank_final.json @@ -0,0 +1,351 @@ +{ + "protocol": "Effective rank from the participation ratio of each field's spectrum; shared directions from principal angles between the two leading eigenspaces. No pairing enters beyond the alignment the fields already carry.", + "width": 32, + "rows": [ + { + "label": "synth-RECOVERS-95pct", + "size": 256, + "correlation": 0.9284809883663888, + "visual_effective_rank": 14.50854049599196, + "text_effective_rank": 11.832081486516831, + "shared_directions_cos_gt_0.7": 27, + "shuffled_null": 1.0, + "shared_above_null": 26.0, + "principal_cosines_top8": [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.999, + 0.999, + 0.999 + ], + "spectrum_profile": [ + { + "index": 0, + "best_text_overlap": 0.9988184979219944 + }, + { + "index": 1, + "best_text_overlap": 0.9038254548424662 + }, + { + "index": 2, + "best_text_overlap": 0.7738637793117231 + }, + { + "index": 3, + "best_text_overlap": 0.7898924505967239 + }, + { + "index": 4, + "best_text_overlap": 0.9633572761985347 + }, + { + "index": 5, + "best_text_overlap": 0.9667594083538564 + }, + { + "index": 6, + "best_text_overlap": 0.8581068112833727 + }, + { + "index": 7, + "best_text_overlap": 0.7720408457258224 + }, + { + "index": 8, + "best_text_overlap": 0.9451869286586472 + }, + { + "index": 9, + "best_text_overlap": 0.9346387523699597 + }, + { + "index": 10, + "best_text_overlap": 0.9744620235608439 + }, + { + "index": 11, + "best_text_overlap": 0.8287855141725994 + }, + { + "index": 12, + "best_text_overlap": 0.8450929971524885 + }, + { + "index": 13, + "best_text_overlap": 0.7475087759782241 + }, + { + "index": 14, + "best_text_overlap": 0.5363062788775304 + }, + { + "index": 15, + "best_text_overlap": 0.5851587348701632 + } + ] + }, + { + "label": "synth-v0-fails-5pct", + "size": 256, + "correlation": 0.507656498620682, + "visual_effective_rank": 31.301831479053796, + "text_effective_rank": 27.887166453738427, + "shared_directions_cos_gt_0.7": 19, + "shuffled_null": 1.0, + "shared_above_null": 18.0, + "principal_cosines_top8": [ + 0.999, + 0.99, + 0.986, + 0.983, + 0.981, + 0.979, + 0.975, + 0.971 + ], + "spectrum_profile": [ + { + "index": 0, + "best_text_overlap": 0.9711463046324016 + }, + { + "index": 1, + "best_text_overlap": 0.5862398136648852 + }, + { + "index": 2, + "best_text_overlap": 0.40411629072393973 + }, + { + "index": 3, + "best_text_overlap": 0.3782147186014658 + }, + { + "index": 4, + "best_text_overlap": 0.49623067040441227 + }, + { + "index": 5, + "best_text_overlap": 0.38858486646786783 + }, + { + "index": 6, + "best_text_overlap": 0.4204576749257697 + }, + { + "index": 7, + "best_text_overlap": 0.4531244365205851 + }, + { + "index": 8, + "best_text_overlap": 0.5071647502947677 + }, + { + "index": 9, + "best_text_overlap": 0.43985083861605223 + }, + { + "index": 10, + "best_text_overlap": 0.4285920808891206 + }, + { + "index": 11, + "best_text_overlap": 0.416942537981572 + }, + { + "index": 12, + "best_text_overlap": 0.36849147041514496 + }, + { + "index": 13, + "best_text_overlap": 0.3540029990986704 + }, + { + "index": 14, + "best_text_overlap": 0.41250502299139413 + }, + { + "index": 15, + "best_text_overlap": 0.34226328782237314 + } + ] + }, + { + "label": "nat-baseline-0.656", + "size": 256, + "correlation": 0.6558829559111284, + "visual_effective_rank": 18.735297664859708, + "text_effective_rank": 13.915532152821992, + "shared_directions_cos_gt_0.7": 11, + "shuffled_null": 1.0, + "shared_above_null": 10.0, + "principal_cosines_top8": [ + 0.993, + 0.971, + 0.954, + 0.938, + 0.922, + 0.868, + 0.845, + 0.826 + ], + "spectrum_profile": [ + { + "index": 0, + "best_text_overlap": 0.8308561390616345 + }, + { + "index": 1, + "best_text_overlap": 0.6425639957390433 + }, + { + "index": 2, + "best_text_overlap": 0.6401247564780709 + }, + { + "index": 3, + "best_text_overlap": 0.5245819731835861 + }, + { + "index": 4, + "best_text_overlap": 0.5324575387488661 + }, + { + "index": 5, + "best_text_overlap": 0.504776020548726 + }, + { + "index": 6, + "best_text_overlap": 0.5017207831704809 + }, + { + "index": 7, + "best_text_overlap": 0.26453910202691044 + }, + { + "index": 8, + "best_text_overlap": 0.3831674979777484 + }, + { + "index": 9, + "best_text_overlap": 0.34779386653289623 + }, + { + "index": 10, + "best_text_overlap": 0.2366745915766192 + }, + { + "index": 11, + "best_text_overlap": 0.1973585194151871 + }, + { + "index": 12, + "best_text_overlap": 0.21439811143331272 + }, + { + "index": 13, + "best_text_overlap": 0.2925437897693861 + }, + { + "index": 14, + "best_text_overlap": 0.09814723704154468 + }, + { + "index": 15, + "best_text_overlap": 0.20755663736183314 + } + ] + }, + { + "label": "nat-best-0.716", + "size": 256, + "correlation": 0.7157944505313559, + "visual_effective_rank": 39.69132496762981, + "text_effective_rank": 47.98422953390366, + "shared_directions_cos_gt_0.7": 16, + "shuffled_null": 1.0, + "shared_above_null": 15.0, + "principal_cosines_top8": [ + 0.987, + 0.978, + 0.96, + 0.941, + 0.934, + 0.922, + 0.899, + 0.89 + ], + "spectrum_profile": [ + { + "index": 0, + "best_text_overlap": 0.8406807387214822 + }, + { + "index": 1, + "best_text_overlap": 0.6748753041856854 + }, + { + "index": 2, + "best_text_overlap": 0.7174150319587717 + }, + { + "index": 3, + "best_text_overlap": 0.4742164704027202 + }, + { + "index": 4, + "best_text_overlap": 0.7105187496542062 + }, + { + "index": 5, + "best_text_overlap": 0.5435228035336661 + }, + { + "index": 6, + "best_text_overlap": 0.5173316630294161 + }, + { + "index": 7, + "best_text_overlap": 0.3720956303455928 + }, + { + "index": 8, + "best_text_overlap": 0.4031225977235423 + }, + { + "index": 9, + "best_text_overlap": 0.5909531802492844 + }, + { + "index": 10, + "best_text_overlap": 0.373881786434362 + }, + { + "index": 11, + "best_text_overlap": 0.2816693575587397 + }, + { + "index": 12, + "best_text_overlap": 0.21822240451244762 + }, + { + "index": 13, + "best_text_overlap": 0.3929212823566225 + }, + { + "index": 14, + "best_text_overlap": 0.4024276537497661 + }, + { + "index": 15, + "best_text_overlap": 0.2659834432053085 + } + ] + } + ], + "reading": "The narrow modality is the one to upgrade. A shared count far below both effective ranks means each side is rich but they are rich about different things." +} \ No newline at end of file diff --git a/artifacts/vg_5k/shared_rank_scale.json b/artifacts/vg_5k/shared_rank_scale.json new file mode 100644 index 0000000..b0d6e98 --- /dev/null +++ b/artifacts/vg_5k/shared_rank_scale.json @@ -0,0 +1,259 @@ +{ + "protocol": "Effective rank from the participation ratio of each field's spectrum; shared directions from principal angles between the two leading eigenspaces. No pairing enters beyond the alignment the fields already carry.", + "width": 48, + "rows": [ + { + "label": "scenes-256", + "size": 256, + "correlation": 0.7157944505313559, + "visual_effective_rank": 39.69132496762981, + "text_effective_rank": 47.98422953390366, + "shared_directions_cos_gt_0.7": 23, + "principal_cosines_top8": [ + 0.992, + 0.984, + 0.971, + 0.963, + 0.962, + 0.949, + 0.94, + 0.935 + ], + "spectrum_profile": [ + { + "index": 0, + "best_text_overlap": 0.8406807387214822 + }, + { + "index": 1, + "best_text_overlap": 0.6748753041856854 + }, + { + "index": 2, + "best_text_overlap": 0.7174150319587717 + }, + { + "index": 3, + "best_text_overlap": 0.4742164704027202 + }, + { + "index": 4, + "best_text_overlap": 0.7105187496542062 + }, + { + "index": 5, + "best_text_overlap": 0.5435228035336661 + }, + { + "index": 6, + "best_text_overlap": 0.5173316630294161 + }, + { + "index": 7, + "best_text_overlap": 0.3720956303455928 + }, + { + "index": 8, + "best_text_overlap": 0.4031225977235423 + }, + { + "index": 9, + "best_text_overlap": 0.5909531802492844 + }, + { + "index": 10, + "best_text_overlap": 0.373881786434362 + }, + { + "index": 11, + "best_text_overlap": 0.2816693575587397 + }, + { + "index": 12, + "best_text_overlap": 0.21822240451244762 + }, + { + "index": 13, + "best_text_overlap": 0.3929212823566225 + }, + { + "index": 14, + "best_text_overlap": 0.4024276537497661 + }, + { + "index": 15, + "best_text_overlap": 0.2659834432053085 + } + ] + }, + { + "label": "scenes-512", + "size": 512, + "correlation": 0.720884905640894, + "visual_effective_rank": 44.16644435195353, + "text_effective_rank": 56.80473066126686, + "shared_directions_cos_gt_0.7": 20, + "principal_cosines_top8": [ + 0.99, + 0.973, + 0.96, + 0.95, + 0.938, + 0.928, + 0.907, + 0.903 + ], + "spectrum_profile": [ + { + "index": 0, + "best_text_overlap": 0.8697350573768952 + }, + { + "index": 1, + "best_text_overlap": 0.7200022519825038 + }, + { + "index": 2, + "best_text_overlap": 0.7564612333353183 + }, + { + "index": 3, + "best_text_overlap": 0.501988106773722 + }, + { + "index": 4, + "best_text_overlap": 0.783113379169079 + }, + { + "index": 5, + "best_text_overlap": 0.37328195185984525 + }, + { + "index": 6, + "best_text_overlap": 0.4801278579299998 + }, + { + "index": 7, + "best_text_overlap": 0.383298070449205 + }, + { + "index": 8, + "best_text_overlap": 0.47046163940920954 + }, + { + "index": 9, + "best_text_overlap": 0.4732632962224147 + }, + { + "index": 10, + "best_text_overlap": 0.348622185453265 + }, + { + "index": 11, + "best_text_overlap": 0.4225508394457381 + }, + { + "index": 12, + "best_text_overlap": 0.21406071363464227 + }, + { + "index": 13, + "best_text_overlap": 0.3098353728024743 + }, + { + "index": 14, + "best_text_overlap": 0.3400635764036258 + }, + { + "index": 15, + "best_text_overlap": 0.23608736474375264 + } + ] + }, + { + "label": "scenes-1024", + "size": 1024, + "correlation": 0.6922444253804424, + "visual_effective_rank": 49.142293255949674, + "text_effective_rank": 62.90043889867087, + "shared_directions_cos_gt_0.7": 19, + "principal_cosines_top8": [ + 0.988, + 0.962, + 0.949, + 0.938, + 0.922, + 0.916, + 0.908, + 0.9 + ], + "spectrum_profile": [ + { + "index": 0, + "best_text_overlap": 0.8301375686311265 + }, + { + "index": 1, + "best_text_overlap": 0.6075551782428065 + }, + { + "index": 2, + "best_text_overlap": 0.6710977881848478 + }, + { + "index": 3, + "best_text_overlap": 0.6555164924250194 + }, + { + "index": 4, + "best_text_overlap": 0.6194066121363382 + }, + { + "index": 5, + "best_text_overlap": 0.40182630965483374 + }, + { + "index": 6, + "best_text_overlap": 0.5020646929681399 + }, + { + "index": 7, + "best_text_overlap": 0.3892962240055352 + }, + { + "index": 8, + "best_text_overlap": 0.47168546897479857 + }, + { + "index": 9, + "best_text_overlap": 0.40763120184534807 + }, + { + "index": 10, + "best_text_overlap": 0.44277607795178114 + }, + { + "index": 11, + "best_text_overlap": 0.2873332187039861 + }, + { + "index": 12, + "best_text_overlap": 0.40798659732142045 + }, + { + "index": 13, + "best_text_overlap": 0.3901617780561322 + }, + { + "index": 14, + "best_text_overlap": 0.3381332001188724 + }, + { + "index": 15, + "best_text_overlap": 0.26462763928061106 + } + ] + } + ], + "reading": "The narrow modality is the one to upgrade. A shared count far below both effective ranks means each side is rich but they are rich about different things." +} \ No newline at end of file diff --git a/artifacts/vg_5k/shared_rank_segments.json b/artifacts/vg_5k/shared_rank_segments.json new file mode 100644 index 0000000..7408bd4 --- /dev/null +++ b/artifacts/vg_5k/shared_rank_segments.json @@ -0,0 +1,343 @@ +{ + "protocol": "Effective rank from the participation ratio of each field's spectrum; shared directions from principal angles between the two leading eigenspaces. No pairing enters beyond the alignment the fields already carry.", + "width": 32, + "rows": [ + { + "label": "synth-RECOVERS", + "size": 256, + "correlation": 0.9284809883663888, + "visual_effective_rank": 14.50854049599196, + "text_effective_rank": 11.832081486516831, + "shared_directions_cos_gt_0.7": 27, + "principal_cosines_top8": [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.999, + 0.999, + 0.999 + ], + "spectrum_profile": [ + { + "index": 0, + "best_text_overlap": 0.9988184979219944 + }, + { + "index": 1, + "best_text_overlap": 0.9038254548424662 + }, + { + "index": 2, + "best_text_overlap": 0.7738637793117231 + }, + { + "index": 3, + "best_text_overlap": 0.7898924505967239 + }, + { + "index": 4, + "best_text_overlap": 0.9633572761985347 + }, + { + "index": 5, + "best_text_overlap": 0.9667594083538564 + }, + { + "index": 6, + "best_text_overlap": 0.8581068112833727 + }, + { + "index": 7, + "best_text_overlap": 0.7720408457258224 + }, + { + "index": 8, + "best_text_overlap": 0.9451869286586472 + }, + { + "index": 9, + "best_text_overlap": 0.9346387523699597 + }, + { + "index": 10, + "best_text_overlap": 0.9744620235608439 + }, + { + "index": 11, + "best_text_overlap": 0.8287855141725994 + }, + { + "index": 12, + "best_text_overlap": 0.8450929971524885 + }, + { + "index": 13, + "best_text_overlap": 0.7475087759782241 + }, + { + "index": 14, + "best_text_overlap": 0.5363062788775304 + }, + { + "index": 15, + "best_text_overlap": 0.5851587348701632 + } + ] + }, + { + "label": "nat-6seg", + "size": 256, + "correlation": 0.6922159706308013, + "visual_effective_rank": 39.69132496762981, + "text_effective_rank": 31.7662539087451, + "shared_directions_cos_gt_0.7": 15, + "principal_cosines_top8": [ + 0.989, + 0.978, + 0.956, + 0.938, + 0.916, + 0.892, + 0.881, + 0.87 + ], + "spectrum_profile": [ + { + "index": 0, + "best_text_overlap": 0.7532219338133863 + }, + { + "index": 1, + "best_text_overlap": 0.618182152081914 + }, + { + "index": 2, + "best_text_overlap": 0.6411572047678387 + }, + { + "index": 3, + "best_text_overlap": 0.5609614131227192 + }, + { + "index": 4, + "best_text_overlap": 0.5786521313263311 + }, + { + "index": 5, + "best_text_overlap": 0.6212040366073345 + }, + { + "index": 6, + "best_text_overlap": 0.6014409101136969 + }, + { + "index": 7, + "best_text_overlap": 0.3241154791286174 + }, + { + "index": 8, + "best_text_overlap": 0.33772536144199544 + }, + { + "index": 9, + "best_text_overlap": 0.46886651280081304 + }, + { + "index": 10, + "best_text_overlap": 0.339879585332424 + }, + { + "index": 11, + "best_text_overlap": 0.2697704959742108 + }, + { + "index": 12, + "best_text_overlap": 0.3089464131279491 + }, + { + "index": 13, + "best_text_overlap": 0.34824951852897246 + }, + { + "index": 14, + "best_text_overlap": 0.26546032832701905 + }, + { + "index": 15, + "best_text_overlap": 0.17108445671033506 + } + ] + }, + { + "label": "nat-16seg", + "size": 256, + "correlation": 0.6794444345566248, + "visual_effective_rank": 40.32463000927306, + "text_effective_rank": 31.7662539087451, + "shared_directions_cos_gt_0.7": 14, + "principal_cosines_top8": [ + 0.991, + 0.979, + 0.955, + 0.937, + 0.917, + 0.897, + 0.882, + 0.87 + ], + "spectrum_profile": [ + { + "index": 0, + "best_text_overlap": 0.7366894502381554 + }, + { + "index": 1, + "best_text_overlap": 0.6371340906520718 + }, + { + "index": 2, + "best_text_overlap": 0.6670233369588643 + }, + { + "index": 3, + "best_text_overlap": 0.7305683273676973 + }, + { + "index": 4, + "best_text_overlap": 0.5178189484095195 + }, + { + "index": 5, + "best_text_overlap": 0.6108081877127542 + }, + { + "index": 6, + "best_text_overlap": 0.5743712616080214 + }, + { + "index": 7, + "best_text_overlap": 0.30576250785088804 + }, + { + "index": 8, + "best_text_overlap": 0.31705130049377683 + }, + { + "index": 9, + "best_text_overlap": 0.434975496479508 + }, + { + "index": 10, + "best_text_overlap": 0.2627438939568967 + }, + { + "index": 11, + "best_text_overlap": 0.22483152594639236 + }, + { + "index": 12, + "best_text_overlap": 0.24208088194854407 + }, + { + "index": 13, + "best_text_overlap": 0.2850328753356361 + }, + { + "index": 14, + "best_text_overlap": 0.24894409456686264 + }, + { + "index": 15, + "best_text_overlap": 0.3923832160325988 + } + ] + }, + { + "label": "nat-oracle-regions", + "size": 256, + "correlation": 0.702748890720233, + "visual_effective_rank": 39.32779841860138, + "text_effective_rank": 31.7662539087451, + "shared_directions_cos_gt_0.7": 15, + "principal_cosines_top8": [ + 0.99, + 0.978, + 0.959, + 0.939, + 0.921, + 0.899, + 0.89, + 0.876 + ], + "spectrum_profile": [ + { + "index": 0, + "best_text_overlap": 0.7416685704809811 + }, + { + "index": 1, + "best_text_overlap": 0.6591035296451755 + }, + { + "index": 2, + "best_text_overlap": 0.6730514886891261 + }, + { + "index": 3, + "best_text_overlap": 0.5813764120097434 + }, + { + "index": 4, + "best_text_overlap": 0.5492379792963318 + }, + { + "index": 5, + "best_text_overlap": 0.6616657410048805 + }, + { + "index": 6, + "best_text_overlap": 0.5607722465245946 + }, + { + "index": 7, + "best_text_overlap": 0.3913414443532297 + }, + { + "index": 8, + "best_text_overlap": 0.3431217610653001 + }, + { + "index": 9, + "best_text_overlap": 0.39446274885223964 + }, + { + "index": 10, + "best_text_overlap": 0.2624025040480312 + }, + { + "index": 11, + "best_text_overlap": 0.31050951747572164 + }, + { + "index": 12, + "best_text_overlap": 0.37391245736141376 + }, + { + "index": 13, + "best_text_overlap": 0.2550412331810302 + }, + { + "index": 14, + "best_text_overlap": 0.29254528024572746 + }, + { + "index": 15, + "best_text_overlap": 0.3608253612065695 + } + ] + } + ], + "reading": "The narrow modality is the one to upgrade. A shared count far below both effective ranks means each side is rich but they are rich about different things." +} \ No newline at end of file diff --git a/artifacts/vg_5k/shared_rank_text.json b/artifacts/vg_5k/shared_rank_text.json new file mode 100644 index 0000000..6aa5a58 --- /dev/null +++ b/artifacts/vg_5k/shared_rank_text.json @@ -0,0 +1,343 @@ +{ + "protocol": "Effective rank from the participation ratio of each field's spectrum; shared directions from principal angles between the two leading eigenspaces. No pairing enters beyond the alignment the fields already carry.", + "width": 32, + "rows": [ + { + "label": "text-dim24", + "size": 256, + "correlation": 0.6614852336553844, + "visual_effective_rank": 39.69132496762981, + "text_effective_rank": 18.960351283615108, + "shared_directions_cos_gt_0.7": 10, + "principal_cosines_top8": [ + 0.989, + 0.977, + 0.944, + 0.917, + 0.909, + 0.861, + 0.844, + 0.826 + ], + "spectrum_profile": [ + { + "index": 0, + "best_text_overlap": 0.7251810720780995 + }, + { + "index": 1, + "best_text_overlap": 0.619527952553889 + }, + { + "index": 2, + "best_text_overlap": 0.6084134190397698 + }, + { + "index": 3, + "best_text_overlap": 0.4814861406651484 + }, + { + "index": 4, + "best_text_overlap": 0.47994567975555563 + }, + { + "index": 5, + "best_text_overlap": 0.36947588861735575 + }, + { + "index": 6, + "best_text_overlap": 0.3930833060287113 + }, + { + "index": 7, + "best_text_overlap": 0.3941645312523783 + }, + { + "index": 8, + "best_text_overlap": 0.2802360778324725 + }, + { + "index": 9, + "best_text_overlap": 0.2539445797009068 + }, + { + "index": 10, + "best_text_overlap": 0.2263224976556755 + }, + { + "index": 11, + "best_text_overlap": 0.24920478236754334 + }, + { + "index": 12, + "best_text_overlap": 0.23839820384648105 + }, + { + "index": 13, + "best_text_overlap": 0.32390692869819593 + }, + { + "index": 14, + "best_text_overlap": 0.18319221000344854 + }, + { + "index": 15, + "best_text_overlap": 0.17296430682365208 + } + ] + }, + { + "label": "text-dim48", + "size": 256, + "correlation": 0.6922159706308013, + "visual_effective_rank": 39.69132496762981, + "text_effective_rank": 31.7662539087451, + "shared_directions_cos_gt_0.7": 15, + "principal_cosines_top8": [ + 0.989, + 0.978, + 0.956, + 0.938, + 0.916, + 0.892, + 0.881, + 0.87 + ], + "spectrum_profile": [ + { + "index": 0, + "best_text_overlap": 0.7532219338133863 + }, + { + "index": 1, + "best_text_overlap": 0.618182152081914 + }, + { + "index": 2, + "best_text_overlap": 0.6411572047678387 + }, + { + "index": 3, + "best_text_overlap": 0.5609614131227192 + }, + { + "index": 4, + "best_text_overlap": 0.5786521313263311 + }, + { + "index": 5, + "best_text_overlap": 0.6212040366073345 + }, + { + "index": 6, + "best_text_overlap": 0.6014409101136969 + }, + { + "index": 7, + "best_text_overlap": 0.3241154791286174 + }, + { + "index": 8, + "best_text_overlap": 0.33772536144199544 + }, + { + "index": 9, + "best_text_overlap": 0.46886651280081304 + }, + { + "index": 10, + "best_text_overlap": 0.339879585332424 + }, + { + "index": 11, + "best_text_overlap": 0.2697704959742108 + }, + { + "index": 12, + "best_text_overlap": 0.3089464131279491 + }, + { + "index": 13, + "best_text_overlap": 0.34824951852897246 + }, + { + "index": 14, + "best_text_overlap": 0.26546032832701905 + }, + { + "index": 15, + "best_text_overlap": 0.17108445671033506 + } + ] + }, + { + "label": "text-dim128", + "size": 256, + "correlation": 0.7157944505313559, + "visual_effective_rank": 39.69132496762981, + "text_effective_rank": 47.98422953390366, + "shared_directions_cos_gt_0.7": 16, + "principal_cosines_top8": [ + 0.987, + 0.978, + 0.96, + 0.941, + 0.934, + 0.922, + 0.899, + 0.89 + ], + "spectrum_profile": [ + { + "index": 0, + "best_text_overlap": 0.8406807387214822 + }, + { + "index": 1, + "best_text_overlap": 0.6748753041856854 + }, + { + "index": 2, + "best_text_overlap": 0.7174150319587717 + }, + { + "index": 3, + "best_text_overlap": 0.4742164704027202 + }, + { + "index": 4, + "best_text_overlap": 0.7105187496542062 + }, + { + "index": 5, + "best_text_overlap": 0.5435228035336661 + }, + { + "index": 6, + "best_text_overlap": 0.5173316630294161 + }, + { + "index": 7, + "best_text_overlap": 0.3720956303455928 + }, + { + "index": 8, + "best_text_overlap": 0.4031225977235423 + }, + { + "index": 9, + "best_text_overlap": 0.5909531802492844 + }, + { + "index": 10, + "best_text_overlap": 0.373881786434362 + }, + { + "index": 11, + "best_text_overlap": 0.2816693575587397 + }, + { + "index": 12, + "best_text_overlap": 0.21822240451244762 + }, + { + "index": 13, + "best_text_overlap": 0.3929212823566225 + }, + { + "index": 14, + "best_text_overlap": 0.4024276537497661 + }, + { + "index": 15, + "best_text_overlap": 0.2659834432053085 + } + ] + }, + { + "label": "text-dim256", + "size": 256, + "correlation": 0.7000821325988256, + "visual_effective_rank": 39.69132496762981, + "text_effective_rank": 53.019713413426004, + "shared_directions_cos_gt_0.7": 17, + "principal_cosines_top8": [ + 0.985, + 0.972, + 0.956, + 0.95, + 0.937, + 0.928, + 0.918, + 0.907 + ], + "spectrum_profile": [ + { + "index": 0, + "best_text_overlap": 0.8762239900209906 + }, + { + "index": 1, + "best_text_overlap": 0.6354174458640104 + }, + { + "index": 2, + "best_text_overlap": 0.6663671679367767 + }, + { + "index": 3, + "best_text_overlap": 0.5750584418168163 + }, + { + "index": 4, + "best_text_overlap": 0.6301701541550169 + }, + { + "index": 5, + "best_text_overlap": 0.5674100516608417 + }, + { + "index": 6, + "best_text_overlap": 0.5656412745853614 + }, + { + "index": 7, + "best_text_overlap": 0.40296826460328056 + }, + { + "index": 8, + "best_text_overlap": 0.39820202744406064 + }, + { + "index": 9, + "best_text_overlap": 0.53793615928177 + }, + { + "index": 10, + "best_text_overlap": 0.2919657497931648 + }, + { + "index": 11, + "best_text_overlap": 0.39265961704593844 + }, + { + "index": 12, + "best_text_overlap": 0.310451119516457 + }, + { + "index": 13, + "best_text_overlap": 0.5334702303308833 + }, + { + "index": 14, + "best_text_overlap": 0.24458631502415207 + }, + { + "index": 15, + "best_text_overlap": 0.306773828748064 + } + ] + } + ], + "reading": "The narrow modality is the one to upgrade. A shared count far below both effective ranks means each side is rich but they are rich about different things." +} \ No newline at end of file diff --git a/logs/backbone_battery.log b/logs/backbone_battery.log new file mode 100644 index 0000000..e69de29 diff --git a/logs/obj_base_gpu.log b/logs/obj_base_gpu.log new file mode 100644 index 0000000..de196aa --- /dev/null +++ b/logs/obj_base_gpu.log @@ -0,0 +1,3 @@ +Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. +`torch_dtype` is deprecated! Use `dtype` instead! + segment: 0%| | 0/250 [00:00 argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--fields", default="artifacts/vg_5k/natural_pipeline_w64.pt") + parser.add_argument("--size", type=int, default=128) + parser.add_argument("--trials", type=int, default=5) + parser.add_argument("--device", default="cuda:3") + parser.add_argument("--output", default="artifacts/vg_5k/diversity_subset.json") + return parser.parse_args() + + +def offdiagonal(matrix: np.ndarray) -> np.ndarray: + return matrix[~np.eye(len(matrix), dtype=bool)] + + +def normalise(matrix: np.ndarray) -> np.ndarray: + values = offdiagonal(matrix) + out = (matrix - values.mean()) / values.std() + np.fill_diagonal(out, 0.0) + return out + + +def farthest_point(field: np.ndarray, size: int, start: int) -> np.ndarray: + """Greedy farthest-point selection under the visual field's similarity. + + High field entry means similar, so dissimilarity is the negated entry and + each new scene is the one whose closest already-chosen neighbour is as far + as possible. + """ + chosen = [start] + closest = field[start].copy() + for _ in range(size - 1): + candidate = int(np.argmin(np.where(np.isin(np.arange(len(field)), chosen), + np.inf, closest))) + chosen.append(candidate) + closest = np.maximum(closest, field[candidate]) + return np.array(chosen) + + +def evaluate(visual: np.ndarray, text: np.ndarray, device, trials: int) -> dict: + size = len(visual) + visual = normalise(visual) + text = normalise(text) + mask = ~np.eye(size, dtype=bool) + rho = float(np.corrcoef(visual[mask], text[mask])[0, 1]) + swaps = all_swaps(size, device) + accuracies = [] + for trial in range(trials): + generator = np.random.default_rng(trial) + hidden = generator.permutation(size) + shuffled = text[np.ix_(hidden, hidden)] + columns = grampa(visual, shuffled, 1.0) + energy = ClosedFormEnergy( + standardized(torch.from_numpy(shuffled).to(device)).float(), + standardized(torch.from_numpy(visual).to(device)).float(), + 1.0, 1.0, 256, + ) + final, _ = steepest_descent( + energy, torch.from_numpy(columns.copy()).to(device), swaps, 3000 + ) + accuracies.append(float((hidden[final.cpu().numpy()] == np.arange(size)).mean())) + # how much of the correlation survives stripping the leading directions + symmetric = (visual + visual.T) / 2.0 + values, vectors = np.linalg.eigh(symmetric) + order = np.argsort(np.abs(values))[::-1][10:] + stripped_visual = (vectors[:, order] * values[order]) @ vectors[:, order].T + symmetric_text = (text + text.T) / 2.0 + values_t, vectors_t = np.linalg.eigh(symmetric_text) + order_t = np.argsort(np.abs(values_t))[::-1][10:] + stripped_text = (vectors_t[:, order_t] * values_t[order_t]) @ vectors_t[:, order_t].T + retained = float( + np.corrcoef(offdiagonal(stripped_visual), offdiagonal(stripped_text))[0, 1] + ) + return { + "correlation": rho, + "correlation_after_strip10": retained, + "recovery": float(np.mean(accuracies)), + "chance": 1.0 / size, + } + + +def main() -> None: + args = parse_args() + state = torch.load(args.fields, map_location="cpu", weights_only=False) + visual_full = state["visual_field"].double().numpy() + text_full = state["text_field"].double().numpy() + population = len(visual_full) + device = torch.device(args.device) + size = min(args.size, population) + + rows = [] + for start in range(3): + index = farthest_point(visual_full, size, start * 37 % population) + result = evaluate( + visual_full[np.ix_(index, index)], text_full[np.ix_(index, index)], + device, args.trials, + ) + result["selection"] = f"diverse (start {start})" + rows.append(result) + print(json.dumps(result), flush=True) + + for trial in range(3): + generator = np.random.default_rng(100 + trial) + index = generator.choice(population, size, replace=False) + result = evaluate( + visual_full[np.ix_(index, index)], text_full[np.ix_(index, index)], + device, args.trials, + ) + result["selection"] = f"random (seed {trial})" + rows.append(result) + print(json.dumps(result), flush=True) + + diverse = [r for r in rows if r["selection"].startswith("diverse")] + random_rows = [r for r in rows if r["selection"].startswith("random")] + summary = { + "protocol": ( + "DIAGNOSTIC, NOT A METHOD: scenes selected by visual dissimilarity " + "and their partners taken from the hidden pairing. One-sided " + "selection cannot be realised without the pairing, so this is an " + "upper bound used to price the idea before building the legal " + "unbalanced version." + ), + "population": population, + "subset_size": size, + "rows": rows, + "diverse_mean_correlation": float(np.mean([r["correlation"] for r in diverse])), + "random_mean_correlation": float(np.mean([r["correlation"] for r in random_rows])), + "diverse_mean_recovery": float(np.mean([r["recovery"] for r in diverse])), + "random_mean_recovery": float(np.mean([r["recovery"] for r in random_rows])), + } + print(json.dumps({k: v for k, v in summary.items() if k != "rows"})) + write_json(args.output, summary) + + +if __name__ == "__main__": + main() diff --git a/worldalign/field_anatomy.py b/worldalign/field_anatomy.py new file mode 100644 index 0000000..6156018 --- /dev/null +++ b/worldalign/field_anatomy.py @@ -0,0 +1,148 @@ +"""What is the field correlation made of, and how much of it can matching use? + +A puzzle motivates this. At 256 scenes the information-theoretic threshold is +0.29 and the fields correlate at 0.656, so the pairing is identifiable by a +wide margin, yet even at 64 scenes -- where the threshold is 0.51 and the +search space is trivially small -- recovery reaches 14%. Either the theory is +the wrong reference or the statistic is not measuring what the theory means. + +The suspect is nuisance structure shared by both fields. Scene similarity has +a component that says only "this scene is busy, so it resembles everything", +and both modalities see it. That component inflates the correlation over all +pairs while carrying no information about which scene is which, because it is +the same for every candidate pairing of a scene to a partner of similar +busyness. Correlated-matching theory assumes the correlation is in the +exchangeable noise, so a nuisance-inflated statistic overstates what the +theory would predict. + +This decomposes each field into a degree component -- the rank-one part +predicted by row and column means -- and the residual, and reports the +correlation of each. The residual correlation is the honest input to the +threshold, and if it falls far below 0.656 the recovery gap is explained and +the target moves. +""" + +from __future__ import annotations + +import argparse +import json + +import numpy as np +import torch + +from .common import write_json + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--fields", default="artifacts/vg_5k/natural_pipeline.pt") + parser.add_argument("--label", default="natural") + parser.add_argument("--output", default="artifacts/vg_5k/field_anatomy.json") + return parser.parse_args() + + +def offdiagonal(matrix: np.ndarray) -> np.ndarray: + return matrix[~np.eye(len(matrix), dtype=bool)] + + +def correlate(first: np.ndarray, second: np.ndarray) -> float: + return float(np.corrcoef(offdiagonal(first), offdiagonal(second))[0, 1]) + + +def degree_part(matrix: np.ndarray) -> tuple[np.ndarray, np.ndarray]: + """Split into the additive row/column-mean model and its residual. + + The two-way additive fit m + r_i + c_j is what a pure busyness effect + produces: every entry explained by how prominent each of its two scenes + is, with nothing said about the pair. + """ + size = len(matrix) + mask = ~np.eye(size, dtype=bool) + work = matrix.copy().astype(np.float64) + np.fill_diagonal(work, np.nan) + grand = np.nanmean(work) + rows = np.nanmean(work, axis=1, keepdims=True) - grand + cols = np.nanmean(work, axis=0, keepdims=True) - grand + fitted = grand + rows + cols + residual = np.where(mask, work - fitted, 0.0) + return np.where(mask, fitted, 0.0), residual + + +def spectral_strip(matrix: np.ndarray, components: int) -> np.ndarray: + """Remove the leading eigen-components, a stronger nuisance model.""" + symmetric = (matrix + matrix.T) / 2.0 + values, vectors = np.linalg.eigh(symmetric) + order = np.argsort(np.abs(values))[::-1] + keep = order[components:] + return (vectors[:, keep] * values[keep]) @ vectors[:, keep].T + + +def main() -> None: + args = parse_args() + state = torch.load(args.fields, map_location="cpu", weights_only=False) + visual = state["visual_field"].double().numpy() + text = state["text_field"].double().numpy() + size = len(visual) + + total = correlate(visual, text) + visual_fit, visual_residual = degree_part(visual) + text_fit, text_residual = degree_part(text) + + rows = [ + {"component": "raw field", "correlation": total}, + {"component": "degree model only", "correlation": correlate(visual_fit, text_fit)}, + { + "component": "residual after degree", + "correlation": correlate(visual_residual, text_residual), + }, + ] + for components in (1, 2, 5, 10): + rows.append( + { + "component": f"residual after top-{components} eigen", + "correlation": correlate( + spectral_strip(visual, components), spectral_strip(text, components) + ), + } + ) + + # how much of each field's own variance the nuisance model absorbs + share = { + "visual_degree_variance_share": float( + offdiagonal(visual_fit).var() / offdiagonal(visual).var() + ), + "text_degree_variance_share": float( + offdiagonal(text_fit).var() / offdiagonal(text).var() + ), + } + + residual_rho = rows[2]["correlation"] + threshold = float(np.sqrt(4 * np.log(size) / size)) + summary = { + "protocol": ( + "Both fields split into an additive row/column-mean model and its " + "residual, then correlated component by component. Hidden pairs " + "are read only to align the two fields." + ), + "label": args.label, + "size": size, + "rows": rows, + **share, + "it_threshold_at_size": threshold, + "residual_above_it_threshold": bool(residual_rho > threshold), + "reading": ( + "If the residual correlation is far below the raw correlation, the " + "headline statistic is inflated by nuisance structure that carries " + "no matching information, and the residual is the number that " + "should be compared against the recovery threshold." + ), + } + for row in rows: + print(f"{row['component']:<34} rho={row['correlation']:.4f}") + print(json.dumps(share)) + print(f"IT threshold at N={size}: {threshold:.4f}") + write_json(args.output, summary) + + +if __name__ == "__main__": + main() diff --git a/worldalign/natural_objects.py b/worldalign/natural_objects.py index 65e035b..6e18259 100644 --- a/worldalign/natural_objects.py +++ b/worldalign/natural_objects.py @@ -62,7 +62,8 @@ def parse_args() -> argparse.Namespace: def spectral_segments( - features: torch.Tensor, grid: int, segments: int, seed: int + features: torch.Tensor, grid: int, segments: int, seed: int, + device: torch.device | str = "cpu", ) -> np.ndarray: """Segment a patch grid by clustering the normalised affinity spectrum. @@ -70,22 +71,42 @@ def spectral_segments( stay connected, then the leading eigenvectors of the normalised Laplacian are clustered. This is the standard unsupervised recipe over self-supervised features. + + The eigendecomposition dominates the run: one dense symmetric problem of + side grid^2 per image, which on a contended CPU costs more than the + forward pass that produced the features. Running it on the accelerator + that is already holding the model turns a multi-hour extraction into a + short one, and the spatial prior is built once and cached rather than + rebuilt per image. """ from sklearn.cluster import KMeans - normalised = F.normalize(features.double(), dim=-1) - affinity = (normalised @ normalised.T).clamp_min(0).numpy() - coordinates = np.stack( - np.meshgrid(np.arange(grid), np.arange(grid), indexing="ij"), -1 - ).reshape(-1, 2).astype(np.float64) - distance = ((coordinates[:, None, :] - coordinates[None, :, :]) ** 2).sum(-1) - affinity = affinity * np.exp(-distance / (2 * (grid / 4.0) ** 2)) + normalised = F.normalize(features.to(device).double(), dim=-1) + affinity = (normalised @ normalised.T).clamp_min(0) + affinity = affinity * _spatial_prior(grid, device) degree = affinity.sum(1) - laplacian = affinity / np.sqrt(np.outer(degree, degree) + 1e-9) - values, vectors = np.linalg.eigh(laplacian) + laplacian = affinity / torch.sqrt(torch.outer(degree, degree) + 1e-9) + values, vectors = torch.linalg.eigh(laplacian) embedding = vectors[:, -segments:] - embedding /= np.linalg.norm(embedding, axis=1, keepdims=True).clip(1e-9) - return KMeans(segments, n_init=10, random_state=seed).fit_predict(embedding) + embedding = embedding / embedding.norm(dim=1, keepdim=True).clamp_min(1e-9) + return KMeans(segments, n_init=10, random_state=seed).fit_predict( + embedding.cpu().numpy() + ) + + +_PRIOR_CACHE: dict[tuple, torch.Tensor] = {} + + +def _spatial_prior(grid: int, device: torch.device | str) -> torch.Tensor: + """Gaussian locality weight on the patch lattice, built once per grid.""" + key = (grid, str(device)) + if key not in _PRIOR_CACHE: + axis = torch.arange(grid, dtype=torch.float64, device=device) + rows, cols = torch.meshgrid(axis, axis, indexing="ij") + coordinates = torch.stack([rows.flatten(), cols.flatten()], dim=-1) + distance = torch.cdist(coordinates, coordinates) ** 2 + _PRIOR_CACHE[key] = torch.exp(-distance / (2 * (grid / 4.0) ** 2)) + return _PRIOR_CACHE[key] def describe_segments( @@ -220,7 +241,7 @@ def main() -> None: ) else: labels = spectral_segments( - patches[position], grid, args.segments, args.seed + patches[position], grid, args.segments, args.seed, args.device ) described = describe_segments( labels, patches[position], raw[position], grid, args.min_patches diff --git a/worldalign/natural_pipeline.py b/worldalign/natural_pipeline.py index c30af4d..1a4f63e 100644 --- a/worldalign/natural_pipeline.py +++ b/worldalign/natural_pipeline.py @@ -41,9 +41,18 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--min-count", type=int, default=60) parser.add_argument("--max-vocabulary", type=int, default=600) parser.add_argument("--context-window", type=int, default=2) - parser.add_argument("--keep", type=int, default=8) + parser.add_argument("--keep", type=int, default=64) parser.add_argument("--shrinkage", type=float, default=0.05) + parser.add_argument( + "--weight-power", + type=float, + default=0.5, + help="Scale each kept direction by its discriminant eigenvalue to " + "this power. 0 reproduces the unweighted projection.", + ) parser.add_argument("--views", type=int, default=1) + parser.add_argument("--moment-degree", type=int, default=2, choices=[2, 3]) + parser.add_argument("--third-width", type=int, default=12) parser.add_argument("--seed", type=int, default=0) parser.add_argument("--output", default="artifacts/vg_5k/natural_pipeline.pt") return parser.parse_args() @@ -77,10 +86,43 @@ def word_vectors( return {word: embedding[row] for row, word in enumerate(vocabulary)} +def moment_field_degree( + sets: list[torch.Tensor], degree: int, third_width: int +) -> torch.Tensor: + """Set kernel carrying moments up to `degree`. + + A kernel carrying third moments spans more directions than one carrying + two, so raising the degree raises the field's rank by construction -- + which is the quantity the rank experiment showed to govern recovery. The + third moment is taken over a narrower basis because its size is cubic; + the first two are kept at full width. + """ + if degree < 3: + return moment_field(sets) + phis = [] + for members in sets: + first = members.mean(0) + second = (members[:, :, None] * members[:, None, :]).mean(0).flatten() + narrow = members[:, :third_width] + outer = narrow[:, :, None] * narrow[:, None, :] + third = (outer[:, :, :, None] * narrow[:, None, None, :]).mean(0).flatten() + phi = torch.cat([first, second, third]) + phis.append(phi / phi.norm().clamp_min(1e-9)) + stacked = torch.stack(phis) + return stacked @ stacked.T + + def content_directions( sets: list[np.ndarray], shrinkage: float -) -> tuple[np.ndarray, np.ndarray]: - """Directions separating scenes more than they separate parts.""" +) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + """Directions separating scenes more than they separate parts. + + Returns the discriminant eigenvalues alongside the basis. Scaling each + direction by its own eigenvalue lets a wide basis be kept without the + weakly separating directions drowning the strong ones, which is what + makes a wide keep beat a narrow one -- and a wide keep is what raises + the rank of the resulting field. + """ means = np.stack([item.mean(0) for item in sets]) centre = means.mean(0) within = np.concatenate([item - item.mean(0, keepdims=True) for item in sets]) @@ -91,7 +133,8 @@ def content_directions( values, vectors = eigh( scatter_between, scatter_within + shrinkage * trace * np.eye(len(scatter_within)) ) - return centre, vectors[:, np.argsort(values)[::-1]] + order = np.argsort(values)[::-1] + return centre, vectors[:, order], np.maximum(values[order], 0.0) def main() -> None: @@ -144,8 +187,8 @@ def main() -> None: language_fit = [item for item in language_fit if item is not None and len(item) > 1] vision_fit = [vision_state(p["vision_node_id"], 0) for p in fit] vision_fit = [item for item in vision_fit if item is not None and len(item) > 1] - language_centre, language_basis = content_directions(language_fit, args.shrinkage) - vision_centre, vision_basis = content_directions(vision_fit, args.shrinkage) + language_fitted = content_directions(language_fit, args.shrinkage) + vision_fitted = content_directions(vision_fit, args.shrinkage) evaluated = [ pair for pair in pairs[: args.samples] @@ -153,25 +196,32 @@ def main() -> None: ] keep = args.keep - def project(raw: np.ndarray, centre: np.ndarray, basis: np.ndarray) -> torch.Tensor: + def project(raw: np.ndarray, fitted: tuple) -> torch.Tensor: + centre, basis, values = fitted width = min(keep, basis.shape[1]) + scale = values[:width] ** args.weight_power if args.weight_power else 1.0 return F.normalize( - torch.tensor((raw - centre) @ basis[:, :width], dtype=torch.float32), dim=-1 + torch.tensor( + (raw - centre) @ basis[:, :width] * scale, dtype=torch.float32 + ), dim=-1 ) text_sets = [ - project(language_state(p["text_node_id"]), language_centre, language_basis) - for p in evaluated + project(language_state(p["text_node_id"]), language_fitted) for p in evaluated ] view_fields = [] for view in range(views): sets = [ - project(vision_state(p["vision_node_id"], view), vision_centre, vision_basis) + project(vision_state(p["vision_node_id"], view), vision_fitted) for p in evaluated ] - view_fields.append(moment_field(sets)) + view_fields.append( + moment_field_degree(sets, args.moment_degree, args.third_width) + ) visual_field = torch.stack(view_fields).mean(0) - text_field = moment_field(text_sets) + text_field = moment_field_degree( + text_sets, args.moment_degree, args.third_width + ) mask = ~np.eye(len(evaluated), dtype=bool) correlation = float( @@ -193,6 +243,8 @@ def main() -> None: "samples": len(evaluated), "views": views, "kept_directions": keep, + "weight_power": args.weight_power, + "moment_degree": args.moment_degree, "field_correlation_at_truth": correlation, "recovery_threshold": 0.9, } diff --git a/worldalign/natural_recovery.py b/worldalign/natural_recovery.py new file mode 100644 index 0000000..58de5e5 --- /dev/null +++ b/worldalign/natural_recovery.py @@ -0,0 +1,130 @@ +"""Blind recovery on natural data, run because the old gate was wrong. + +The standing gate said the correlation had to reach 0.9 and natural data sat +at 0.656, so no recovery run was warranted and none was made. The rank +experiment retires that rule: a synthetic field truncated to rank eight has a +correlation of 0.906 and recovers 13%, while the same field at rank sixteen +recovers 96%. Correlation alone neither predicts nor forbids recovery, and the +width of the shared spectrum is the variable that moves it. + +By the width measure the best natural configuration now sits at sixteen shared +directions, inside the interval where the synthetic ladder crosses from 13% to +96%. Its correlation is far lower, so the joint condition is probably not met +-- but the honest way to find out is to run the search rather than to infer +the answer from a statistic that has just been shown not to govern it. + +Hidden pairs are generated per trial and read only for scoring. +""" + +from __future__ import annotations + +import argparse +import json + +import numpy as np +import torch + +from .common import write_json +from .spectral_match import grampa, umeyama +from .synth_fast_gate import ClosedFormEnergy, all_swaps, steepest_descent +from .synth_triangle_gate import standardized + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--fields", default="artifacts/vg_5k/np_n256.pt") + parser.add_argument("--label", default="natural-best") + parser.add_argument("--trials", type=int, default=5) + parser.add_argument("--etas", type=float, nargs="+", default=[0.2, 0.5, 1.0, 2.0]) + parser.add_argument("--device", default="cuda:3") + parser.add_argument("--output", default="artifacts/vg_5k/natural_recovery.json") + return parser.parse_args() + + +def offdiagonal(matrix: np.ndarray) -> np.ndarray: + return matrix[~np.eye(len(matrix), dtype=bool)] + + +def normalise(matrix: np.ndarray) -> np.ndarray: + values = offdiagonal(matrix) + out = (matrix - values.mean()) / values.std() + np.fill_diagonal(out, 0.0) + return out + + +def main() -> None: + args = parse_args() + state = torch.load(args.fields, map_location="cpu", weights_only=False) + visual = normalise(state["visual_field"].double().numpy()) + text = normalise(state["text_field"].double().numpy()) + size = len(visual) + device = torch.device(args.device) + swaps = all_swaps(size, device) + mask = ~np.eye(size, dtype=bool) + rho = float(np.corrcoef(visual[mask], text[mask])[0, 1]) + + rows = [] + for trial in range(args.trials): + generator = np.random.default_rng(trial) + hidden = generator.permutation(size) + shuffled = text[np.ix_(hidden, hidden)] + energy = ClosedFormEnergy( + standardized(torch.from_numpy(shuffled).to(device)).float(), + standardized(torch.from_numpy(visual).to(device)).float(), + 1.0, 1.0, 256, + ) + + def score(columns: np.ndarray) -> float: + return float((hidden[columns] == np.arange(size)).mean()) + + # several spectral starts, keep the one the energy prefers -- model + # selection by energy only, never by accuracy + best_energy, best_columns, best_tag = np.inf, None, "" + candidates = [("umeyama", umeyama(visual, shuffled))] + candidates += [(f"grampa eta={eta}", grampa(visual, shuffled, eta)) + for eta in args.etas] + for tag, columns in candidates: + start = torch.from_numpy(columns.copy()).to(device) + value = float(energy.energy(start[None])[0]) + if value < best_energy: + best_energy, best_columns, best_tag = value, columns, tag + + initial = score(best_columns) + final, _ = steepest_descent( + energy, torch.from_numpy(best_columns.copy()).to(device), swaps, 5000 + ) + refined = score(final.cpu().numpy()) + rows.append( + { + "trial": trial, + "chosen_start": best_tag, + "spectral_accuracy": initial, + "refined_accuracy": refined, + } + ) + print( + f"trial {trial}: start={best_tag:<14} spectral={initial:.4f} " + f"refined={refined:.4f} (chance {1/size:.4f})", + flush=True, + ) + + summary = { + "protocol": ( + "Blind: the hidden permutation is generated per trial and used " + "only for scoring. The spectral start is chosen by energy, never " + "by accuracy." + ), + "label": args.label, + "size": size, + "field_correlation_at_truth": rho, + "chance": 1.0 / size, + "rows": rows, + "mean_spectral": float(np.mean([r["spectral_accuracy"] for r in rows])), + "mean_refined": float(np.mean([r["refined_accuracy"] for r in rows])), + } + print(json.dumps({k: v for k, v in summary.items() if k != "rows"})) + write_json(args.output, summary) + + +if __name__ == "__main__": + main() diff --git a/worldalign/projection_sweep.py b/worldalign/projection_sweep.py new file mode 100644 index 0000000..2326461 --- /dev/null +++ b/worldalign/projection_sweep.py @@ -0,0 +1,240 @@ +"""Sweep the content projection, the largest measured lever on natural data. + +Content projection took the field correlation from 0.489 to 0.656, further +than any other single choice, and it was adopted at one setting without a +sweep. This prices its free parameters -- how many directions to keep, how +hard to shrink the within-scene scatter -- and two variants of the projection +itself. + +The whitened variant rescales each kept direction by its own between-scene +spread, so a direction that separates scenes weakly is not drowned by one +that separates them strongly. The kernel variant fits the same discriminant +in a random Fourier feature space, testing whether the directions that +separate scenes are linear in the encoder's coordinates at all. + +Word vectors and object states are computed once and reused, so the whole +sweep costs one pipeline run. +""" + +from __future__ import annotations + +import argparse +import json +import re +from pathlib import Path + +import numpy as np +import torch +import torch.nn.functional as F +from scipy.linalg import eigh + +from .common import seed_everything, write_json +from .natural_families import load_phrases +from .natural_pipeline import word_vectors +from .synth_cc_battery import moment_field + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--vg-dir", default="artifacts/vg_5k") + parser.add_argument("--objects", default="artifacts/vg_5k/natural_objects.pt") + parser.add_argument("--samples", type=int, default=256) + parser.add_argument("--fit-scenes", type=int, default=1200) + parser.add_argument("--word-vectors", type=int, default=48) + parser.add_argument("--min-count", type=int, default=60) + parser.add_argument("--max-vocabulary", type=int, default=600) + parser.add_argument("--context-window", type=int, default=2) + parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--rff", type=int, default=256, help="Random Fourier features for the kernel variant.") + parser.add_argument("--output", default="artifacts/vg_5k/projection_sweep.json") + return parser.parse_args() + + +def discriminant( + sets: list[np.ndarray], shrinkage: float +) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + """Directions separating scenes more than parts, with their eigenvalues.""" + means = np.stack([item.mean(0) for item in sets]) + centre = means.mean(0) + within = np.concatenate([item - item.mean(0, keepdims=True) for item in sets]) + scatter_within = within.T @ within / max(len(within) - 1, 1) + centred = means - centre + scatter_between = centred.T @ centred / max(len(centred) - 1, 1) + trace = np.trace(scatter_within) / len(scatter_within) + values, vectors = eigh( + scatter_between, scatter_within + shrinkage * trace * np.eye(len(scatter_within)) + ) + order = np.argsort(values)[::-1] + return centre, vectors[:, order], np.maximum(values[order], 0.0) + + +def lift(raw: np.ndarray, weights: np.ndarray, offsets: np.ndarray) -> np.ndarray: + """Random Fourier features: an explicit map into an RBF feature space.""" + return np.cos(raw @ weights + offsets) * np.sqrt(2.0 / weights.shape[1]) + + +def correlation(vision_sets, text_sets) -> float: + visual_field = moment_field(vision_sets) + text_field = moment_field(text_sets) + mask = ~np.eye(len(vision_sets), dtype=bool) + return float( + np.corrcoef( + visual_field.double().numpy()[mask], text_field.double().numpy()[mask] + )[0, 1] + ) + + +def main() -> None: + args = parse_args() + seed_everything(args.seed) + vg_dir = Path(args.vg_dir) + truth = [ + json.loads(line) + for line in (vg_dir / "ground_truth.private.jsonl") + .read_text(encoding="utf-8") + .splitlines() + if line.strip() + ] + records = { + json.loads(line)["node_id"]: json.loads(line) + for line in (vg_dir / "text_nodes.jsonl").read_text(encoding="utf-8").splitlines() + if line.strip() + } + state = torch.load(args.objects, map_location="cpu", weights_only=False) + index = {node: position for position, node in enumerate(state["node_ids"])} + vectors = word_vectors(load_phrases(vg_dir / "text_nodes.jsonl", "region_closed"), args) + + def language_state(node: str) -> np.ndarray | None: + rows = [] + for phrase in records[node]["region_closed"]: + hits = [vectors[token] for token in re.findall(r"[a-z]+", phrase.lower()) + if token in vectors] + if hits: + rows.append(np.mean(hits, axis=0)) + return np.stack(rows) if rows else None + + def vision_state(node: str) -> np.ndarray | None: + segments = state["segments"][index[node]] + return ( + np.stack([item["feature"] for item in segments]).astype(np.float64) + if segments + else None + ) + + pairs = [ + pair for pair in truth + if pair["vision_node_id"] in index and pair["text_node_id"] in records + ] + fit = pairs[args.samples : args.samples + args.fit_scenes] + language_fit = [language_state(p["text_node_id"]) for p in fit] + language_fit = [i for i in language_fit if i is not None and len(i) > 1] + vision_fit = [vision_state(p["vision_node_id"]) for p in fit] + vision_fit = [i for i in vision_fit if i is not None and len(i) > 1] + evaluated = [ + pair for pair in pairs[: args.samples] + if language_state(pair["text_node_id"]) is not None + and vision_state(pair["vision_node_id"]) is not None + ] + language_eval = [language_state(p["text_node_id"]) for p in evaluated] + vision_eval = [vision_state(p["vision_node_id"]) for p in evaluated] + print(f"fit {len(vision_fit)} vision / {len(language_fit)} text, " + f"evaluated {len(evaluated)}", flush=True) + + rows = [] + + def record(variant: str, keep, shrinkage, value: float) -> None: + rows.append({"variant": variant, "keep": keep, "shrinkage": shrinkage, + "correlation": value}) + print(f"{variant:<10} keep={str(keep):<5} shrink={shrinkage:<6} rho={value:.4f}", + flush=True) + + # --- linear discriminant: sweep kept width and shrinkage --- + for shrinkage in (0.01, 0.05, 0.2, 1.0): + lang = discriminant(language_fit, shrinkage) + vis = discriminant(vision_fit, shrinkage) + for keep in (4, 8, 16, 32, 64): + def project(raw, fitted): + centre, basis, _ = fitted + width = min(keep, basis.shape[1]) + return F.normalize( + torch.tensor((raw - centre) @ basis[:, :width], dtype=torch.float32), dim=-1 + ) + value = correlation( + [project(v, vis) for v in vision_eval], + [project(t, lang) for t in language_eval], + ) + record("linear", keep, shrinkage, value) + + # --- eigenvalue-weighted: scale each direction by its own separation --- + for shrinkage in (0.05, 0.2): + lang = discriminant(language_fit, shrinkage) + vis = discriminant(vision_fit, shrinkage) + for keep in (8, 16, 32, 64): + for power in (0.5, 1.0): + def project(raw, fitted): + centre, basis, values = fitted + width = min(keep, basis.shape[1]) + scale = values[:width] ** power + return F.normalize( + torch.tensor( + (raw - centre) @ basis[:, :width] * scale, dtype=torch.float32 + ), dim=-1 + ) + value = correlation( + [project(v, vis) for v in vision_eval], + [project(t, lang) for t in language_eval], + ) + record(f"weighted^{power}", keep, shrinkage, value) + + # --- kernel discriminant in a random Fourier feature space --- + generator = np.random.default_rng(args.seed) + + def kernel_space(fit_sets, eval_sets, gamma_scale): + stacked = np.concatenate(fit_sets) + spread = np.median(np.linalg.norm(stacked - stacked.mean(0), axis=1)) + gamma = gamma_scale / max(spread ** 2, 1e-9) + weights = generator.normal( + 0.0, np.sqrt(2 * gamma), size=(stacked.shape[1], args.rff) + ) + offsets = generator.uniform(0, 2 * np.pi, size=args.rff) + return ( + [lift(item, weights, offsets) for item in fit_sets], + [lift(item, weights, offsets) for item in eval_sets], + ) + + for gamma_scale in (0.25, 1.0): + lang_fit_k, lang_eval_k = kernel_space(language_fit, language_eval, gamma_scale) + vis_fit_k, vis_eval_k = kernel_space(vision_fit, vision_eval, gamma_scale) + lang = discriminant(lang_fit_k, 0.05) + vis = discriminant(vis_fit_k, 0.05) + for keep in (8, 16, 32): + def project(raw, fitted): + centre, basis, _ = fitted + width = min(keep, basis.shape[1]) + return F.normalize( + torch.tensor((raw - centre) @ basis[:, :width], dtype=torch.float32), dim=-1 + ) + value = correlation( + [project(v, vis) for v in vis_eval_k], + [project(t, lang) for t in lang_eval_k], + ) + record(f"kernel g={gamma_scale}", keep, 0.05, value) + + best = max(rows, key=lambda row: row["correlation"]) + summary = { + "protocol": ( + "Projection fitted per modality on scenes outside the evaluated " + "set; hidden pairs read only for the correlation. Baseline is " + "linear, keep=8, shrinkage=0.05." + ), + "evaluated": len(evaluated), + "rows": rows, + "best": best, + "recovery_threshold": 0.9, + } + print(json.dumps(best)) + write_json(args.output, summary) + + +if __name__ == "__main__": + main() diff --git a/worldalign/rank_causal.py b/worldalign/rank_causal.py new file mode 100644 index 0000000..09ae55e --- /dev/null +++ b/worldalign/rank_causal.py @@ -0,0 +1,141 @@ +"""Is the effective rank of the shared field the binding variable, not its magnitude? + +Stripping ten eigen-directions costs the synthetic fields that recover almost +nothing -- 0.929 to 0.870 -- and costs the natural fields almost everything -- +0.656 to 0.080. The natural shared signal is therefore concentrated in about +ten directions while the synthetic one is spread across the spectrum. That is +a correlation between rank and recovery, and this makes it causal by +intervention: take the synthetic fields that do recover, project them to +successively lower rank, and watch recovery as a function of rank while the +correlation between the two truncated fields stays high. + +If recovery dies at low rank while the correlation is still far above the +threshold, then rank is the binding variable and the headline correlation is +the wrong target. The theory agrees in advance: correlated-matching thresholds +assume the agreement lives in exchangeable full-rank noise, so N^2/2 entries +each carry an independent constraint. A rank-r shared component supplies about +rN, and at 256 scenes with r near ten that is an eighth of what the threshold +calculation assumes. +""" + +from __future__ import annotations + +import argparse +import json + +import numpy as np +import torch + +from .common import write_json +from .spectral_match import grampa +from .synth_fast_gate import ClosedFormEnergy, all_swaps, steepest_descent +from .synth_triangle_gate import standardized + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--fields", default="artifacts/synth_v1/fields_tier0_ws_256.pt") + parser.add_argument("--label", default="synthetic-watershed") + parser.add_argument("--trials", type=int, default=3) + parser.add_argument("--device", default="cuda:3") + parser.add_argument("--output", default="artifacts/synth_v1/rank_causal.json") + return parser.parse_args() + + +def offdiagonal(matrix: np.ndarray) -> np.ndarray: + return matrix[~np.eye(len(matrix), dtype=bool)] + + +def normalise(matrix: np.ndarray) -> np.ndarray: + values = offdiagonal(matrix) + out = (matrix - values.mean()) / values.std() + np.fill_diagonal(out, 0.0) + return out + + +def truncate(matrix: np.ndarray, rank: int) -> np.ndarray: + """Keep the leading `rank` eigen-components of the symmetrised field.""" + symmetric = (matrix + matrix.T) / 2.0 + values, vectors = np.linalg.eigh(symmetric) + order = np.argsort(np.abs(values))[::-1][:rank] + return (vectors[:, order] * values[order]) @ vectors[:, order].T + + +def effective_rank(matrix: np.ndarray) -> float: + """Participation ratio of the eigen-spectrum: how many directions carry it.""" + values = np.abs(np.linalg.eigvalsh((matrix + matrix.T) / 2.0)) + weights = values / values.sum() + weights = weights[weights > 0] + return float(np.exp(-(weights * np.log(weights)).sum())) + + +def main() -> None: + args = parse_args() + state = torch.load(args.fields, map_location="cpu", weights_only=False) + visual_full = state["visual_field"].double().numpy() + text_full = state["text_field"].double().numpy() + size = len(visual_full) + device = torch.device(args.device) + swaps = all_swaps(size, device) + + rows = [] + ranks = [4, 8, 16, 32, 64, 128, size] + for rank in ranks: + visual = normalise(truncate(visual_full, rank) if rank < size else visual_full) + text = normalise(truncate(text_full, rank) if rank < size else text_full) + mask = ~np.eye(size, dtype=bool) + rho = float(np.corrcoef(visual[mask], text[mask])[0, 1]) + + accuracies = [] + for trial in range(args.trials): + generator = np.random.default_rng(trial) + hidden = generator.permutation(size) + shuffled = text[np.ix_(hidden, hidden)] + columns = grampa(visual, shuffled, 1.0) + energy = ClosedFormEnergy( + standardized(torch.from_numpy(shuffled).to(device)).float(), + standardized(torch.from_numpy(visual).to(device)).float(), + 1.0, 1.0, 256, + ) + final, _ = steepest_descent( + energy, torch.from_numpy(columns.copy()).to(device), swaps, 3000 + ) + accuracies.append( + float((hidden[final.cpu().numpy()] == np.arange(size)).mean()) + ) + row = { + "rank": rank, + "correlation": rho, + "recovery": float(np.mean(accuracies)), + "visual_effective_rank": effective_rank(visual), + } + rows.append(row) + print( + f"rank={rank:<5} rho={rho:.4f} recovery={row['recovery']:.3f} " + f"eff_rank={row['visual_effective_rank']:.1f}", + flush=True, + ) + + summary = { + "protocol": ( + "Both fields truncated to the same rank, then matched blind from a " + "spectral start with exact refinement. Hidden pairing generated " + "per trial and read only for scoring." + ), + "label": args.label, + "size": size, + "chance": 1.0 / size, + "rows": rows, + "reading": ( + "Recovery collapsing at low rank while the correlation stays above " + "the recovery threshold shows the headline correlation is not " + "sufficient and the shared spectrum's width is the binding " + "constraint." + ), + } + write_json(args.output, summary) + print(json.dumps({"chance": 1.0 / size})) + + +if __name__ == "__main__": + main() diff --git a/worldalign/shared_rank.py b/worldalign/shared_rank.py new file mode 100644 index 0000000..629ae0b --- /dev/null +++ b/worldalign/shared_rank.py @@ -0,0 +1,135 @@ +"""Which side is narrow? Per-modality rank against the shared rank. + +Truncating a recovering synthetic field to rank four leaves its correlation at +0.90 and its recovery at 6%, so the width of the shared spectrum is a separate +and binding constraint that the correlation does not express. The engineering +question follows immediately: which modality is narrow. Upgrading the vision +encoder is worth its compute only if vision is what limits the shared width, +and if the text side is the narrow one no vision backbone will help. + +Three numbers per field pair. Each field's own effective rank, from the +participation ratio of its spectrum, says how many directions the modality +uses at all. The principal angles between the two leading eigenspaces say how +many of those directions the two modalities share. And the per-direction +correlation profile says where along the spectrum agreement dies. +""" + +from __future__ import annotations + +import argparse +import json + +import numpy as np +import torch + +from .common import write_json + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--fields", nargs="+", required=True) + parser.add_argument("--labels", nargs="+", default=None) + parser.add_argument("--width", type=int, default=32) + parser.add_argument("--output", default="artifacts/vg_5k/shared_rank.json") + return parser.parse_args() + + +def spectrum(matrix: np.ndarray) -> tuple[np.ndarray, np.ndarray]: + symmetric = (matrix + matrix.T) / 2.0 + values, vectors = np.linalg.eigh(symmetric) + order = np.argsort(np.abs(values))[::-1] + return values[order], vectors[:, order] + + +def effective_rank(values: np.ndarray) -> float: + weights = np.abs(values) / np.abs(values).sum() + weights = weights[weights > 1e-15] + return float(np.exp(-(weights * np.log(weights)).sum())) + + +def analyse(path: str, label: str, width: int) -> dict: + state = torch.load(path, map_location="cpu", weights_only=False) + visual = state["visual_field"].double().numpy() + text = state["text_field"].double().numpy() + size = len(visual) + mask = ~np.eye(size, dtype=bool) + + visual_values, visual_vectors = spectrum(visual) + text_values, text_vectors = spectrum(text) + width = min(width, size) + + # principal angles between the two leading eigenspaces: cosines near one + # count as directions the two modalities agree on. The raw count inflates + # with width -- two random w-dimensional subspaces of R^N already overlap + # at cosines near sqrt(w/N) -- so a scene-shuffled null is subtracted and + # every comparison is made at matched width. + overlap = visual_vectors[:, :width].T @ text_vectors[:, :width] + cosines = np.clip(np.linalg.svd(overlap, compute_uv=False), 0.0, 1.0) + shared = int((cosines > 0.7).sum()) + + null_counts = [] + for trial in range(5): + generator = np.random.default_rng(trial) + order = generator.permutation(size) + shuffled = text[np.ix_(order, order)] + _, shuffled_vectors = spectrum(shuffled) + null_overlap = visual_vectors[:, :width].T @ shuffled_vectors[:, :width] + null_cosines = np.clip(np.linalg.svd(null_overlap, compute_uv=False), 0.0, 1.0) + null_counts.append(int((null_cosines > 0.7).sum())) + null = float(np.mean(null_counts)) + + # where along the spectrum does agreement die + profile = [] + for index in range(min(width, 16)): + visual_direction = visual_vectors[:, index] + best = float(np.abs(text_vectors[:, :width].T @ visual_direction).max()) + profile.append({"index": index, "best_text_overlap": best}) + + return { + "label": label, + "size": size, + "correlation": float(np.corrcoef(visual[mask], text[mask])[0, 1]), + "visual_effective_rank": effective_rank(visual_values), + "text_effective_rank": effective_rank(text_values), + "shared_directions_cos_gt_0.7": shared, + "shuffled_null": null, + "shared_above_null": shared - null, + "principal_cosines_top8": [round(float(c), 3) for c in cosines[:8]], + "spectrum_profile": profile, + } + + +def main() -> None: + args = parse_args() + labels = args.labels or [path.split("/")[-1] for path in args.fields] + rows = [analyse(path, label, args.width) + for path, label in zip(args.fields, labels)] + for row in rows: + print( + f"{row['label']:<26} rho={row['correlation']:.3f} " + f"eff_rank vision={row['visual_effective_rank']:6.1f} " + f"text={row['text_effective_rank']:6.1f} " + f"shared={row['shared_directions_cos_gt_0.7']:3d} " + f"(null {row['shuffled_null']:4.1f}, net {row['shared_above_null']:5.1f})", + flush=True, + ) + summary = { + "protocol": ( + "Effective rank from the participation ratio of each field's " + "spectrum; shared directions from principal angles between the " + "two leading eigenspaces. No pairing enters beyond the alignment " + "the fields already carry." + ), + "width": args.width, + "rows": rows, + "reading": ( + "The narrow modality is the one to upgrade. A shared count far " + "below both effective ranks means each side is rich but they are " + "rich about different things." + ), + } + write_json(args.output, summary) + + +if __name__ == "__main__": + main() diff --git a/worldalign/tier0_pipeline.py b/worldalign/tier0_pipeline.py index 778e20f..01c78d2 100644 --- a/worldalign/tier0_pipeline.py +++ b/worldalign/tier0_pipeline.py @@ -56,6 +56,12 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--seed", type=int, default=0) parser.add_argument("--output", required=True) parser.add_argument("--states-output", default="") + parser.add_argument( + "--text-omit", + default="", + help="Comma-separated factors the captions never state " + "(colour, count, size). Vision still sees them.", + ) return parser.parse_args() @@ -188,7 +194,17 @@ def factor_vector(colour: int, count: int, size: int, classes: int) -> torch.Ten return vector -def encode_caption(caption: str, colour_rank: dict[str, int], classes: int) -> torch.Tensor: +def encode_caption( + caption: str, colour_rank: dict[str, int], classes: int, omit: frozenset = frozenset() +) -> torch.Tensor: + """Encode a caption, optionally suppressing factors the text never states. + + Suppressing a factor makes the caption describe strictly less of the scene + while vision continues to see all of it, which is the controlled form of + the situation on photographs: a region description and a patch descriptor + overlap on some aspects and not others. It is the intervention that tests + whether corpus overlap is what sets the width of the shared spectrum. + """ vectors = [] for phrase in parse_group_phrases(caption): tokens = phrase.split() @@ -201,10 +217,20 @@ def encode_caption(caption: str, colour_rank: dict[str, int], classes: int) -> t ) ) size = 0 if "small" in tokens else (2 if "large" in tokens else 1) - vectors.append(factor_vector(colour, count, size, classes)) + vector = factor_vector(colour, count, size, classes) + if "colour" in omit: + vector[:classes] = 0.0 + if "count" in omit: + vector[classes : classes + 4] = 0.0 + if "size" in omit: + vector[classes + 4 :] = 0.0 + vectors.append(vector) if not vectors: vectors = [factor_vector(0, 1, 1, classes)] - return F.normalize(torch.stack(vectors), dim=-1) + stacked = torch.stack(vectors) + if stacked.abs().sum() == 0: + stacked = stacked + 1.0 + return F.normalize(stacked, dim=-1) def moment_state(states: torch.Tensor) -> torch.Tensor: @@ -264,7 +290,10 @@ def main() -> None: view_states = [moment_state(item) for item in sets] visual_field = torch.stack(per_view_fields).mean(0) - text_sets = [encode_caption(captions[row][0], colour_rank, classes) for row in rows] + omit = frozenset(f for f in args.text_omit.split(",") if f) + text_sets = [ + encode_caption(captions[row][0], colour_rank, classes, omit) for row in rows + ] text_field = moment_field(text_sets) mask = ~np.eye(len(rows), dtype=bool) -- cgit v1.2.3