diff options
| -rw-r--r-- | RANK_RESULTS.md | 67 | ||||
| -rw-r--r-- | artifacts/vg_5k/anchor_bound.json | 66 | ||||
| -rw-r--r-- | logs/rho_full_width.log | 1 | ||||
| -rw-r--r-- | worldalign/anchor_bound.py | 139 |
4 files changed, 271 insertions, 2 deletions
diff --git a/RANK_RESULTS.md b/RANK_RESULTS.md index d520e5e..ffee8c9 100644 --- a/RANK_RESULTS.md +++ b/RANK_RESULTS.md @@ -1,7 +1,16 @@ # The gate was measuring the wrong thing -*2026-08-01. Supersedes the correlation threshold used as the project's -go/no-go statistic since the synthetic world results.* +*2026-08-01. Retires the correlation threshold that has been the project's +go/no-go statistic since the synthetic world results, records two replacement +statistics that were proposed and refuted the same day, and settles on a third +that survives its controls: an upper bound obtained by giving the solver half +the correspondence and asking it to recover the rest.* + +**Result in one line.** The correlation does not govern recovery in either +direction; what does is whether a representation can identify scenes at all, +measured by matching against anchors whose correspondence is given. Visual +Genome scores 0.291 on that bound against 0.98 for the synthetic world that +recovers, and its blind recovery is 0.0000 against 0.0039 chance. ## The correlation does not govern recovery @@ -102,6 +111,60 @@ than everything found (recoverable), tied with what is found (symmetry), or shallower than what is found (information deficit). The gate is running on both fields; the sections below should be read against this correction. +## Resolution: ask the matching question, and give one thing away + +Three statistics failed the same way — fields agreeing on the number and +disagreeing on recovery — and the pattern in the failures is that each was +invented by staring at the fields rather than by asking what matching needs. +The one that works asks directly, with a single concession. + +Declare half the scenes anchors and hand over their correspondence. Describe +every remaining scene, in each modality, by its field row against the anchors. +Match the two descriptions one-to-one by Hungarian assignment. Nothing is +searched; the answer is a linear program, and it costs seconds. + +This **upper-bounds blind recovery**, because blind recovery must additionally +discover the anchor correspondence. It held in every field measured: + +| Field | ρ | anchor bound | blind recovery | search gap | +|---|---|---|---|---| +| synthetic, full | 0.928 | 0.989 | 0.958 | +0.031 | +| synthetic, noise 0.35 | 0.828 | 0.984 | 0.947 | +0.037 | +| synthetic, caption omits size | 0.830 | **0.997** | 0.056 | **+0.941** | +| synthetic, rank 8 | 0.906 | 0.930 | 0.129 | +0.801 | +| Visual Genome, best | 0.731 | **0.291** | 0.000 | +0.291 | +| Visual Genome, baseline | 0.656 | 0.195 | 0.000 | +0.195 | + +**The bound splits the deficit in two, and the two halves call for opposite +work.** The bound itself says whether the representation identifies scenes at +all; the gap between bound and blind recovery says whether the search can find +what the representation contains. + +Read down the table and the three regimes are unmistakable. The two recovering +fields have a bound near one and a gap of three points — representation good, +search working. Caption-omission and rank-8 truncation have a bound of 0.93 to +**0.997** and lose almost everything to the search: the information is intact, +perfectly so in the omission case, and blind matching cannot reach it. Visual +Genome is the opposite failure — its bound is 0.291, so **even with the +correspondence handed over, seven scenes in ten cannot be identified**, and no +solver improvement addresses that. + +This corrects two things I wrote earlier today. The caption-suppression ladder +does not show information being destroyed; it shows blind searchability being +destroyed while the information stays. Suppressing a factor makes scenes that +differed only in that factor interchangeable *under permutation*, which the +given anchors break and blind matching does not. And Visual Genome's problem +was never the width of its shared spectrum — it is that a region description +and a patch descriptor, compared through relation profiles, identify a +photograph about 29% of the time. + +The bound is an empirical upper bound rather than a proven one: handing over +anchors removes search but also changes the estimation problem. It was not +violated in six fields spanning the full range of outcomes, and unlike the +three refuted statistics it separates every case they collapsed. It is now the +project's cheap gate, and a representation that cannot pass it should not be +searched. + ## What the shared spectrum measures Each field is eigendecomposed and the principal angles between the two leading diff --git a/artifacts/vg_5k/anchor_bound.json b/artifacts/vg_5k/anchor_bound.json new file mode 100644 index 0000000..2de1390 --- /dev/null +++ b/artifacts/vg_5k/anchor_bound.json @@ -0,0 +1,66 @@ +{ + "protocol": "Half the scenes are anchors with their correspondence given. The rest are described by their field rows against the anchors and matched by Hungarian assignment. Upper-bounds blind recovery, which must additionally discover the anchor correspondence.", + "rows": [ + { + "label": "synth-full", + "correlation": 0.928480988366389, + "probe_scenes": 128, + "chance": 0.0078125, + "nearest_neighbour": 0.9375, + "anchor_bound": 0.9890625, + "blind_recovery": 0.958, + "search_gap": 0.031062499999999993 + }, + { + "label": "synth-noise035", + "correlation": 0.8276655471168534, + "probe_scenes": 128, + "chance": 0.0078125, + "nearest_neighbour": 0.896875, + "anchor_bound": 0.984375, + "blind_recovery": 0.947, + "search_gap": 0.03737500000000005 + }, + { + "label": "synth-omitsize", + "correlation": 0.8301825436675786, + "probe_scenes": 128, + "chance": 0.0078125, + "nearest_neighbour": 0.89375, + "anchor_bound": 0.996875, + "blind_recovery": 0.056, + "search_gap": 0.9408749999999999 + }, + { + "label": "synth-rank8", + "correlation": 0.9058447537972103, + "probe_scenes": 128, + "chance": 0.0078125, + "nearest_neighbour": 0.746875, + "anchor_bound": 0.9296875, + "blind_recovery": 0.129, + "search_gap": 0.8006875 + }, + { + "label": "natural-best", + "correlation": 0.7309253746041517, + "probe_scenes": 128, + "chance": 0.0078125, + "nearest_neighbour": 0.1671875, + "anchor_bound": 0.290625, + "blind_recovery": 0.0, + "search_gap": 0.290625 + }, + { + "label": "natural-baseline", + "correlation": 0.6558829559111277, + "probe_scenes": 128, + "chance": 0.0078125, + "nearest_neighbour": 0.1296875, + "anchor_bound": 0.1953125, + "blind_recovery": 0.0, + "search_gap": 0.1953125 + } + ], + "reading": "A low bound means the representation does not identify scenes and no solver will help. A high bound with low blind recovery means the information is present and the search or the landscape is at fault." +}
\ No newline at end of file diff --git a/logs/rho_full_width.log b/logs/rho_full_width.log index b81ce33..b33469d 100644 --- a/logs/rho_full_width.log +++ b/logs/rho_full_width.log @@ -2,3 +2,4 @@ noise=0.0 rho=0.928 shared= 21.0 recovery=0.961 noise=0.2 rho=0.893 shared= 18.0 recovery=0.971 noise=0.35 rho=0.828 shared= 15.7 recovery=0.947 noise=0.5 rho=0.743 shared= 13.3 recovery=0.707 +noise=0.7 rho=0.624 shared= 11.3 recovery=0.201 diff --git a/worldalign/anchor_bound.py b/worldalign/anchor_bound.py new file mode 100644 index 0000000..49f4ed3 --- /dev/null +++ b/worldalign/anchor_bound.py @@ -0,0 +1,139 @@ +"""A cheap gate that survives its controls, built as an upper bound. + +Three statistics have now been proposed as go/no-go predictors and three have +failed: the field correlation, the width of the shared spectrum, and the +held-out shared dimension. Each failed the same way -- fields agreeing on the +statistic and disagreeing on recovery. The lesson is that a predictor invented +by staring at the fields keeps finding quantities that are not about matching. + +So this asks the matching question directly, with one thing given away. Half +the scenes are declared anchors and their correspondence is handed over. Every +remaining scene is described, in each modality, by its field row against the +anchors, and the two descriptions are matched one-to-one by Hungarian +assignment. Nothing is searched: the answer is read off a linear program. + +The result upper-bounds what blind recovery can achieve, because blind recovery +must discover the anchor correspondence as well. It is therefore a **necessary +condition**: a representation that cannot be matched with the anchors given +will not be matched without them, and it can be rejected in seconds rather than +after a search. It is an empirical bound rather than a proven one -- the given +anchors also change the estimation problem, not only remove search -- but it +held in every field measured, and unlike the three refuted statistics it +separates the cases they collapse. + +It also splits the deficit in two, which is what makes it diagnostic rather +than merely predictive. The bound itself measures whether the representation +identifies scenes at all. The gap between the bound and blind recovery measures +whether the search can find what the representation contains. +""" + +from __future__ import annotations + +import argparse +import json + +import numpy as np +import torch +from scipy.optimize import linear_sum_assignment + +from .common import write_json + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--fields", nargs="+", required=True) + parser.add_argument("--labels", nargs="+", default=None) + parser.add_argument("--repeats", type=int, default=5) + parser.add_argument( + "--blind", type=float, nargs="*", default=None, + help="Known blind recovery per field, to report the search gap.", + ) + parser.add_argument("--output", default="artifacts/vg_5k/anchor_bound.json") + return parser.parse_args() + + +def standardise(matrix: np.ndarray) -> np.ndarray: + values = matrix[~np.eye(len(matrix), dtype=bool)] + out = (matrix - values.mean()) / values.std() + np.fill_diagonal(out, 0.0) + return out + + +def rows_against(field: np.ndarray, probe: np.ndarray, anchors: np.ndarray) -> np.ndarray: + block = field[np.ix_(probe, anchors)] + block = block - block.mean(1, keepdims=True) + return block / block.std(1, keepdims=True).clip(1e-9) + + +def analyse(path: str, label: str, repeats: int, blind: float | None) -> dict: + state = torch.load(path, map_location="cpu", weights_only=False) + visual = standardise(state["visual_field"].double().numpy()) + text = standardise(state["text_field"].double().numpy()) + size = len(visual) + mask = ~np.eye(size, dtype=bool) + rho = float(np.corrcoef(visual[mask], text[mask])[0, 1]) + + nearest, assigned = [], [] + for repeat in range(repeats): + generator = np.random.default_rng(repeat) + order = generator.permutation(size) + half = size // 2 + anchors, probe = order[:half], order[half:] + left = rows_against(visual, probe, anchors) + right = rows_against(text, probe, anchors) + similarity = left @ right.T / left.shape[1] + truth = np.arange(len(probe)) + nearest.append(float((similarity.argmax(1) == truth).mean())) + _, columns = linear_sum_assignment(-similarity) + assigned.append(float((columns == truth).mean())) + + row = { + "label": label, + "correlation": rho, + "probe_scenes": size - size // 2, + "chance": 1.0 / (size - size // 2), + "nearest_neighbour": float(np.mean(nearest)), + "anchor_bound": float(np.mean(assigned)), + } + if blind is not None: + row["blind_recovery"] = blind + row["search_gap"] = float(np.mean(assigned)) - blind + return row + + +def main() -> None: + args = parse_args() + labels = args.labels or [path.split("/")[-1] for path in args.fields] + blinds = args.blind if args.blind else [None] * len(args.fields) + rows = [] + for path, label, blind in zip(args.fields, labels, blinds): + row = analyse(path, label, args.repeats, blind) + rows.append(row) + tail = ( + f" blind={row['blind_recovery']:.3f} gap={row['search_gap']:+.3f}" + if "blind_recovery" in row else "" + ) + print( + f"{row['label']:<26} rho={row['correlation']:.3f} " + f"bound={row['anchor_bound']:.3f}{tail}", + flush=True, + ) + write_json(args.output, { + "protocol": ( + "Half the scenes are anchors with their correspondence given. The " + "rest are described by their field rows against the anchors and " + "matched by Hungarian assignment. Upper-bounds blind recovery, " + "which must additionally discover the anchor correspondence." + ), + "rows": rows, + "reading": ( + "A low bound means the representation does not identify scenes and " + "no solver will help. A high bound with low blind recovery means " + "the information is present and the search or the landscape is at " + "fault." + ), + }) + + +if __name__ == "__main__": + main() |