analysis: validate the frozen CLLN panel

1 files changed, 59 insertions, 0 deletions

diff --git a/experiments/analyze_coupled_ladder_scaling.py b/experiments/analyze_coupled_ladder_scaling.py
index 456b6ed..bfa31f0 100644
--- a/experiments/analyze_coupled_ladder_scaling.py
+++ b/experiments/analyze_coupled_ladder_scaling.py
@@ -35,6 +35,17 @@ STYLE = {
     "sdil": dict(color="#0072B2", marker="o", linestyle="-"),
 }
 
+FROZEN_SIZES = (4, 8, 12, 16, 24, 32)
+FROZEN_DEVICE_SEEDS = (20260830, 20260831, 20260832)
+FROZEN_LEARNING_TIMES = {
+    4: 0.01,
+    8: 0.01,
+    12: 0.03,
+    16: 0.03,
+    24: 0.03,
+    32: 0.01,
+}
+
 
 def parse_args() -> argparse.Namespace:
     parser = argparse.ArgumentParser()
@@ -90,6 +101,52 @@ def merge_reports(reports: list[dict]) -> list[dict]:
     return merged
 
 
+def validate_core_confirmation(reports: list[dict], records: list[dict]) -> None:
+    """Reject a confirmatory label unless the frozen core panel is complete."""
+    expected_cells = {
+        (side, task, seed)
+        for side in FROZEN_SIZES
+        for task in range(40)
+        for seed in FROZEN_DEVICE_SEEDS
+    }
+    actual_cells = {
+        (record["side"], record["task_index"], record["device_seed"])
+        for record in records
+    }
+    if actual_cells != expected_cells:
+        missing = len(expected_cells - actual_cells)
+        extra = len(actual_cells - expected_cells)
+        raise ValueError(
+            f"incomplete frozen confirmation: {missing} missing, {extra} extra cells")
+    required_methods = set(METHOD_ORDER)
+    for record in records:
+        if set(record["methods"]) != required_methods:
+            raise ValueError("confirmatory cells must contain all five core methods")
+        if any(
+            method["status"] != "completed"
+            for method in record["methods"].values()
+        ):
+            raise ValueError("confirmatory cells must complete without failures")
+    observed_sizes = set()
+    for report in reports:
+        protocol = report["protocol"]
+        if protocol["rotations_per_input_diameter"] != 8:
+            raise ValueError("confirmation requires all eight task rotations")
+        if protocol["epochs"] != 600:
+            raise ValueError("confirmation requires the frozen 600 epochs")
+        if tuple(protocol["device_seeds"]) != FROZEN_DEVICE_SEEDS:
+            raise ValueError("confirmation device seeds do not match the frozen panel")
+        if set(protocol["methods"]) != required_methods:
+            raise ValueError("confirmation methods do not match the frozen core panel")
+        for side in protocol["sizes"]:
+            observed_sizes.add(side)
+            observed = float(protocol["learning_time_seconds_by_side"][str(side)])
+            if abs(observed - FROZEN_LEARNING_TIMES[side]) > 1e-15:
+                raise ValueError(f"side {side} uses an unfrozen learning exposure")
+    if observed_sizes != set(FROZEN_SIZES):
+        raise ValueError("confirmation sources do not cover all frozen sizes")
+
+
 def task_cluster_values(
     records: list[dict], side: int, method: str, metric: str
 ) -> tuple[np.ndarray, np.ndarray]:
@@ -502,6 +559,8 @@ def main() -> None:
     source_paths = list(args.core) + list(args.baselines)
     reports = [json.loads(path.read_text()) for path in source_paths]
     records = merge_reports(reports)
+    if args.confirmatory:
+        validate_core_confirmation(reports, records)
     analysis = build_analysis(
         records,
         replicates=args.bootstrap_replicates,