test: falsify oral A recovery boundaries

3 files changed, 154 insertions, 57 deletions

diff --git a/experiments/analyze_oral_a_dynamic_scaling.py b/experiments/analyze_oral_a_dynamic_scaling.py
index ccd4fa9..38ce967 100755
--- a/experiments/analyze_oral_a_dynamic_scaling.py
+++ b/experiments/analyze_oral_a_dynamic_scaling.py
@@ -199,6 +199,70 @@ def dynamic_invariants(record, depth, seed, failures):
     }
 
 
+def oral_a_checks(accuracies, alignments, failures):
+    """Apply the frozen score gate to already validated paired arrays."""
+    bp_deficits = {depth: [bp - dynamic for bp, dynamic in zip(
+        accuracies["bp"][depth], accuracies["dynamic"][depth])]
+        for depth in DEPTHS}
+    kp_deficits = {depth: [kp - dynamic for kp, dynamic in zip(
+        accuracies["clean_kp"][depth], accuracies["dynamic"][depth])]
+        for depth in DEPTHS}
+    d56_dfa_advantage = [dynamic - dfa for dynamic, dfa in zip(
+        accuracies["dynamic"][56], accuracies["dfa"][56])]
+    dynamic_depth_gain = [deep - shallow for shallow, deep in zip(
+        accuracies["dynamic"][20], accuracies["dynamic"][56])]
+    bp_depth_gain = [deep - shallow for shallow, deep in zip(
+        accuracies["bp"][20], accuracies["bp"][56])]
+    checks = {
+        "all_60_records_and_audited_values_finite": not failures,
+        "bp_mean_accuracy_at_least_0p90_each_depth": all(
+            statistics.mean(accuracies["bp"][depth]) >= 0.90
+            for depth in DEPTHS),
+        "every_dynamic_accuracy_at_least_0p87": min(
+            value for depth in DEPTHS
+            for value in accuracies["dynamic"][depth]) >= 0.87,
+        "dynamic_mean_within_2pt_bp_each_depth": all(
+            statistics.mean(bp_deficits[depth]) <= 0.02 for depth in DEPTHS),
+        "dynamic_bp_deficit_upper_bound_at_most_3pt_each_depth": all(
+            upper_confidence_bound(bp_deficits[depth]) <= 0.03
+            for depth in DEPTHS),
+        "dynamic_mean_within_1p5pt_kp_each_depth": all(
+            statistics.mean(kp_deficits[depth]) <= 0.015 for depth in DEPTHS),
+        "dynamic_kp_deficit_upper_bound_at_most_2p5pt_each_depth": all(
+            upper_confidence_bound(kp_deficits[depth]) <= 0.025
+            for depth in DEPTHS),
+        "d56_dynamic_mean_advantage_over_dfa_at_least_2pt":
+            statistics.mean(d56_dfa_advantage) >= 0.02,
+        "d56_dynamic_dfa_advantage_lower_bound_at_least_1pt":
+            lower_confidence_bound(d56_dfa_advantage) >= 0.01,
+        "dynamic_mean_d20_to_d56_gain_at_least_0p5pt":
+            statistics.mean(dynamic_depth_gain) >= 0.005,
+        "dynamic_depth_gain_lower_bound_nonnegative":
+            lower_confidence_bound(dynamic_depth_gain) >= 0.0,
+        "at_least_four_dynamic_seeds_improve_with_depth":
+            sum(value > 0 for value in dynamic_depth_gain) >= 4,
+        "dynamic_depth_gain_within_1pt_of_bp_gain":
+            statistics.mean(dynamic_depth_gain)
+            >= statistics.mean(bp_depth_gain) - 0.01,
+        "dynamic_mean_alignment_at_least_0p85_each_depth": all(
+            statistics.mean(alignments[depth]) >= 0.85 for depth in DEPTHS),
+        "every_d56_dynamic_alignment_at_least_0p80":
+            min(alignments[56]) >= 0.80,
+        "d56_mean_alignment_retains_90pct_of_d20":
+            statistics.mean(alignments[56])
+            >= 0.90 * statistics.mean(alignments[20]),
+        "all_mechanism_query_cost_memory_invariants": not failures,
+    }
+    derived = {
+        "bp_deficit": bp_deficits,
+        "kp_deficit": kp_deficits,
+        "d56_dfa_advantage": d56_dfa_advantage,
+        "dynamic_d20_to_d56_gain": dynamic_depth_gain,
+        "bp_d20_to_d56_gain": bp_depth_gain,
+    }
+    return checks, derived
+
+
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument(
@@ -312,58 +376,7 @@ def main():
                 for seed in SEEDS]
         for depth in DEPTHS
     }
-    bp_deficits = {depth: [bp - dynamic for bp, dynamic in zip(
-        accuracies["bp"][depth], accuracies["dynamic"][depth])]
-        for depth in DEPTHS}
-    kp_deficits = {depth: [kp - dynamic for kp, dynamic in zip(
-        accuracies["clean_kp"][depth], accuracies["dynamic"][depth])]
-        for depth in DEPTHS}
-    d56_dfa_advantage = [dynamic - dfa for dynamic, dfa in zip(
-        accuracies["dynamic"][56], accuracies["dfa"][56])]
-    dynamic_depth_gain = [deep - shallow for shallow, deep in zip(
-        accuracies["dynamic"][20], accuracies["dynamic"][56])]
-    bp_depth_gain = [deep - shallow for shallow, deep in zip(
-        accuracies["bp"][20], accuracies["bp"][56])]
-    checks = {
-        "all_60_records_and_audited_values_finite": not failures,
-        "bp_mean_accuracy_at_least_0p90_each_depth": all(
-            statistics.mean(accuracies["bp"][depth]) >= 0.90
-            for depth in DEPTHS),
-        "every_dynamic_accuracy_at_least_0p87": min(
-            value for depth in DEPTHS
-            for value in accuracies["dynamic"][depth]) >= 0.87,
-        "dynamic_mean_within_2pt_bp_each_depth": all(
-            statistics.mean(bp_deficits[depth]) <= 0.02 for depth in DEPTHS),
-        "dynamic_bp_deficit_upper_bound_at_most_3pt_each_depth": all(
-            upper_confidence_bound(bp_deficits[depth]) <= 0.03
-            for depth in DEPTHS),
-        "dynamic_mean_within_1p5pt_kp_each_depth": all(
-            statistics.mean(kp_deficits[depth]) <= 0.015 for depth in DEPTHS),
-        "dynamic_kp_deficit_upper_bound_at_most_2p5pt_each_depth": all(
-            upper_confidence_bound(kp_deficits[depth]) <= 0.025
-            for depth in DEPTHS),
-        "d56_dynamic_mean_advantage_over_dfa_at_least_2pt":
-            statistics.mean(d56_dfa_advantage) >= 0.02,
-        "d56_dynamic_dfa_advantage_lower_bound_at_least_1pt":
-            lower_confidence_bound(d56_dfa_advantage) >= 0.01,
-        "dynamic_mean_d20_to_d56_gain_at_least_0p5pt":
-            statistics.mean(dynamic_depth_gain) >= 0.005,
-        "dynamic_depth_gain_lower_bound_nonnegative":
-            lower_confidence_bound(dynamic_depth_gain) >= 0.0,
-        "at_least_four_dynamic_seeds_improve_with_depth":
-            sum(value > 0 for value in dynamic_depth_gain) >= 4,
-        "dynamic_depth_gain_within_1pt_of_bp_gain":
-            statistics.mean(dynamic_depth_gain)
-            >= statistics.mean(bp_depth_gain) - 0.01,
-        "dynamic_mean_alignment_at_least_0p85_each_depth": all(
-            statistics.mean(alignments[depth]) >= 0.85 for depth in DEPTHS),
-        "every_d56_dynamic_alignment_at_least_0p80":
-            min(alignments[56]) >= 0.80,
-        "d56_mean_alignment_retains_90pct_of_d20":
-            statistics.mean(alignments[56])
-            >= 0.90 * statistics.mean(alignments[20]),
-        "all_mechanism_query_cost_memory_invariants": not failures,
-    }
+    checks, derived = oral_a_checks(accuracies, alignments, failures)
     passed = all(checks.values())
     output = {
         "protocol": "oral_a_dynamic_innovation_scaling_v1",
@@ -376,11 +389,7 @@ def main():
                 str(depth): statistics.mean(values)
                 for depth, values in by_depth.items()}
                 for method, by_depth in accuracies.items()},
-            "bp_deficit": bp_deficits,
-            "kp_deficit": kp_deficits,
-            "d56_dfa_advantage": d56_dfa_advantage,
-            "dynamic_d20_to_d56_gain": dynamic_depth_gain,
-            "bp_d20_to_d56_gain": bp_depth_gain,
+            **derived,
             "dynamic_alignment": alignments,
             "mechanism": mechanism,
             "invariant_failures": failures,
diff --git a/experiments/finalize_accept.sh b/experiments/finalize_accept.sh
index 95f95e9..4fd8d6b 100755
--- a/experiments/finalize_accept.sh
+++ b/experiments/finalize_accept.sh
@@ -13,6 +13,8 @@ experiments/finalize_claims.sh
   experiments/conv_local_smoke.py
 /home/yurenh2/miniconda3/envs/ep_pascal/bin/python3 \
   experiments/bci_smoke.py
+/home/yurenh2/miniconda3/envs/ep_pascal/bin/python3 \
+  experiments/oral_a_dynamic_scaling_smoke.py
 /home/yurenh2/miniconda3/envs/ep_pascal/bin/python3 -m py_compile \
   experiments/bci_td_run.py experiments/analyze_bci_td_development.py \
   experiments/bci_td_confirmation.py \
diff --git a/experiments/oral_a_dynamic_scaling_smoke.py b/experiments/oral_a_dynamic_scaling_smoke.py
new file mode 100755
index 0000000..a194557
--- /dev/null
+++ b/experiments/oral_a_dynamic_scaling_smoke.py
@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+"""Endpoint-free contract and boundary checks for the oral-A recovery."""
+import copy
+import os
+import sys
+from collections import Counter
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from experiments.analyze_oral_a_dynamic_scaling import (
+    DEPTHS, expected_args, oral_a_checks)
+from experiments.conv_run import parse_args
+from experiments.oral_a_dynamic_scaling import jobs
+
+
+def check_job_contract():
+    cells = jobs("cuda", "results/oral_a_dynamic_scaling")
+    assert len(cells) == 50
+    assert len({(method, depth, seed)
+                for method, depth, seed, _, _ in cells}) == 50
+    assert Counter(method for method, _, _, _, _ in cells) == {
+        "bp": 15, "dfa": 15, "clean_kp": 10, "dynamic": 10}
+    assert Counter(depth for _, depth, _, _, _ in cells) == {
+        20: 10, 32: 20, 56: 20}
+    for method, depth, seed, path, command in cells:
+        saved = sys.argv
+        sys.argv = command[1:]
+        args = parse_args()
+        sys.argv = saved
+        actual = vars(args)
+        for key, value in expected_args(method, depth, seed).items():
+            assert actual[key] == value, (
+                method, depth, seed, key, actual[key], value)
+        assert command[-2:] == ["--out", path]
+    print("oral-A recovery runner/analyzer contract: exact 50 new cells")
+
+
+def passing_arrays():
+    offsets = [-0.002, -0.001, 0.0, 0.001, 0.002]
+    means = {
+        "bp": {20: 0.915, 32: 0.925, 56: 0.935},
+        "dfa": {20: 0.40, 32: 0.40, 56: 0.40},
+        "clean_kp": {20: 0.912, 32: 0.922, 56: 0.932},
+        "dynamic": {20: 0.908, 32: 0.918, 56: 0.928},
+    }
+    accuracies = {method: {
+        depth: [mean + offset for offset in offsets]
+        for depth, mean in by_depth.items()}
+        for method, by_depth in means.items()}
+    alignments = {depth: [0.95 + offset for offset in offsets]
+                  for depth in DEPTHS}
+    return accuracies, alignments
+
+
+def check_gate_boundaries():
+    accuracies, alignments = passing_arrays()
+    checks, _ = oral_a_checks(accuracies, alignments, [])
+    assert all(checks.values())
+
+    collapsed = copy.deepcopy(accuracies)
+    for method in collapsed:
+        for depth in DEPTHS:
+            collapsed[method][depth] = [0.10] * 5
+    collapsed_checks, _ = oral_a_checks(collapsed, alignments, [])
+    assert not collapsed_checks["bp_mean_accuracy_at_least_0p90_each_depth"]
+    assert not collapsed_checks["every_dynamic_accuracy_at_least_0p87"]
+    assert collapsed_checks["dynamic_mean_within_2pt_bp_each_depth"]
+
+    flat = copy.deepcopy(accuracies)
+    flat["dynamic"][56] = list(flat["dynamic"][20])
+    flat_checks, _ = oral_a_checks(flat, alignments, [])
+    assert not flat_checks["dynamic_mean_d20_to_d56_gain_at_least_0p5pt"]
+    assert not flat_checks["at_least_four_dynamic_seeds_improve_with_depth"]
+
+    invariant_checks, _ = oral_a_checks(
+        accuracies, alignments, ["synthetic:cost_failure"])
+    assert not invariant_checks[
+        "all_mechanism_query_cost_memory_invariants"]
+    assert not invariant_checks["all_60_records_and_audited_values_finite"]
+    print("oral-A recovery absolute, depth-benefit, and invariant gates: falsifiable")
+
+
+if __name__ == "__main__":
+    check_job_contract()
+    check_gate_boundaries()
+    print("ALL ORAL-A RECOVERY MECHANICS CHECKS PASSED")