test: falsify oral A recovery boundaries

1 files changed, 66 insertions, 57 deletions

diff --git a/experiments/analyze_oral_a_dynamic_scaling.py b/experiments/analyze_oral_a_dynamic_scaling.py
index ccd4fa9..38ce967 100755
--- a/experiments/analyze_oral_a_dynamic_scaling.py
+++ b/experiments/analyze_oral_a_dynamic_scaling.py
@@ -199,6 +199,70 @@ def dynamic_invariants(record, depth, seed, failures):
     }
 
 
+def oral_a_checks(accuracies, alignments, failures):
+    """Apply the frozen score gate to already validated paired arrays."""
+    bp_deficits = {depth: [bp - dynamic for bp, dynamic in zip(
+        accuracies["bp"][depth], accuracies["dynamic"][depth])]
+        for depth in DEPTHS}
+    kp_deficits = {depth: [kp - dynamic for kp, dynamic in zip(
+        accuracies["clean_kp"][depth], accuracies["dynamic"][depth])]
+        for depth in DEPTHS}
+    d56_dfa_advantage = [dynamic - dfa for dynamic, dfa in zip(
+        accuracies["dynamic"][56], accuracies["dfa"][56])]
+    dynamic_depth_gain = [deep - shallow for shallow, deep in zip(
+        accuracies["dynamic"][20], accuracies["dynamic"][56])]
+    bp_depth_gain = [deep - shallow for shallow, deep in zip(
+        accuracies["bp"][20], accuracies["bp"][56])]
+    checks = {
+        "all_60_records_and_audited_values_finite": not failures,
+        "bp_mean_accuracy_at_least_0p90_each_depth": all(
+            statistics.mean(accuracies["bp"][depth]) >= 0.90
+            for depth in DEPTHS),
+        "every_dynamic_accuracy_at_least_0p87": min(
+            value for depth in DEPTHS
+            for value in accuracies["dynamic"][depth]) >= 0.87,
+        "dynamic_mean_within_2pt_bp_each_depth": all(
+            statistics.mean(bp_deficits[depth]) <= 0.02 for depth in DEPTHS),
+        "dynamic_bp_deficit_upper_bound_at_most_3pt_each_depth": all(
+            upper_confidence_bound(bp_deficits[depth]) <= 0.03
+            for depth in DEPTHS),
+        "dynamic_mean_within_1p5pt_kp_each_depth": all(
+            statistics.mean(kp_deficits[depth]) <= 0.015 for depth in DEPTHS),
+        "dynamic_kp_deficit_upper_bound_at_most_2p5pt_each_depth": all(
+            upper_confidence_bound(kp_deficits[depth]) <= 0.025
+            for depth in DEPTHS),
+        "d56_dynamic_mean_advantage_over_dfa_at_least_2pt":
+            statistics.mean(d56_dfa_advantage) >= 0.02,
+        "d56_dynamic_dfa_advantage_lower_bound_at_least_1pt":
+            lower_confidence_bound(d56_dfa_advantage) >= 0.01,
+        "dynamic_mean_d20_to_d56_gain_at_least_0p5pt":
+            statistics.mean(dynamic_depth_gain) >= 0.005,
+        "dynamic_depth_gain_lower_bound_nonnegative":
+            lower_confidence_bound(dynamic_depth_gain) >= 0.0,
+        "at_least_four_dynamic_seeds_improve_with_depth":
+            sum(value > 0 for value in dynamic_depth_gain) >= 4,
+        "dynamic_depth_gain_within_1pt_of_bp_gain":
+            statistics.mean(dynamic_depth_gain)
+            >= statistics.mean(bp_depth_gain) - 0.01,
+        "dynamic_mean_alignment_at_least_0p85_each_depth": all(
+            statistics.mean(alignments[depth]) >= 0.85 for depth in DEPTHS),
+        "every_d56_dynamic_alignment_at_least_0p80":
+            min(alignments[56]) >= 0.80,
+        "d56_mean_alignment_retains_90pct_of_d20":
+            statistics.mean(alignments[56])
+            >= 0.90 * statistics.mean(alignments[20]),
+        "all_mechanism_query_cost_memory_invariants": not failures,
+    }
+    derived = {
+        "bp_deficit": bp_deficits,
+        "kp_deficit": kp_deficits,
+        "d56_dfa_advantage": d56_dfa_advantage,
+        "dynamic_d20_to_d56_gain": dynamic_depth_gain,
+        "bp_d20_to_d56_gain": bp_depth_gain,
+    }
+    return checks, derived
+
+
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument(
@@ -312,58 +376,7 @@ def main():
                 for seed in SEEDS]
         for depth in DEPTHS
     }
-    bp_deficits = {depth: [bp - dynamic for bp, dynamic in zip(
-        accuracies["bp"][depth], accuracies["dynamic"][depth])]
-        for depth in DEPTHS}
-    kp_deficits = {depth: [kp - dynamic for kp, dynamic in zip(
-        accuracies["clean_kp"][depth], accuracies["dynamic"][depth])]
-        for depth in DEPTHS}
-    d56_dfa_advantage = [dynamic - dfa for dynamic, dfa in zip(
-        accuracies["dynamic"][56], accuracies["dfa"][56])]
-    dynamic_depth_gain = [deep - shallow for shallow, deep in zip(
-        accuracies["dynamic"][20], accuracies["dynamic"][56])]
-    bp_depth_gain = [deep - shallow for shallow, deep in zip(
-        accuracies["bp"][20], accuracies["bp"][56])]
-    checks = {
-        "all_60_records_and_audited_values_finite": not failures,
-        "bp_mean_accuracy_at_least_0p90_each_depth": all(
-            statistics.mean(accuracies["bp"][depth]) >= 0.90
-            for depth in DEPTHS),
-        "every_dynamic_accuracy_at_least_0p87": min(
-            value for depth in DEPTHS
-            for value in accuracies["dynamic"][depth]) >= 0.87,
-        "dynamic_mean_within_2pt_bp_each_depth": all(
-            statistics.mean(bp_deficits[depth]) <= 0.02 for depth in DEPTHS),
-        "dynamic_bp_deficit_upper_bound_at_most_3pt_each_depth": all(
-            upper_confidence_bound(bp_deficits[depth]) <= 0.03
-            for depth in DEPTHS),
-        "dynamic_mean_within_1p5pt_kp_each_depth": all(
-            statistics.mean(kp_deficits[depth]) <= 0.015 for depth in DEPTHS),
-        "dynamic_kp_deficit_upper_bound_at_most_2p5pt_each_depth": all(
-            upper_confidence_bound(kp_deficits[depth]) <= 0.025
-            for depth in DEPTHS),
-        "d56_dynamic_mean_advantage_over_dfa_at_least_2pt":
-            statistics.mean(d56_dfa_advantage) >= 0.02,
-        "d56_dynamic_dfa_advantage_lower_bound_at_least_1pt":
-            lower_confidence_bound(d56_dfa_advantage) >= 0.01,
-        "dynamic_mean_d20_to_d56_gain_at_least_0p5pt":
-            statistics.mean(dynamic_depth_gain) >= 0.005,
-        "dynamic_depth_gain_lower_bound_nonnegative":
-            lower_confidence_bound(dynamic_depth_gain) >= 0.0,
-        "at_least_four_dynamic_seeds_improve_with_depth":
-            sum(value > 0 for value in dynamic_depth_gain) >= 4,
-        "dynamic_depth_gain_within_1pt_of_bp_gain":
-            statistics.mean(dynamic_depth_gain)
-            >= statistics.mean(bp_depth_gain) - 0.01,
-        "dynamic_mean_alignment_at_least_0p85_each_depth": all(
-            statistics.mean(alignments[depth]) >= 0.85 for depth in DEPTHS),
-        "every_d56_dynamic_alignment_at_least_0p80":
-            min(alignments[56]) >= 0.80,
-        "d56_mean_alignment_retains_90pct_of_d20":
-            statistics.mean(alignments[56])
-            >= 0.90 * statistics.mean(alignments[20]),
-        "all_mechanism_query_cost_memory_invariants": not failures,
-    }
+    checks, derived = oral_a_checks(accuracies, alignments, failures)
     passed = all(checks.values())
     output = {
         "protocol": "oral_a_dynamic_innovation_scaling_v1",
@@ -376,11 +389,7 @@ def main():
                 str(depth): statistics.mean(values)
                 for depth, values in by_depth.items()}
                 for method, by_depth in accuracies.items()},
-            "bp_deficit": bp_deficits,
-            "kp_deficit": kp_deficits,
-            "d56_dfa_advantage": d56_dfa_advantage,
-            "dynamic_d20_to_d56_gain": dynamic_depth_gain,
-            "bp_d20_to_d56_gain": bp_depth_gain,
+            **derived,
             "dynamic_alignment": alignments,
             "mechanism": mechanism,
             "invariant_failures": failures,