test: falsify oral B protocol boundaries

3 files changed, 200 insertions, 71 deletions

diff --git a/experiments/analyze_bci_td_confirmation.py b/experiments/analyze_bci_td_confirmation.py
index 8697e38..689f3a7 100755
--- a/experiments/analyze_bci_td_confirmation.py
+++ b/experiments/analyze_bci_td_confirmation.py
@@ -70,6 +70,81 @@ def summarize(values):
     }
 
 
+def r2_checks(metrics, clustered, all_role_cosines, all_signs):
+    """Apply the frozen R2 gate to task-clustered summary statistics."""
+    learning_checks = {
+        "mean_intact_final_at_least_0p70":
+            metrics["intact_final"]["mean"] >= 0.70,
+        "every_task_mean_final_at_least_0p60":
+            min(clustered["intact_final"]) >= 0.60,
+        "intact_final_lower_bound_at_least_0p60":
+            metrics["intact_final"]["one_sided_95pct_lower"] >= 0.60,
+        "mean_learning_gain_at_least_0p10":
+            metrics["intact_gain"]["mean"] >= 0.10,
+        "learning_gain_lower_bound_at_least_0p05":
+            metrics["intact_gain"]["one_sided_95pct_lower"] >= 0.05,
+        "mean_fixed_gap_at_least_0p20":
+            metrics["fixed_final_gap"]["mean"] >= 0.20,
+        "fixed_gap_lower_bound_at_least_0p10":
+            metrics["fixed_final_gap"]["one_sided_95pct_lower"] >= 0.10,
+        "mean_oracle_deficit_at_most_0p10":
+            metrics["oracle_final_deficit"]["mean"] <= 0.10,
+        "oracle_deficit_upper_bound_at_most_0p20":
+            metrics["oracle_final_deficit"]["one_sided_95pct_upper"] <= 0.20,
+        "plasticity_lesion_half_margin_lower_bound_nonnegative":
+            metrics["plasticity_half_margin"]["one_sided_95pct_lower"] >= 0.0,
+        "mean_role_cosine_at_least_0p80":
+            metrics["role_cosine"]["mean"] >= 0.80,
+        "every_role_cosine_at_least_0p70": min(all_role_cosines) >= 0.70,
+    }
+    innovation_checks = {
+        "mean_residual_soma_corr_at_most_0p10":
+            metrics["residual_soma_corr"]["mean"] <= 0.10,
+        "residual_soma_corr_upper_bound_at_most_0p12":
+            metrics["residual_soma_corr"]["one_sided_95pct_upper"] <= 0.12,
+        "mean_raw_residual_corr_gap_at_least_0p20":
+            metrics["raw_residual_corr_gap"]["mean"] >= 0.20,
+        "raw_residual_corr_gap_lower_bound_at_least_0p15":
+            metrics["raw_residual_corr_gap"]["one_sided_95pct_lower"] >= 0.15,
+        "mean_surrounding_event_accuracy_at_least_0p55":
+            metrics["surrounding_event_accuracy"]["mean"] >= 0.55,
+        "surrounding_event_accuracy_lower_bound_at_least_0p52":
+            metrics["surrounding_event_accuracy"]["one_sided_95pct_lower"] >= 0.52,
+        "mean_decoder_distance_corr_at_least_0p10":
+            metrics["decoder_distance_corr"]["mean"] >= 0.10,
+        "decoder_distance_corr_lower_bound_at_least_0p02":
+            metrics["decoder_distance_corr"]["one_sided_95pct_lower"] >= 0.02,
+    }
+    vectorization_checks = {
+        "mean_residual_outcome_accuracy_at_least_0p57":
+            metrics["residual_outcome_accuracy"]["mean"] >= 0.57,
+        "residual_outcome_accuracy_lower_bound_at_least_0p53":
+            metrics["residual_outcome_accuracy"]["one_sided_95pct_lower"] >= 0.53,
+        "mean_residual_soma_outcome_gap_at_least_0p03":
+            metrics["residual_soma_outcome_gap"]["mean"] >= 0.03,
+        "residual_soma_outcome_gap_lower_bound_nonnegative":
+            metrics["residual_soma_outcome_gap"]["one_sided_95pct_lower"] >= 0.0,
+        "positive_sign_inversion_in_at_least_25_of_30":
+            sum(value > 0 for value in all_signs) >= 25,
+        "positive_sign_inversion_in_every_task_cluster":
+            min(clustered["sign_inversion"]) > 0.0,
+        "mean_velocity_advantage_at_least_0p05":
+            metrics["velocity_advantage"]["mean"] >= 0.05,
+        "velocity_advantage_lower_bound_nonnegative":
+            metrics["velocity_advantage"]["one_sided_95pct_lower"] >= 0.0,
+        "mean_longitudinal_prediction_at_least_0p30":
+            metrics["longitudinal_prediction"]["mean"] >= 0.30,
+        "longitudinal_prediction_lower_bound_at_least_0p10":
+            metrics["longitudinal_prediction"]["one_sided_95pct_lower"] >= 0.10,
+    }
+    return {
+        "all_records_finite_paired_and_cost_audited": True,
+        "learning_and_plasticity": learning_checks,
+        "innovation_identification_and_network_prediction": innovation_checks,
+        "outcome_and_causal_role_vectorization": vectorization_checks,
+    }
+
+
 def validate(row, path, eta, digests):
     require(row.get("schema_version") == 1, f"{path}: schema")
     args = row.get("args", {})
@@ -260,77 +335,7 @@ def main():
                         for row in records.values()]
     all_signs = [getters["sign_inversion"](row)
                  for row in records.values()]
-    learning_checks = {
-        "mean_intact_final_at_least_0p70":
-            metrics["intact_final"]["mean"] >= 0.70,
-        "every_task_mean_final_at_least_0p60":
-            min(clustered["intact_final"]) >= 0.60,
-        "intact_final_lower_bound_at_least_0p60":
-            metrics["intact_final"]["one_sided_95pct_lower"] >= 0.60,
-        "mean_learning_gain_at_least_0p10":
-            metrics["intact_gain"]["mean"] >= 0.10,
-        "learning_gain_lower_bound_at_least_0p05":
-            metrics["intact_gain"]["one_sided_95pct_lower"] >= 0.05,
-        "mean_fixed_gap_at_least_0p20":
-            metrics["fixed_final_gap"]["mean"] >= 0.20,
-        "fixed_gap_lower_bound_at_least_0p10":
-            metrics["fixed_final_gap"]["one_sided_95pct_lower"] >= 0.10,
-        "mean_oracle_deficit_at_most_0p10":
-            metrics["oracle_final_deficit"]["mean"] <= 0.10,
-        "oracle_deficit_upper_bound_at_most_0p20":
-            metrics["oracle_final_deficit"]["one_sided_95pct_upper"] <= 0.20,
-        "plasticity_lesion_half_margin_lower_bound_nonnegative":
-            metrics["plasticity_half_margin"]["one_sided_95pct_lower"] >= 0.0,
-        "mean_role_cosine_at_least_0p80":
-            metrics["role_cosine"]["mean"] >= 0.80,
-        "every_role_cosine_at_least_0p70": min(all_role_cosines) >= 0.70,
-    }
-    innovation_checks = {
-        "mean_residual_soma_corr_at_most_0p10":
-            metrics["residual_soma_corr"]["mean"] <= 0.10,
-        "residual_soma_corr_upper_bound_at_most_0p12":
-            metrics["residual_soma_corr"]["one_sided_95pct_upper"] <= 0.12,
-        "mean_raw_residual_corr_gap_at_least_0p20":
-            metrics["raw_residual_corr_gap"]["mean"] >= 0.20,
-        "raw_residual_corr_gap_lower_bound_at_least_0p15":
-            metrics["raw_residual_corr_gap"]["one_sided_95pct_lower"] >= 0.15,
-        "mean_surrounding_event_accuracy_at_least_0p55":
-            metrics["surrounding_event_accuracy"]["mean"] >= 0.55,
-        "surrounding_event_accuracy_lower_bound_at_least_0p52":
-            metrics["surrounding_event_accuracy"]["one_sided_95pct_lower"] >= 0.52,
-        "mean_decoder_distance_corr_at_least_0p10":
-            metrics["decoder_distance_corr"]["mean"] >= 0.10,
-        "decoder_distance_corr_lower_bound_at_least_0p02":
-            metrics["decoder_distance_corr"]["one_sided_95pct_lower"] >= 0.02,
-    }
-    vectorization_checks = {
-        "mean_residual_outcome_accuracy_at_least_0p57":
-            metrics["residual_outcome_accuracy"]["mean"] >= 0.57,
-        "residual_outcome_accuracy_lower_bound_at_least_0p53":
-            metrics["residual_outcome_accuracy"]["one_sided_95pct_lower"] >= 0.53,
-        "mean_residual_soma_outcome_gap_at_least_0p03":
-            metrics["residual_soma_outcome_gap"]["mean"] >= 0.03,
-        "residual_soma_outcome_gap_lower_bound_nonnegative":
-            metrics["residual_soma_outcome_gap"]["one_sided_95pct_lower"] >= 0.0,
-        "positive_sign_inversion_in_at_least_25_of_30":
-            sum(value > 0 for value in all_signs) >= 25,
-        "positive_sign_inversion_in_every_task_cluster":
-            min(clustered["sign_inversion"]) > 0.0,
-        "mean_velocity_advantage_at_least_0p05":
-            metrics["velocity_advantage"]["mean"] >= 0.05,
-        "velocity_advantage_lower_bound_nonnegative":
-            metrics["velocity_advantage"]["one_sided_95pct_lower"] >= 0.0,
-        "mean_longitudinal_prediction_at_least_0p30":
-            metrics["longitudinal_prediction"]["mean"] >= 0.30,
-        "longitudinal_prediction_lower_bound_at_least_0p10":
-            metrics["longitudinal_prediction"]["one_sided_95pct_lower"] >= 0.10,
-    }
-    checks = {
-        "all_records_finite_paired_and_cost_audited": True,
-        "learning_and_plasticity": learning_checks,
-        "innovation_identification_and_network_prediction": innovation_checks,
-        "outcome_and_causal_role_vectorization": vectorization_checks,
-    }
+    checks = r2_checks(metrics, clustered, all_role_cosines, all_signs)
     passed = all(
         value
         for category, values in checks.items()
diff --git a/experiments/bci_td_protocol_smoke.py b/experiments/bci_td_protocol_smoke.py
new file mode 100755
index 0000000..ed42db5
--- /dev/null
+++ b/experiments/bci_td_protocol_smoke.py
@@ -0,0 +1,122 @@
+#!/usr/bin/env python3
+"""Endpoint-free falsification checks for the frozen oral-B R1/R2 gates."""
+import copy
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from experiments.analyze_bci_td_confirmation import r2_checks, summarize
+from experiments.analyze_bci_td_development import seed_checks
+
+
+def check_r1_boundaries():
+    row = {
+        "conditions": {
+            "intact": {
+                "learning_gain": 0.20, "final_success": 0.80,
+                "role_cosine_after_training": 0.90,
+            },
+            "fixed_vectorizer": {"final_success": 0.50},
+            "plasticity_lesion": {"learning_gain": 0.05},
+            "oracle_role": {"final_success": 0.85},
+        },
+        "signatures": {
+            "causal_role_sign_inversion_index": 0.10,
+            "velocity_minus_error_abs_cv_corr": 0.10,
+            "mean_abs_residual_soma_corr": 0.05,
+            "raw_minus_residual_abs_soma_corr": 0.30,
+        },
+    }
+    assert all(seed_checks(row).values())
+    no_learning = copy.deepcopy(row)
+    no_learning["conditions"]["intact"]["learning_gain"] = 0.0
+    no_learning["conditions"]["plasticity_lesion"]["learning_gain"] = 0.0
+    assert not seed_checks(no_learning)["learning_gain_at_least_0p10"]
+    wrong_sign = copy.deepcopy(row)
+    wrong_sign["signatures"]["causal_role_sign_inversion_index"] = -0.01
+    assert not seed_checks(wrong_sign)["sign_inversion_at_least_0p01"]
+    no_vectorizer_gap = copy.deepcopy(row)
+    no_vectorizer_gap["conditions"]["fixed_vectorizer"]["final_success"] = 0.75
+    assert not seed_checks(no_vectorizer_gap)[
+        "fixed_vectorizer_gap_at_least_0p20"]
+    print("oral-B R1 learning, sign, and vectorizer gates: falsifiable")
+
+
+def passing_r2_inputs():
+    values = {
+        "intact_final": 0.80,
+        "intact_gain": 0.20,
+        "fixed_final_gap": 0.30,
+        "oracle_final_deficit": 0.05,
+        "plasticity_half_margin": 0.05,
+        "role_cosine": 0.90,
+        "residual_soma_corr": 0.05,
+        "raw_residual_corr_gap": 0.30,
+        "surrounding_event_accuracy": 0.60,
+        "decoder_distance_corr": 0.20,
+        "residual_outcome_accuracy": 0.65,
+        "residual_soma_outcome_gap": 0.10,
+        "sign_inversion": 0.10,
+        "velocity_advantage": 0.10,
+        "longitudinal_prediction": 0.40,
+    }
+    clustered = {name: [value] * 6 for name, value in values.items()}
+    metrics = {name: summarize(rows) for name, rows in clustered.items()}
+    return metrics, clustered, [0.90] * 30, [0.10] * 30
+
+
+def flatten(checks):
+    return [value for values in checks.values()
+            for value in ([values] if isinstance(values, bool)
+                          else values.values())]
+
+
+def check_r2_boundaries():
+    metrics, clustered, roles, signs = passing_r2_inputs()
+    checks = r2_checks(metrics, clustered, roles, signs)
+    assert all(flatten(checks))
+
+    no_learning_metrics = copy.deepcopy(metrics)
+    no_learning_clusters = copy.deepcopy(clustered)
+    no_learning_clusters["intact_gain"] = [0.0] * 6
+    no_learning_metrics["intact_gain"] = summarize(
+        no_learning_clusters["intact_gain"])
+    no_learning = r2_checks(
+        no_learning_metrics, no_learning_clusters, roles, signs)
+    assert not no_learning["learning_and_plasticity"][
+        "mean_learning_gain_at_least_0p10"]
+
+    only_24_positive = [0.10] * 24 + [-0.10] * 6
+    sign_failure = r2_checks(metrics, clustered, roles, only_24_positive)
+    assert not sign_failure["outcome_and_causal_role_vectorization"][
+        "positive_sign_inversion_in_at_least_25_of_30"]
+
+    heterogeneous_metrics = copy.deepcopy(metrics)
+    heterogeneous_clusters = copy.deepcopy(clustered)
+    heterogeneous_clusters["fixed_final_gap"] = [0.30] * 5 + [-0.20]
+    heterogeneous_metrics["fixed_final_gap"] = summarize(
+        heterogeneous_clusters["fixed_final_gap"])
+    heterogeneous = r2_checks(
+        heterogeneous_metrics, heterogeneous_clusters, roles, signs)
+    assert heterogeneous["learning_and_plasticity"][
+        "mean_fixed_gap_at_least_0p20"]
+    assert not heterogeneous["learning_and_plasticity"][
+        "fixed_gap_lower_bound_at_least_0p10"]
+
+    no_longitudinal_metrics = copy.deepcopy(metrics)
+    no_longitudinal_clusters = copy.deepcopy(clustered)
+    no_longitudinal_clusters["longitudinal_prediction"] = [0.0] * 6
+    no_longitudinal_metrics["longitudinal_prediction"] = summarize(
+        no_longitudinal_clusters["longitudinal_prediction"])
+    no_longitudinal = r2_checks(
+        no_longitudinal_metrics, no_longitudinal_clusters, roles, signs)
+    assert not no_longitudinal["outcome_and_causal_role_vectorization"][
+        "mean_longitudinal_prediction_at_least_0p30"]
+    print("oral-B R2 learning, clustered robustness, sign, and prediction gates: falsifiable")
+
+
+if __name__ == "__main__":
+    check_r1_boundaries()
+    check_r2_boundaries()
+    print("ALL ORAL-B PROTOCOL BOUNDARY CHECKS PASSED")
diff --git a/experiments/finalize_accept.sh b/experiments/finalize_accept.sh
index 4fd8d6b..692a6ae 100755
--- a/experiments/finalize_accept.sh
+++ b/experiments/finalize_accept.sh
@@ -14,6 +14,8 @@ experiments/finalize_claims.sh
 /home/yurenh2/miniconda3/envs/ep_pascal/bin/python3 \
   experiments/bci_smoke.py
 /home/yurenh2/miniconda3/envs/ep_pascal/bin/python3 \
+  experiments/bci_td_protocol_smoke.py
+/home/yurenh2/miniconda3/envs/ep_pascal/bin/python3 \
   experiments/oral_a_dynamic_scaling_smoke.py
 /home/yurenh2/miniconda3/envs/ep_pascal/bin/python3 -m py_compile \
   experiments/bci_td_run.py experiments/analyze_bci_td_development.py \