test: falsify oral A recovery boundaries

1 files changed, 86 insertions, 0 deletions

diff --git a/experiments/oral_a_dynamic_scaling_smoke.py b/experiments/oral_a_dynamic_scaling_smoke.py
new file mode 100755
index 0000000..a194557
--- /dev/null
+++ b/experiments/oral_a_dynamic_scaling_smoke.py
@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+"""Endpoint-free contract and boundary checks for the oral-A recovery."""
+import copy
+import os
+import sys
+from collections import Counter
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from experiments.analyze_oral_a_dynamic_scaling import (
+    DEPTHS, expected_args, oral_a_checks)
+from experiments.conv_run import parse_args
+from experiments.oral_a_dynamic_scaling import jobs
+
+
+def check_job_contract():
+    cells = jobs("cuda", "results/oral_a_dynamic_scaling")
+    assert len(cells) == 50
+    assert len({(method, depth, seed)
+                for method, depth, seed, _, _ in cells}) == 50
+    assert Counter(method for method, _, _, _, _ in cells) == {
+        "bp": 15, "dfa": 15, "clean_kp": 10, "dynamic": 10}
+    assert Counter(depth for _, depth, _, _, _ in cells) == {
+        20: 10, 32: 20, 56: 20}
+    for method, depth, seed, path, command in cells:
+        saved = sys.argv
+        sys.argv = command[1:]
+        args = parse_args()
+        sys.argv = saved
+        actual = vars(args)
+        for key, value in expected_args(method, depth, seed).items():
+            assert actual[key] == value, (
+                method, depth, seed, key, actual[key], value)
+        assert command[-2:] == ["--out", path]
+    print("oral-A recovery runner/analyzer contract: exact 50 new cells")
+
+
+def passing_arrays():
+    offsets = [-0.002, -0.001, 0.0, 0.001, 0.002]
+    means = {
+        "bp": {20: 0.915, 32: 0.925, 56: 0.935},
+        "dfa": {20: 0.40, 32: 0.40, 56: 0.40},
+        "clean_kp": {20: 0.912, 32: 0.922, 56: 0.932},
+        "dynamic": {20: 0.908, 32: 0.918, 56: 0.928},
+    }
+    accuracies = {method: {
+        depth: [mean + offset for offset in offsets]
+        for depth, mean in by_depth.items()}
+        for method, by_depth in means.items()}
+    alignments = {depth: [0.95 + offset for offset in offsets]
+                  for depth in DEPTHS}
+    return accuracies, alignments
+
+
+def check_gate_boundaries():
+    accuracies, alignments = passing_arrays()
+    checks, _ = oral_a_checks(accuracies, alignments, [])
+    assert all(checks.values())
+
+    collapsed = copy.deepcopy(accuracies)
+    for method in collapsed:
+        for depth in DEPTHS:
+            collapsed[method][depth] = [0.10] * 5
+    collapsed_checks, _ = oral_a_checks(collapsed, alignments, [])
+    assert not collapsed_checks["bp_mean_accuracy_at_least_0p90_each_depth"]
+    assert not collapsed_checks["every_dynamic_accuracy_at_least_0p87"]
+    assert collapsed_checks["dynamic_mean_within_2pt_bp_each_depth"]
+
+    flat = copy.deepcopy(accuracies)
+    flat["dynamic"][56] = list(flat["dynamic"][20])
+    flat_checks, _ = oral_a_checks(flat, alignments, [])
+    assert not flat_checks["dynamic_mean_d20_to_d56_gain_at_least_0p5pt"]
+    assert not flat_checks["at_least_four_dynamic_seeds_improve_with_depth"]
+
+    invariant_checks, _ = oral_a_checks(
+        accuracies, alignments, ["synthetic:cost_failure"])
+    assert not invariant_checks[
+        "all_mechanism_query_cost_memory_invariants"]
+    assert not invariant_checks["all_60_records_and_audited_values_finite"]
+    print("oral-A recovery absolute, depth-benefit, and invariant gates: falsifiable")
+
+
+if __name__ == "__main__":
+    check_job_contract()
+    check_gate_boundaries()
+    print("ALL ORAL-A RECOVERY MECHANICS CHECKS PASSED")