From 1d0d538144e754cf5d39badb0fe14d2ec3d9e431 Mon Sep 17 00:00:00 2001 From: YurenHao0426 Date: Wed, 22 Jul 2026 20:43:58 -0500 Subject: test: falsify oral A recovery boundaries --- experiments/analyze_oral_a_dynamic_scaling.py | 123 ++++++++++++++------------ 1 file changed, 66 insertions(+), 57 deletions(-) (limited to 'experiments/analyze_oral_a_dynamic_scaling.py') diff --git a/experiments/analyze_oral_a_dynamic_scaling.py b/experiments/analyze_oral_a_dynamic_scaling.py index ccd4fa9..38ce967 100755 --- a/experiments/analyze_oral_a_dynamic_scaling.py +++ b/experiments/analyze_oral_a_dynamic_scaling.py @@ -199,6 +199,70 @@ def dynamic_invariants(record, depth, seed, failures): } +def oral_a_checks(accuracies, alignments, failures): + """Apply the frozen score gate to already validated paired arrays.""" + bp_deficits = {depth: [bp - dynamic for bp, dynamic in zip( + accuracies["bp"][depth], accuracies["dynamic"][depth])] + for depth in DEPTHS} + kp_deficits = {depth: [kp - dynamic for kp, dynamic in zip( + accuracies["clean_kp"][depth], accuracies["dynamic"][depth])] + for depth in DEPTHS} + d56_dfa_advantage = [dynamic - dfa for dynamic, dfa in zip( + accuracies["dynamic"][56], accuracies["dfa"][56])] + dynamic_depth_gain = [deep - shallow for shallow, deep in zip( + accuracies["dynamic"][20], accuracies["dynamic"][56])] + bp_depth_gain = [deep - shallow for shallow, deep in zip( + accuracies["bp"][20], accuracies["bp"][56])] + checks = { + "all_60_records_and_audited_values_finite": not failures, + "bp_mean_accuracy_at_least_0p90_each_depth": all( + statistics.mean(accuracies["bp"][depth]) >= 0.90 + for depth in DEPTHS), + "every_dynamic_accuracy_at_least_0p87": min( + value for depth in DEPTHS + for value in accuracies["dynamic"][depth]) >= 0.87, + "dynamic_mean_within_2pt_bp_each_depth": all( + statistics.mean(bp_deficits[depth]) <= 0.02 for depth in DEPTHS), + "dynamic_bp_deficit_upper_bound_at_most_3pt_each_depth": all( + upper_confidence_bound(bp_deficits[depth]) <= 0.03 + for depth in DEPTHS), + "dynamic_mean_within_1p5pt_kp_each_depth": all( + statistics.mean(kp_deficits[depth]) <= 0.015 for depth in DEPTHS), + "dynamic_kp_deficit_upper_bound_at_most_2p5pt_each_depth": all( + upper_confidence_bound(kp_deficits[depth]) <= 0.025 + for depth in DEPTHS), + "d56_dynamic_mean_advantage_over_dfa_at_least_2pt": + statistics.mean(d56_dfa_advantage) >= 0.02, + "d56_dynamic_dfa_advantage_lower_bound_at_least_1pt": + lower_confidence_bound(d56_dfa_advantage) >= 0.01, + "dynamic_mean_d20_to_d56_gain_at_least_0p5pt": + statistics.mean(dynamic_depth_gain) >= 0.005, + "dynamic_depth_gain_lower_bound_nonnegative": + lower_confidence_bound(dynamic_depth_gain) >= 0.0, + "at_least_four_dynamic_seeds_improve_with_depth": + sum(value > 0 for value in dynamic_depth_gain) >= 4, + "dynamic_depth_gain_within_1pt_of_bp_gain": + statistics.mean(dynamic_depth_gain) + >= statistics.mean(bp_depth_gain) - 0.01, + "dynamic_mean_alignment_at_least_0p85_each_depth": all( + statistics.mean(alignments[depth]) >= 0.85 for depth in DEPTHS), + "every_d56_dynamic_alignment_at_least_0p80": + min(alignments[56]) >= 0.80, + "d56_mean_alignment_retains_90pct_of_d20": + statistics.mean(alignments[56]) + >= 0.90 * statistics.mean(alignments[20]), + "all_mechanism_query_cost_memory_invariants": not failures, + } + derived = { + "bp_deficit": bp_deficits, + "kp_deficit": kp_deficits, + "d56_dfa_advantage": d56_dfa_advantage, + "dynamic_d20_to_d56_gain": dynamic_depth_gain, + "bp_d20_to_d56_gain": bp_depth_gain, + } + return checks, derived + + def main(): parser = argparse.ArgumentParser() parser.add_argument( @@ -312,58 +376,7 @@ def main(): for seed in SEEDS] for depth in DEPTHS } - bp_deficits = {depth: [bp - dynamic for bp, dynamic in zip( - accuracies["bp"][depth], accuracies["dynamic"][depth])] - for depth in DEPTHS} - kp_deficits = {depth: [kp - dynamic for kp, dynamic in zip( - accuracies["clean_kp"][depth], accuracies["dynamic"][depth])] - for depth in DEPTHS} - d56_dfa_advantage = [dynamic - dfa for dynamic, dfa in zip( - accuracies["dynamic"][56], accuracies["dfa"][56])] - dynamic_depth_gain = [deep - shallow for shallow, deep in zip( - accuracies["dynamic"][20], accuracies["dynamic"][56])] - bp_depth_gain = [deep - shallow for shallow, deep in zip( - accuracies["bp"][20], accuracies["bp"][56])] - checks = { - "all_60_records_and_audited_values_finite": not failures, - "bp_mean_accuracy_at_least_0p90_each_depth": all( - statistics.mean(accuracies["bp"][depth]) >= 0.90 - for depth in DEPTHS), - "every_dynamic_accuracy_at_least_0p87": min( - value for depth in DEPTHS - for value in accuracies["dynamic"][depth]) >= 0.87, - "dynamic_mean_within_2pt_bp_each_depth": all( - statistics.mean(bp_deficits[depth]) <= 0.02 for depth in DEPTHS), - "dynamic_bp_deficit_upper_bound_at_most_3pt_each_depth": all( - upper_confidence_bound(bp_deficits[depth]) <= 0.03 - for depth in DEPTHS), - "dynamic_mean_within_1p5pt_kp_each_depth": all( - statistics.mean(kp_deficits[depth]) <= 0.015 for depth in DEPTHS), - "dynamic_kp_deficit_upper_bound_at_most_2p5pt_each_depth": all( - upper_confidence_bound(kp_deficits[depth]) <= 0.025 - for depth in DEPTHS), - "d56_dynamic_mean_advantage_over_dfa_at_least_2pt": - statistics.mean(d56_dfa_advantage) >= 0.02, - "d56_dynamic_dfa_advantage_lower_bound_at_least_1pt": - lower_confidence_bound(d56_dfa_advantage) >= 0.01, - "dynamic_mean_d20_to_d56_gain_at_least_0p5pt": - statistics.mean(dynamic_depth_gain) >= 0.005, - "dynamic_depth_gain_lower_bound_nonnegative": - lower_confidence_bound(dynamic_depth_gain) >= 0.0, - "at_least_four_dynamic_seeds_improve_with_depth": - sum(value > 0 for value in dynamic_depth_gain) >= 4, - "dynamic_depth_gain_within_1pt_of_bp_gain": - statistics.mean(dynamic_depth_gain) - >= statistics.mean(bp_depth_gain) - 0.01, - "dynamic_mean_alignment_at_least_0p85_each_depth": all( - statistics.mean(alignments[depth]) >= 0.85 for depth in DEPTHS), - "every_d56_dynamic_alignment_at_least_0p80": - min(alignments[56]) >= 0.80, - "d56_mean_alignment_retains_90pct_of_d20": - statistics.mean(alignments[56]) - >= 0.90 * statistics.mean(alignments[20]), - "all_mechanism_query_cost_memory_invariants": not failures, - } + checks, derived = oral_a_checks(accuracies, alignments, failures) passed = all(checks.values()) output = { "protocol": "oral_a_dynamic_innovation_scaling_v1", @@ -376,11 +389,7 @@ def main(): str(depth): statistics.mean(values) for depth, values in by_depth.items()} for method, by_depth in accuracies.items()}, - "bp_deficit": bp_deficits, - "kp_deficit": kp_deficits, - "d56_dfa_advantage": d56_dfa_advantage, - "dynamic_d20_to_d56_gain": dynamic_depth_gain, - "bp_d20_to_d56_gain": bp_depth_gain, + **derived, "dynamic_alignment": alignments, "mechanism": mechanism, "invariant_failures": failures, -- cgit v1.2.3