From 60e5c8d2ca992e197b640c58a5ba276bb6299bf4 Mon Sep 17 00:00:00 2001 From: Anonymous Authors Date: Sat, 25 Jul 2026 13:42:41 -0500 Subject: Remove non-paper judge keyword checks --- README.md | 2 +- STAGE_MAP.md | 2 +- src/gap_pipeline/kernel_models.py | 18 ------------------ src/gap_pipeline/models.py | 18 +----------------- src/gap_pipeline/paper_pipeline.py | 3 --- src/gap_pipeline/pipeline.py | 2 +- tests/test_kernel_models.py | 13 +++++++++++++ tests/test_models.py | 24 ++++++++++-------------- 8 files changed, 27 insertions(+), 55 deletions(-) diff --git a/README.md b/README.md index d27b600..3e57d5a 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ separately auditable. known dependencies, acyclicity, and terminal connectivity. Replacement plans record the exact old/new value, source node, guard condition, and guard justification. Diffusion must preserve every node ID, dependency, and method -label. Judges must cover every proof-node ID before their verdict counts. +label. The five LLM judges' structured verdicts drive acceptance and repair. ## Install and test diff --git a/STAGE_MAP.md b/STAGE_MAP.md index fe90ebb..4d01981 100644 --- a/STAGE_MAP.md +++ b/STAGE_MAP.md @@ -31,7 +31,7 @@ uses the byte-pinned Appendix F.3 judge prompt. - The diffused proof must preserve node IDs, dependencies, and method labels. - The rendered solution must cite every node in order and retain the diffused terminal answer. -- Every judge must discuss every method-plan node ID. +- The five LLM judges' structured verdicts drive acceptance and repair. ## Per-item artifacts diff --git a/src/gap_pipeline/kernel_models.py b/src/gap_pipeline/kernel_models.py index f835473..0840af5 100644 --- a/src/gap_pipeline/kernel_models.py +++ b/src/gap_pipeline/kernel_models.py @@ -309,24 +309,6 @@ class JudgeVerdict(StrictModel): raise ValueError("reject verdict must identify a blocking issue") return self - def validate_coverage( - self, - dag: ProofDAG, - ) -> "JudgeVerdict": - missing_nodes = [ - node_id - for node_id in dag.node_ids() - if re.search( - rf"(? list[str]: - """Keep the judge input a sequence while making node coverage auditable.""" + """Keep the judge input as an ordered, labeled sequence.""" return [f"{node.node_id}: {node.method_label}" for node in self.nodes] @@ -217,21 +216,6 @@ class JudgeVerdict(StrictModel): raise ValueError("reject verdict must identify a blocking issue") return self - def validate_coverage(self, dag: ProofPlanDAG) -> "JudgeVerdict": - missing = [ - node.node_id - for node in dag.nodes - if re.search( - rf"(? None: ) assert verdict.blocking_issues == "" assert verdict.patch_suggestion == "" + + +def test_reject_verdict_accepts_llm_summary_without_keyword_expansion() -> None: + verdict = JudgeVerdict( + verdict="reject", + step_by_step_check=( + "n1–n8 are correct, but the announced slot replacement never " + "appears in the candidate problem." + ), + blocking_issues="the candidate problem is identical to the original", + patch_suggestion="apply the declared replacement to the problem", + ) + assert verdict.verdict == "reject" diff --git a/tests/test_models.py b/tests/test_models.py index b329ecd..2c08bce 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -96,23 +96,20 @@ def test_path_dag_rejects_skipped_dependency() -> None: ) -def test_judge_verdict_requires_every_dag_node(plan_dict: dict) -> None: - dag = ProofPlanDAG.from_plan(KernelPlan.model_validate(plan_dict)) +def test_judge_verdict_does_not_require_keyword_coverage() -> None: verdict = JudgeVerdict( - verdict="accept", - step_by_step_check="n1 is valid; n2 is valid", + verdict="reject", + step_by_step_check=( + "The early derivation is correct, but the announced replacement " + "never appears in the candidate problem." + ), + blocking_issues="the candidate is unchanged from the original", + patch_suggestion="apply the declared replacement", ) - assert verdict.validate_coverage(dag) is verdict + assert verdict.verdict == "reject" - with pytest.raises(ValueError, match="does not cover DAG nodes.*n2"): - JudgeVerdict( - verdict="accept", - step_by_step_check="n1 is valid", - ).validate_coverage(dag) - -def test_judge_verdict_normalizes_structured_text_fields(plan_dict: dict) -> None: - dag = ProofPlanDAG.from_plan(KernelPlan.model_validate(plan_dict)) +def test_judge_verdict_normalizes_structured_text_fields() -> None: verdict = JudgeVerdict.model_validate( { "verdict": "accept", @@ -128,7 +125,6 @@ def test_judge_verdict_normalizes_structured_text_fields(plan_dict: dict) -> Non assert '"n1"' in verdict.step_by_step_check assert verdict.blocking_issues == "" assert verdict.patch_suggestion == "" - assert verdict.validate_coverage(dag) is verdict def test_dag_labels_must_match_prompt_a_plan(plan_dict: dict) -> None: -- cgit v1.2.3