diff options
| -rw-r--r-- | README.md | 2 | ||||
| -rw-r--r-- | STAGE_MAP.md | 2 | ||||
| -rw-r--r-- | src/gap_pipeline/kernel_models.py | 18 | ||||
| -rw-r--r-- | src/gap_pipeline/models.py | 18 | ||||
| -rw-r--r-- | src/gap_pipeline/paper_pipeline.py | 3 | ||||
| -rw-r--r-- | src/gap_pipeline/pipeline.py | 2 | ||||
| -rw-r--r-- | tests/test_kernel_models.py | 13 | ||||
| -rw-r--r-- | tests/test_models.py | 24 |
8 files changed, 27 insertions, 55 deletions
@@ -59,7 +59,7 @@ separately auditable. known dependencies, acyclicity, and terminal connectivity. Replacement plans record the exact old/new value, source node, guard condition, and guard justification. Diffusion must preserve every node ID, dependency, and method -label. Judges must cover every proof-node ID before their verdict counts. +label. The five LLM judges' structured verdicts drive acceptance and repair. ## Install and test diff --git a/STAGE_MAP.md b/STAGE_MAP.md index fe90ebb..4d01981 100644 --- a/STAGE_MAP.md +++ b/STAGE_MAP.md @@ -31,7 +31,7 @@ uses the byte-pinned Appendix F.3 judge prompt. - The diffused proof must preserve node IDs, dependencies, and method labels. - The rendered solution must cite every node in order and retain the diffused terminal answer. -- Every judge must discuss every method-plan node ID. +- The five LLM judges' structured verdicts drive acceptance and repair. ## Per-item artifacts diff --git a/src/gap_pipeline/kernel_models.py b/src/gap_pipeline/kernel_models.py index f835473..0840af5 100644 --- a/src/gap_pipeline/kernel_models.py +++ b/src/gap_pipeline/kernel_models.py @@ -309,24 +309,6 @@ class JudgeVerdict(StrictModel): raise ValueError("reject verdict must identify a blocking issue") return self - def validate_coverage( - self, - dag: ProofDAG, - ) -> "JudgeVerdict": - missing_nodes = [ - node_id - for node_id in dag.node_ids() - if re.search( - rf"(?<![A-Za-z0-9_]){re.escape(node_id)}(?![A-Za-z0-9_])", - self.step_by_step_check, - ) - is None - ] - if missing_nodes: - raise ValueError(f"judge coverage incomplete: nodes={missing_nodes}") - return self - - class VerificationIteration(StrictModel): iteration: int bundle_sha256: str diff --git a/src/gap_pipeline/models.py b/src/gap_pipeline/models.py index 9b0f214..51bc49e 100644 --- a/src/gap_pipeline/models.py +++ b/src/gap_pipeline/models.py @@ -3,7 +3,6 @@ from __future__ import annotations import json -import re from datetime import datetime, timezone from typing import Any, Literal @@ -103,7 +102,7 @@ class ProofPlanDAG(StrictModel): return cls(nodes=nodes, terminal_node_id=nodes[-1].node_id) def method_labels_payload(self) -> list[str]: - """Keep the judge input a sequence while making node coverage auditable.""" + """Keep the judge input as an ordered, labeled sequence.""" return [f"{node.node_id}: {node.method_label}" for node in self.nodes] @@ -217,21 +216,6 @@ class JudgeVerdict(StrictModel): raise ValueError("reject verdict must identify a blocking issue") return self - def validate_coverage(self, dag: ProofPlanDAG) -> "JudgeVerdict": - missing = [ - node.node_id - for node in dag.nodes - if re.search( - rf"(?<![A-Za-z0-9_]){re.escape(node.node_id)}(?![A-Za-z0-9_])", - self.step_by_step_check, - ) - is None - ] - if missing: - raise ValueError(f"judge check does not cover DAG nodes: {missing}") - return self - - class IterationRecord(StrictModel): iteration: int candidate_sha256: str diff --git a/src/gap_pipeline/paper_pipeline.py b/src/gap_pipeline/paper_pipeline.py index ffe9f15..6998a17 100644 --- a/src/gap_pipeline/paper_pipeline.py +++ b/src/gap_pipeline/paper_pipeline.py @@ -343,7 +343,6 @@ class PaperKernelPipeline: judge: JsonLLM, *, item: CanonicalItem, - dag: ProofDAG, methods: MethodPlan, bundle: CandidateBundle, iteration: int, @@ -361,7 +360,6 @@ class PaperKernelPipeline: bundle.variant, ), response_model=JudgeVerdict, - validate=lambda value: value.validate_coverage(dag), ) return verdict @@ -398,7 +396,6 @@ class PaperKernelPipeline: self._judge_once( judge, item=item, - dag=dag, methods=methods, bundle=bundle, iteration=iteration, diff --git a/src/gap_pipeline/pipeline.py b/src/gap_pipeline/pipeline.py index 9d03525..7876151 100644 --- a/src/gap_pipeline/pipeline.py +++ b/src/gap_pipeline/pipeline.py @@ -171,7 +171,7 @@ class KernelPipeline: system_prompt=JUDGE_SYSTEM_PROMPT, user_prompt=judge_user(item, plan, dag, candidate), ) - return JudgeVerdict.model_validate(payload).validate_coverage(dag) + return JudgeVerdict.model_validate(payload) async def _repair( self, diff --git a/tests/test_kernel_models.py b/tests/test_kernel_models.py index 0dc653d..8ccfd16 100644 --- a/tests/test_kernel_models.py +++ b/tests/test_kernel_models.py @@ -199,3 +199,16 @@ def test_accept_verdict_normalizes_explicit_none_sentinel() -> None: ) assert verdict.blocking_issues == "" assert verdict.patch_suggestion == "" + + +def test_reject_verdict_accepts_llm_summary_without_keyword_expansion() -> None: + verdict = JudgeVerdict( + verdict="reject", + step_by_step_check=( + "n1–n8 are correct, but the announced slot replacement never " + "appears in the candidate problem." + ), + blocking_issues="the candidate problem is identical to the original", + patch_suggestion="apply the declared replacement to the problem", + ) + assert verdict.verdict == "reject" diff --git a/tests/test_models.py b/tests/test_models.py index b329ecd..2c08bce 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -96,23 +96,20 @@ def test_path_dag_rejects_skipped_dependency() -> None: ) -def test_judge_verdict_requires_every_dag_node(plan_dict: dict) -> None: - dag = ProofPlanDAG.from_plan(KernelPlan.model_validate(plan_dict)) +def test_judge_verdict_does_not_require_keyword_coverage() -> None: verdict = JudgeVerdict( - verdict="accept", - step_by_step_check="n1 is valid; n2 is valid", + verdict="reject", + step_by_step_check=( + "The early derivation is correct, but the announced replacement " + "never appears in the candidate problem." + ), + blocking_issues="the candidate is unchanged from the original", + patch_suggestion="apply the declared replacement", ) - assert verdict.validate_coverage(dag) is verdict + assert verdict.verdict == "reject" - with pytest.raises(ValueError, match="does not cover DAG nodes.*n2"): - JudgeVerdict( - verdict="accept", - step_by_step_check="n1 is valid", - ).validate_coverage(dag) - -def test_judge_verdict_normalizes_structured_text_fields(plan_dict: dict) -> None: - dag = ProofPlanDAG.from_plan(KernelPlan.model_validate(plan_dict)) +def test_judge_verdict_normalizes_structured_text_fields() -> None: verdict = JudgeVerdict.model_validate( { "verdict": "accept", @@ -128,7 +125,6 @@ def test_judge_verdict_normalizes_structured_text_fields(plan_dict: dict) -> Non assert '"n1"' in verdict.step_by_step_check assert verdict.blocking_issues == "" assert verdict.patch_suggestion == "" - assert verdict.validate_coverage(dag) is verdict def test_dag_labels_must_match_prompt_a_plan(plan_dict: dict) -> None: |