diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/gap_pipeline/e2e.py | 3 | ||||
| -rw-r--r-- | src/gap_pipeline/kernel_models.py | 54 | ||||
| -rw-r--r-- | src/gap_pipeline/kernel_prompts.py | 153 | ||||
| -rw-r--r-- | src/gap_pipeline/paper_pipeline.py | 90 | ||||
| -rw-r--r-- | src/gap_pipeline/pipeline.py | 2 | ||||
| -rw-r--r-- | src/gap_pipeline/release.py | 21 |
6 files changed, 185 insertions, 138 deletions
diff --git a/src/gap_pipeline/e2e.py b/src/gap_pipeline/e2e.py index 66101e7..e0859dc 100644 --- a/src/gap_pipeline/e2e.py +++ b/src/gap_pipeline/e2e.py @@ -26,7 +26,6 @@ def _review_accept() -> dict[str, str]: return { "verdict": "accept", "step_by_step_check": "n1 passes; n2 passes; n3 passes", - "replacement_check": "s1 satisfies its positivity guard", "blocking_issues": "", "patch_suggestion": "", } @@ -204,7 +203,7 @@ async def run_offline_smoke(work_root: Path) -> dict[str, Any]: f"{item_id}.stage3.replacement": { "changes": [ { - "slot_id": "s1", + "slot_id": "slot1", "source_node_id": "n1", "description": "positive square root at equality", "original_value": "1", diff --git a/src/gap_pipeline/kernel_models.py b/src/gap_pipeline/kernel_models.py index 32abf39..f835473 100644 --- a/src/gap_pipeline/kernel_models.py +++ b/src/gap_pipeline/kernel_models.py @@ -38,6 +38,11 @@ class ProofDAG(StrictModel): def node_ids(self) -> list[str]: return [node.node_id for node in self.nodes] + def leaf_node_ids(self) -> list[str]: + """Return source leaves: nodes with no prerequisite dependencies.""" + + return [node.node_id for node in self.nodes if not node.dependencies] + @model_validator(mode="after") def validate_graph(self) -> "ProofDAG": node_ids = self.node_ids() @@ -113,6 +118,8 @@ class ReplacementChange(StrictModel): ] if any(not value.strip() for value in text_fields): raise ValueError("replacement fields must be non-empty") + if not re.fullmatch(r"slot[1-9][0-9]*", self.slot_id): + raise ValueError("replacement slot IDs must be slot1, slot2, ...") if self.original_value.strip() == self.replacement_value.strip(): raise ValueError("replacement must differ from the original value") return self @@ -140,6 +147,34 @@ class ReplacementPlan(StrictModel): } if unknown: raise ValueError(f"replacement plan references unknown nodes {sorted(unknown)}") + leaves = set(dag.leaf_node_ids()) + non_leaf = { + change.source_node_id + for change in self.changes + if change.source_node_id not in leaves + } + if non_leaf: + raise ValueError( + "replacement plan must target source leaf nodes; " + f"received {sorted(non_leaf)}" + ) + return self + + def validate_repair_of( + self, + previous: "ReplacementPlan", + ) -> "ReplacementPlan": + current_targets = [ + (change.slot_id, change.source_node_id) for change in self.changes + ] + previous_targets = [ + (change.slot_id, change.source_node_id) + for change in previous.changes + ] + if current_targets != previous_targets: + raise ValueError( + "repair must preserve replacement slot IDs and source leaf nodes" + ) return self @@ -234,13 +269,11 @@ class CandidateBundle(StrictModel): class JudgeVerdict(StrictModel): verdict: Literal["accept", "reject"] step_by_step_check: str - replacement_check: str blocking_issues: str = "" patch_suggestion: str = "" @field_validator( "step_by_step_check", - "replacement_check", "blocking_issues", "patch_suggestion", mode="before", @@ -279,7 +312,6 @@ class JudgeVerdict(StrictModel): def validate_coverage( self, dag: ProofDAG, - replacement_plan: ReplacementPlan, ) -> "JudgeVerdict": missing_nodes = [ node_id @@ -290,20 +322,8 @@ class JudgeVerdict(StrictModel): ) is None ] - missing_slots = [ - change.slot_id - for change in replacement_plan.changes - if re.search( - rf"(?<![A-Za-z0-9_]){re.escape(change.slot_id)}(?![A-Za-z0-9_])", - self.replacement_check, - ) - is None - ] - if missing_nodes or missing_slots: - raise ValueError( - "judge coverage incomplete: " - f"nodes={missing_nodes}, slots={missing_slots}" - ) + if missing_nodes: + raise ValueError(f"judge coverage incomplete: nodes={missing_nodes}") return self diff --git a/src/gap_pipeline/kernel_prompts.py b/src/gap_pipeline/kernel_prompts.py index c4a6128..f932d0a 100644 --- a/src/gap_pipeline/kernel_prompts.py +++ b/src/gap_pipeline/kernel_prompts.py @@ -1,8 +1,9 @@ -"""Prompts for an explicit implementation of the paper's five kernel stages. +"""Prompts for the paper's explicit five-stage kernel implementation. -The byte-pinned historical Prompt-A/Prompt-B remain in ``prompts.py``. These -prompts make the richer five-stage manuscript description executable and keep -every transformation decision in a typed artifact. +The consolidated Prompt-A/Prompt-B interface remains byte-pinned in +``prompts.py``. The default pipeline exposes the manuscript's five operations +as separately validated artifacts and uses its Appendix F.3 judge prompt +verbatim. """ from __future__ import annotations @@ -16,6 +17,7 @@ from .kernel_models import ( ReplacementPlan, ) from .models import CanonicalItem +from .prompts import JUDGE_SYSTEM_PROMPT, JUDGE_USER_TEMPLATE DAG_SYSTEM = "You are a rigorous competition-math proof analyst." @@ -55,18 +57,19 @@ replacements that create a genuinely new problem while preserving the supplied method plan. For every change: -- state the exact source DAG node; +- target one of the disclosed source leaf DAG nodes; +- state that exact source DAG node; - record the original and replacement values explicitly; - state a mathematical guard condition derived from the original problem; - explain why the replacement satisfies the guard. The list must be closed: stages 4 and 5 may introduce no mathematical change that is not declared here. Variable renaming alone is not a kernel change. -{feedback} +{repair_context} Return JSON only: {{"changes":[ - {{"slot_id":"s1","source_node_id":"n1","description":"...", + {{"slot_id":"slot1","source_node_id":"n1","description":"...", "original_value":"...","replacement_value":"...", "guard_condition":"...","guard_justification":"..."}} ], @@ -81,6 +84,9 @@ OFFICIAL SOLUTION: CONCRETE PROOF DAG: {dag} +SOURCE LEAF NODE IDS: +{leaf_node_ids} + CONTENT-FREE METHOD PLAN: {methods}""" @@ -90,6 +96,7 @@ DIFFUSION_USER = """Propagate only the declared replacements through the proof DAG, node by node. Return exactly one row for every source node, preserving its ID, dependencies, and method label. Do not introduce undeclared constants, objects, assumptions, reductions, or proof methods. +{repair_context} Return JSON only: {{"nodes":[ @@ -115,6 +122,7 @@ DECLARED REPLACEMENTS: RENDER_SYSTEM = "You render a verified proof DAG into a self-contained Putnam problem." RENDER_USER = """Render the diffused proof into one complete problem statement and one complete solution. +{repair_context} Requirements: - the question must be fully determined by the diffused terminal claim; @@ -135,51 +143,7 @@ DIFFUSED PROOF: {diffused}""" -JUDGE_SYSTEM = """You are a verification judge for a literal five-stage GAP -kernel transformation. Verification, not de novo solving, is your task.""" -JUDGE_USER = """Check the candidate against every disclosed artifact. - -You must verify: -1. every replacement satisfies its guard and all mathematical changes are - declared in the replacement plan; -2. each diffused node instantiates the matching source node and method label; -3. dependencies and proof order are preserved; -4. the rendered problem is well-posed and the rendered solution proves it; -5. the terminal answer agrees across diffusion, rendering, and solution. - -In step_by_step_check, mention each of these node IDs explicitly: -{required_node_ids} -In replacement_check, mention each of these slot IDs explicitly: -{required_slot_ids} -{format_feedback} - -Return JSON only: -{{"verdict":"accept" or "reject", - "step_by_step_check":"...", - "replacement_check":"...", - "blocking_issues":"...", - "patch_suggestion":"..."}} - -ORIGINAL PROBLEM: -<<<{question}>>> - -OFFICIAL SOLUTION: -<<<{solution}>>> - -SOURCE PROOF DAG: -{dag} - -METHOD PLAN: -{methods} - -REPLACEMENT PLAN: -{replacements} - -DIFFUSED PROOF: -{diffused} - -RENDERED VARIANT: -{variant}""" +JUDGE_SYSTEM = JUDGE_SYSTEM_PROMPT def _dump(value: object) -> str: @@ -201,18 +165,27 @@ def replacement_user( dag: ProofDAG, methods: MethodPlan, *, + previous_replacements: ReplacementPlan | None = None, feedback: str = "", ) -> str: - feedback_block = ( - f"Previous verification feedback to address:\n{feedback}" - if feedback - else "This is the initial replacement proposal." + repair_context = ( + "This is the initial replacement proposal." + if previous_replacements is None + else ( + "This is a repair pass. Preserve the previous slot IDs, source leaf " + "nodes, and accepted changes unless the verification feedback " + "identifies them as the blocking issue. Make only the smallest " + "necessary correction.\n\nPREVIOUS REPLACEMENT PLAN:\n" + f"{_dump(previous_replacements)}\n\nVERIFICATION FEEDBACK:\n" + f"{feedback or 'No additional textual feedback was supplied.'}" + ) ) return REPLACEMENT_USER.format( - feedback=feedback_block, + repair_context=repair_context, question=item.problem, solution=item.solution, dag=_dump(dag), + leaf_node_ids=json.dumps(dag.leaf_node_ids(), ensure_ascii=False), methods=_dump(methods), ) @@ -222,8 +195,23 @@ def diffusion_user( dag: ProofDAG, methods: MethodPlan, replacements: ReplacementPlan, + *, + previous_diffused: DiffusedProof | None = None, + feedback: str = "", ) -> str: + repair_context = ( + "This is the initial DAG diffusion." + if previous_diffused is None + else ( + "This is a repair pass. Apply only corrections required by the " + "verification feedback; preserve every unaffected node.\n\n" + f"PREVIOUS DIFFUSED PROOF:\n{_dump(previous_diffused)}\n\n" + f"VERIFICATION FEEDBACK:\n" + f"{feedback or 'No additional textual feedback was supplied.'}" + ) + ) return DIFFUSION_USER.format( + repair_context=repair_context, question=item.problem, dag=_dump(dag), methods=_dump(methods), @@ -234,8 +222,23 @@ def diffusion_user( def render_user( replacements: ReplacementPlan, diffused: DiffusedProof, + *, + previous_variant: object | None = None, + feedback: str = "", ) -> str: + repair_context = ( + "This is the initial rendering." + if previous_variant is None + else ( + "This is a repair pass. Preserve all unaffected wording and apply " + "only corrections required by the verification feedback.\n\n" + f"PREVIOUS RENDERED VARIANT:\n{_dump(previous_variant)}\n\n" + f"VERIFICATION FEEDBACK:\n" + f"{feedback or 'No additional textual feedback was supplied.'}" + ) + ) return RENDER_USER.format( + repair_context=repair_context, node_order=json.dumps( [node.node_id for node in diffused.nodes], ensure_ascii=False, @@ -247,30 +250,26 @@ def render_user( def judge_user( item: CanonicalItem, - dag: ProofDAG, methods: MethodPlan, replacements: ReplacementPlan, - diffused: DiffusedProof, variant: object, - *, - format_feedback: str = "", ) -> str: - return JUDGE_USER.format( - required_node_ids=json.dumps(dag.node_ids(), ensure_ascii=False), - required_slot_ids=json.dumps( - [change.slot_id for change in replacements.changes], + variant_payload = ( + variant.model_dump(mode="json") + if hasattr(variant, "model_dump") + else variant + ) + return JUDGE_USER_TEMPLATE.format( + original_problem=item.problem, + original_solution=item.solution, + method_labels=json.dumps( + [ + f"{node.node_id}: {node.method_label}" + for node in methods.nodes + ], ensure_ascii=False, ), - format_feedback=( - f"Previous report-format error: {format_feedback}" - if format_feedback - else "" - ), - question=item.problem, - solution=item.solution, - dag=_dump(dag), - methods=_dump(methods), - replacements=_dump(replacements), - diffused=_dump(diffused), - variant=_dump(variant), + slot_replacement=_dump(replacements), + candidate_problem=str(variant_payload["question"]), + candidate_proof=str(variant_payload["solution"]), ) diff --git a/src/gap_pipeline/paper_pipeline.py b/src/gap_pipeline/paper_pipeline.py index 84f34bf..c90f3f6 100644 --- a/src/gap_pipeline/paper_pipeline.py +++ b/src/gap_pipeline/paper_pipeline.py @@ -133,6 +133,7 @@ class PaperKernelPipeline: methods: MethodPlan, *, version: int, + previous_replacements: ReplacementPlan | None = None, feedback: str = "", ) -> ReplacementPlan: request_id = f"{item.item_id}.stage3.replacement.v{version:02d}" @@ -145,10 +146,13 @@ class PaperKernelPipeline: item, dag, methods, + previous_replacements=previous_replacements, feedback=feedback, ), ) ).validate_against(dag) + if previous_replacements is not None: + replacements.validate_repair_of(previous_replacements) self.store.write_stage( f"03_replacement_v{version:02d}", replacements, @@ -164,6 +168,8 @@ class PaperKernelPipeline: replacements: ReplacementPlan, *, version: int, + previous_diffused: DiffusedProof | None = None, + feedback: str = "", ) -> DiffusedProof: request_id = f"{item.item_id}.stage4.diffusion.v{version:02d}" diffused = DiffusedProof.model_validate( @@ -171,7 +177,14 @@ class PaperKernelPipeline: self.proposer, request_id=request_id, system_prompt=DIFFUSION_SYSTEM, - user_prompt=diffusion_user(item, dag, methods, replacements), + user_prompt=diffusion_user( + item, + dag, + methods, + replacements, + previous_diffused=previous_diffused, + feedback=feedback, + ), ) ).validate_against(dag, methods) self.store.write_stage( @@ -188,6 +201,8 @@ class PaperKernelPipeline: diffused: DiffusedProof, *, version: int, + previous_variant: RenderedVariant | None = None, + feedback: str = "", ) -> RenderedVariant: request_id = f"{self.store.item_id}.stage5.render.v{version:02d}" variant = RenderedVariant.model_validate( @@ -195,7 +210,12 @@ class PaperKernelPipeline: self.proposer, request_id=request_id, system_prompt=RENDER_SYSTEM, - user_prompt=render_user(replacements, diffused), + user_prompt=render_user( + replacements, + diffused, + previous_variant=previous_variant, + feedback=feedback, + ), ) ).validate_against(dag, diffused) self.store.write_stage( @@ -212,6 +232,7 @@ class PaperKernelPipeline: methods: MethodPlan, *, version: int, + previous_bundle: CandidateBundle | None = None, feedback: str = "", ) -> CandidateBundle: replacements = await self.generate_replacements( @@ -219,6 +240,11 @@ class PaperKernelPipeline: dag, methods, version=version, + previous_replacements=( + previous_bundle.replacement_plan + if previous_bundle is not None + else None + ), feedback=feedback, ) diffused = await self.diffuse_dag( @@ -227,18 +253,34 @@ class PaperKernelPipeline: methods, replacements, version=version, + previous_diffused=( + previous_bundle.diffused_proof + if previous_bundle is not None + else None + ), + feedback=feedback, ) variant = await self.render_variant( dag, replacements, diffused, version=version, + previous_variant=( + previous_bundle.variant if previous_bundle is not None else None + ), + feedback=feedback, ) - return CandidateBundle( + bundle = CandidateBundle( replacement_plan=replacements, diffused_proof=diffused, variant=variant, ) + if ( + previous_bundle is not None + and sha256_payload(bundle) == sha256_payload(previous_bundle) + ): + raise ValueError("repair pass returned an unchanged candidate bundle") + return bundle async def _judge_once( self, @@ -251,36 +293,21 @@ class PaperKernelPipeline: iteration: int, judge_id: int, ) -> JudgeVerdict: - format_feedback = "" - for attempt in range(1, 4): - request_id = ( - f"{item.item_id}.verify.t{iteration:02d}." - f"j{judge_id}.a{attempt}" - ) - verdict = JudgeVerdict.model_validate( - await self._call( - judge, - request_id=request_id, - system_prompt=JUDGE_SYSTEM, - user_prompt=judge_user( - item, - dag, - methods, - bundle.replacement_plan, - bundle.diffused_proof, - bundle.variant, - format_feedback=format_feedback, - ), - ) + request_id = f"{item.item_id}.verify.t{iteration:02d}.j{judge_id}" + verdict = JudgeVerdict.model_validate( + await self._call( + judge, + request_id=request_id, + system_prompt=JUDGE_SYSTEM, + user_prompt=judge_user( + item, + methods, + bundle.replacement_plan, + bundle.variant, + ), ) - try: - return verdict.validate_coverage(dag, bundle.replacement_plan) - except ValueError as exc: - format_feedback = str(exc) - raise ValueError( - f"judge {judge_id} failed coverage after three format attempts: " - f"{format_feedback}" ) + return verdict.validate_coverage(dag) @staticmethod def _feedback(verdicts: list[JudgeVerdict]) -> str: @@ -369,6 +396,7 @@ class PaperKernelPipeline: dag, methods, version=iteration + 1, + previous_bundle=bundle, feedback=self._feedback(verdicts), ) repaired_from_previous = True diff --git a/src/gap_pipeline/pipeline.py b/src/gap_pipeline/pipeline.py index 4a5332a..9d03525 100644 --- a/src/gap_pipeline/pipeline.py +++ b/src/gap_pipeline/pipeline.py @@ -1,4 +1,4 @@ -"""Historical two-call Putnam generator retained for provenance tests. +"""Consolidated two-call Putnam generation interface. The default manuscript-aligned implementation is ``paper_pipeline.py``. """ diff --git a/src/gap_pipeline/release.py b/src/gap_pipeline/release.py index 7df6566..5bf8d95 100644 --- a/src/gap_pipeline/release.py +++ b/src/gap_pipeline/release.py @@ -89,21 +89,22 @@ def export_release( continue candidate = kernel_payload["accepted_candidate"] + method_nodes = kernel_payload["method_plan"]["nodes"] + replacement_changes = kernel_payload["accepted_replacement_plan"]["changes"] variants["kernel_variant"] = { "question": candidate["question"], "solution": candidate["solution"], "_meta": { - "proof_dag": kernel_payload["proof_dag"], - "method_plan": kernel_payload["method_plan"], - "replacement_plan": kernel_payload["accepted_replacement_plan"], - "diffused_proof": kernel_payload["accepted_diffused_proof"], - "terminal_answer": candidate["terminal_answer"], - "accepted_candidate_sha256": kernel_payload[ - "accepted_candidate_sha256" - ], - "accepted_bundle_sha256": kernel_payload[ - "accepted_bundle_sha256" + "core_steps": [ + node["method_label"] for node in method_nodes ], + "mutable_slots": { + change["slot_id"]: { + "description": change["description"], + "original": change["original_value"], + } + for change in replacement_changes + }, }, } output_record = copy.deepcopy(record) |