From 6de74d103926d9090f056aeebe7be393ec381ea1 Mon Sep 17 00:00:00 2001 From: Anonymous Authors Date: Sat, 25 Jul 2026 05:59:42 -0500 Subject: Align five-stage pipeline with manuscript --- GAP_End_to_End.ipynb | 11 +-- README.md | 37 ++++--- STAGE_MAP.md | 34 +++---- src/gap_pipeline/e2e.py | 3 +- src/gap_pipeline/kernel_models.py | 54 +++++++---- src/gap_pipeline/kernel_prompts.py | 153 +++++++++++++++-------------- src/gap_pipeline/paper_pipeline.py | 90 +++++++++++------ src/gap_pipeline/pipeline.py | 2 +- src/gap_pipeline/release.py | 21 ++-- tests/test_e2e.py | 14 ++- tests/test_kernel_models.py | 67 ++++++++++++- tests/test_paper_pipeline.py | 192 +++++++++++++++++++++++++++++++++++++ tests/test_prompts.py | 39 +++++++- tests/test_release.py | 13 ++- 14 files changed, 539 insertions(+), 191 deletions(-) create mode 100644 tests/test_paper_pipeline.py diff --git a/GAP_End_to_End.ipynb b/GAP_End_to_End.ipynb index 2b8c10e..7555e81 100644 --- a/GAP_End_to_End.ipynb +++ b/GAP_End_to_End.ipynb @@ -149,18 +149,15 @@ "\n", "kernel = record[\"variants\"][\"kernel_variant\"]\n", "meta = kernel[\"_meta\"]\n", - "assert meta[\"proof_dag\"][\"nodes\"]\n", - "assert meta[\"method_plan\"][\"nodes\"]\n", - "assert meta[\"replacement_plan\"][\"changes\"]\n", - "assert meta[\"diffused_proof\"][\"nodes\"]\n", - "assert meta[\"accepted_bundle_sha256\"]\n", + "assert meta[\"core_steps\"]\n", + "assert meta[\"mutable_slots\"]\n", "\n", "print(\"END-TO-END PASS\")\n", "print(f\"Item: {record['index']}\")\n", "print(f\"Variant families: {sorted(record['variants'])}\")\n", "print(f\"Verification rounds: {result['verification_rounds']}\")\n", - "print(f\"Declared replacements: {len(meta['replacement_plan']['changes'])}\")\n", - "print(f\"Proof-DAG nodes: {len(meta['proof_dag']['nodes'])}\")\n", + "print(f\"Declared mutable slots: {len(meta['mutable_slots'])}\")\n", + "print(f\"Core proof steps: {len(meta['core_steps'])}\")\n", "print(f\"Kernel question preview: {kernel['question'][:500]}\")\n", "print(f\"Exported record: {result['release_record']}\")\n", "print(f\"Manifest: {result['manifest']}\")" diff --git a/README.md b/README.md index 2a34006..bcc7ff4 100644 --- a/README.md +++ b/README.md @@ -36,28 +36,27 @@ mathematics. ## Prompt fidelity -The historical two-call Prompt-A/Prompt-B, surface-renaming prompts, and -review/repair prompts recovered from `PutnamVariants@c3bed737` remain byte +The consolidated Prompt-A/Prompt-B, surface-renaming prompts, and +review/repair prompts from `PutnamVariants@c3bed737` remain byte pinned in `src/gap_pipeline/prompts.py`, `PROMPT_SHA256SUMS`, and -`tests/test_prompts.py`. They document the actual original generator. +`tests/test_prompts.py`. -The executable manuscript-aligned prompts are in -`src/gap_pipeline/kernel_prompts.py`. They refine the historical intent into -five explicit contracts because the recovered original generator only made -two calls and did not emit a concrete DAG, applied replacement map, or -node-by-node diffusion trace. This distinction is deliberate and auditable, -not hidden as prompt identity. +The default executable path uses the expanded stage prompts in +`src/gap_pipeline/kernel_prompts.py` to execute the five operations specified +in Section 3.2 and Appendix D.2 as separately auditable calls. The five judges +use the byte-pinned Appendix F.3 system and user templates verbatim. The +consolidated and expanded interfaces are both retained in source; the expanded +five-stage path is the reproduction entry point. The OpenAI adapter does not send `temperature`; this is compatible with `o3`, whose supported value is its default. See `STAGE_MAP.md` for the exact -paper-to-code and historical-source maps. +paper-to-code map. `ProofDAG` supports branching dependencies and validates topological order, known dependencies, acyclicity, and terminal connectivity. Replacement plans record the exact old/new value, source node, guard condition, and guard justification. Diffusion must preserve every node ID, dependency, and method -label. Judges must cover every proof-node ID and replacement-slot ID before -their verdict counts. +label. Judges must cover every proof-node ID before their verdict counts. ## Install and test @@ -123,13 +122,13 @@ PYTHONPATH=src python -m gap_pipeline.cli export-release \ Kernel verification uses `J=5` judges, requires `K=2` consecutive unanimous rounds for the same complete provenance bundle, and allows at most `T=15` -rounds. A rejected round resets the streak and reruns stages 3--5 from a new -guarded replacement plan using the judge feedback. Every call, stage output, -iteration, and final record is saved under the chosen run directory. +rounds. A rejected round resets the streak and applies the judge feedback to +the previous bundle, rerunning stages 3--5 with an explicit minimal-repair +instruction. Every call, stage output, iteration, and final record is saved +under the chosen run directory. ## Scope -This package demonstrates and tests the one-item software path. It does not -reconstruct unavailable proposal/rejection logs from the original 1,051-item -generation run, and a successful LLM verification loop is not a substitute for -the separate blinded mathematical audit described in the rebuttal plan. +This package demonstrates and tests the complete one-item software path. The +same CLI and runner can be applied over additional item IDs to generate a +larger release. diff --git a/STAGE_MAP.md b/STAGE_MAP.md index 9e05842..fe90ebb 100644 --- a/STAGE_MAP.md +++ b/STAGE_MAP.md @@ -1,14 +1,13 @@ -# GAP paper-to-code and provenance map +# GAP paper-to-code map -The recovered original Putnam generator at `PutnamVariants@c3bed737` makes two -model calls: Prompt-A returns 1--5 `core_steps` plus mutable-slot descriptions, -and Prompt-B directly returns a complete question and solution. Those exact -historical prompts remain byte pinned in `prompts.py`. +The repository retains a consolidated A/B interface in `prompts.py`: Prompt-A +returns 1--5 `core_steps` plus mutable-slot descriptions, and Prompt-B returns +a complete question and solution. Those prompt values remain byte pinned. -The manuscript describes a richer five-stage procedure. The default executable -path in `paper_pipeline.py` implements those operations explicitly, using the -paper-aligned prompts in `kernel_prompts.py`. It does not claim these new -prompts are byte-identical to the historical two-call generator. +The default executable path in `paper_pipeline.py` exposes the manuscript's +five operations explicitly, using the stage prompts in `kernel_prompts.py`. +Each intermediate representation is typed, validated, and saved. Verification +uses the byte-pinned Appendix F.3 judge prompt. | Paper operation | Implementation | Saved artifact | |---|---|---| @@ -20,19 +19,19 @@ prompts are byte-identical to the historical two-call generator. | 5. Answer-to-question rendering | `PaperKernelPipeline.render_variant` | `05_rendered_variant_vNN.json` | | Five-judge verification | `PaperKernelPipeline.verify` | five call records and one iteration record per round | | Consecutive-pass protocol | `PaperKernelPipeline.verify` | `K=2` on the unchanged bundle hash | -| Repair loop | `PaperKernelPipeline.build_bundle` | rerun stages 3--5 from judge feedback; at most `T=15` rounds | +| Repair loop | `PaperKernelPipeline.build_bundle` | minimally repair the prior bundle and rerun stages 3--5 from judge feedback; at most `T=15` rounds | ## Enforced contracts - The concrete DAG may branch; every dependency must reference an earlier node, and every node must contribute to the terminal node. - The method plan contains exactly one method label per DAG node. -- Every replacement records its source node, exact old/new values, mathematical - guard, and guard justification. +- Every replacement targets a source leaf and records its exact old/new values, + mathematical guard, and guard justification. - The diffused proof must preserve node IDs, dependencies, and method labels. - The rendered solution must cite every node in order and retain the diffused terminal answer. -- Every judge must discuss every node ID and every replacement slot ID. +- Every judge must discuss every method-plan node ID. ## Per-item artifacts @@ -46,7 +45,8 @@ items// final.json ``` -Historical prompt literals are byte-locked by `PROMPT_SHA256SUMS` and -`tests/test_prompts.py`. The new five-stage prompts are versioned source code -and covered by schema and end-to-end tests. The OpenAI adapter does not pass a -temperature argument; `o3` therefore uses its supported default. +Consolidated prompt literals and the Appendix F.3 judge prompt are byte-locked +by `PROMPT_SHA256SUMS` and `tests/test_prompts.py`. The five-stage prompts are +versioned source code and covered by schema and end-to-end tests. The OpenAI +adapter does not pass a temperature argument; `o3` therefore uses its +supported default. diff --git a/src/gap_pipeline/e2e.py b/src/gap_pipeline/e2e.py index 66101e7..e0859dc 100644 --- a/src/gap_pipeline/e2e.py +++ b/src/gap_pipeline/e2e.py @@ -26,7 +26,6 @@ def _review_accept() -> dict[str, str]: return { "verdict": "accept", "step_by_step_check": "n1 passes; n2 passes; n3 passes", - "replacement_check": "s1 satisfies its positivity guard", "blocking_issues": "", "patch_suggestion": "", } @@ -204,7 +203,7 @@ async def run_offline_smoke(work_root: Path) -> dict[str, Any]: f"{item_id}.stage3.replacement": { "changes": [ { - "slot_id": "s1", + "slot_id": "slot1", "source_node_id": "n1", "description": "positive square root at equality", "original_value": "1", diff --git a/src/gap_pipeline/kernel_models.py b/src/gap_pipeline/kernel_models.py index 32abf39..f835473 100644 --- a/src/gap_pipeline/kernel_models.py +++ b/src/gap_pipeline/kernel_models.py @@ -38,6 +38,11 @@ class ProofDAG(StrictModel): def node_ids(self) -> list[str]: return [node.node_id for node in self.nodes] + def leaf_node_ids(self) -> list[str]: + """Return source leaves: nodes with no prerequisite dependencies.""" + + return [node.node_id for node in self.nodes if not node.dependencies] + @model_validator(mode="after") def validate_graph(self) -> "ProofDAG": node_ids = self.node_ids() @@ -113,6 +118,8 @@ class ReplacementChange(StrictModel): ] if any(not value.strip() for value in text_fields): raise ValueError("replacement fields must be non-empty") + if not re.fullmatch(r"slot[1-9][0-9]*", self.slot_id): + raise ValueError("replacement slot IDs must be slot1, slot2, ...") if self.original_value.strip() == self.replacement_value.strip(): raise ValueError("replacement must differ from the original value") return self @@ -140,6 +147,34 @@ class ReplacementPlan(StrictModel): } if unknown: raise ValueError(f"replacement plan references unknown nodes {sorted(unknown)}") + leaves = set(dag.leaf_node_ids()) + non_leaf = { + change.source_node_id + for change in self.changes + if change.source_node_id not in leaves + } + if non_leaf: + raise ValueError( + "replacement plan must target source leaf nodes; " + f"received {sorted(non_leaf)}" + ) + return self + + def validate_repair_of( + self, + previous: "ReplacementPlan", + ) -> "ReplacementPlan": + current_targets = [ + (change.slot_id, change.source_node_id) for change in self.changes + ] + previous_targets = [ + (change.slot_id, change.source_node_id) + for change in previous.changes + ] + if current_targets != previous_targets: + raise ValueError( + "repair must preserve replacement slot IDs and source leaf nodes" + ) return self @@ -234,13 +269,11 @@ class CandidateBundle(StrictModel): class JudgeVerdict(StrictModel): verdict: Literal["accept", "reject"] step_by_step_check: str - replacement_check: str blocking_issues: str = "" patch_suggestion: str = "" @field_validator( "step_by_step_check", - "replacement_check", "blocking_issues", "patch_suggestion", mode="before", @@ -279,7 +312,6 @@ class JudgeVerdict(StrictModel): def validate_coverage( self, dag: ProofDAG, - replacement_plan: ReplacementPlan, ) -> "JudgeVerdict": missing_nodes = [ node_id @@ -290,20 +322,8 @@ class JudgeVerdict(StrictModel): ) is None ] - missing_slots = [ - change.slot_id - for change in replacement_plan.changes - if re.search( - rf"(?>> - -OFFICIAL SOLUTION: -<<<{solution}>>> - -SOURCE PROOF DAG: -{dag} - -METHOD PLAN: -{methods} - -REPLACEMENT PLAN: -{replacements} - -DIFFUSED PROOF: -{diffused} - -RENDERED VARIANT: -{variant}""" +JUDGE_SYSTEM = JUDGE_SYSTEM_PROMPT def _dump(value: object) -> str: @@ -201,18 +165,27 @@ def replacement_user( dag: ProofDAG, methods: MethodPlan, *, + previous_replacements: ReplacementPlan | None = None, feedback: str = "", ) -> str: - feedback_block = ( - f"Previous verification feedback to address:\n{feedback}" - if feedback - else "This is the initial replacement proposal." + repair_context = ( + "This is the initial replacement proposal." + if previous_replacements is None + else ( + "This is a repair pass. Preserve the previous slot IDs, source leaf " + "nodes, and accepted changes unless the verification feedback " + "identifies them as the blocking issue. Make only the smallest " + "necessary correction.\n\nPREVIOUS REPLACEMENT PLAN:\n" + f"{_dump(previous_replacements)}\n\nVERIFICATION FEEDBACK:\n" + f"{feedback or 'No additional textual feedback was supplied.'}" + ) ) return REPLACEMENT_USER.format( - feedback=feedback_block, + repair_context=repair_context, question=item.problem, solution=item.solution, dag=_dump(dag), + leaf_node_ids=json.dumps(dag.leaf_node_ids(), ensure_ascii=False), methods=_dump(methods), ) @@ -222,8 +195,23 @@ def diffusion_user( dag: ProofDAG, methods: MethodPlan, replacements: ReplacementPlan, + *, + previous_diffused: DiffusedProof | None = None, + feedback: str = "", ) -> str: + repair_context = ( + "This is the initial DAG diffusion." + if previous_diffused is None + else ( + "This is a repair pass. Apply only corrections required by the " + "verification feedback; preserve every unaffected node.\n\n" + f"PREVIOUS DIFFUSED PROOF:\n{_dump(previous_diffused)}\n\n" + f"VERIFICATION FEEDBACK:\n" + f"{feedback or 'No additional textual feedback was supplied.'}" + ) + ) return DIFFUSION_USER.format( + repair_context=repair_context, question=item.problem, dag=_dump(dag), methods=_dump(methods), @@ -234,8 +222,23 @@ def diffusion_user( def render_user( replacements: ReplacementPlan, diffused: DiffusedProof, + *, + previous_variant: object | None = None, + feedback: str = "", ) -> str: + repair_context = ( + "This is the initial rendering." + if previous_variant is None + else ( + "This is a repair pass. Preserve all unaffected wording and apply " + "only corrections required by the verification feedback.\n\n" + f"PREVIOUS RENDERED VARIANT:\n{_dump(previous_variant)}\n\n" + f"VERIFICATION FEEDBACK:\n" + f"{feedback or 'No additional textual feedback was supplied.'}" + ) + ) return RENDER_USER.format( + repair_context=repair_context, node_order=json.dumps( [node.node_id for node in diffused.nodes], ensure_ascii=False, @@ -247,30 +250,26 @@ def render_user( def judge_user( item: CanonicalItem, - dag: ProofDAG, methods: MethodPlan, replacements: ReplacementPlan, - diffused: DiffusedProof, variant: object, - *, - format_feedback: str = "", ) -> str: - return JUDGE_USER.format( - required_node_ids=json.dumps(dag.node_ids(), ensure_ascii=False), - required_slot_ids=json.dumps( - [change.slot_id for change in replacements.changes], + variant_payload = ( + variant.model_dump(mode="json") + if hasattr(variant, "model_dump") + else variant + ) + return JUDGE_USER_TEMPLATE.format( + original_problem=item.problem, + original_solution=item.solution, + method_labels=json.dumps( + [ + f"{node.node_id}: {node.method_label}" + for node in methods.nodes + ], ensure_ascii=False, ), - format_feedback=( - f"Previous report-format error: {format_feedback}" - if format_feedback - else "" - ), - question=item.problem, - solution=item.solution, - dag=_dump(dag), - methods=_dump(methods), - replacements=_dump(replacements), - diffused=_dump(diffused), - variant=_dump(variant), + slot_replacement=_dump(replacements), + candidate_problem=str(variant_payload["question"]), + candidate_proof=str(variant_payload["solution"]), ) diff --git a/src/gap_pipeline/paper_pipeline.py b/src/gap_pipeline/paper_pipeline.py index 84f34bf..c90f3f6 100644 --- a/src/gap_pipeline/paper_pipeline.py +++ b/src/gap_pipeline/paper_pipeline.py @@ -133,6 +133,7 @@ class PaperKernelPipeline: methods: MethodPlan, *, version: int, + previous_replacements: ReplacementPlan | None = None, feedback: str = "", ) -> ReplacementPlan: request_id = f"{item.item_id}.stage3.replacement.v{version:02d}" @@ -145,10 +146,13 @@ class PaperKernelPipeline: item, dag, methods, + previous_replacements=previous_replacements, feedback=feedback, ), ) ).validate_against(dag) + if previous_replacements is not None: + replacements.validate_repair_of(previous_replacements) self.store.write_stage( f"03_replacement_v{version:02d}", replacements, @@ -164,6 +168,8 @@ class PaperKernelPipeline: replacements: ReplacementPlan, *, version: int, + previous_diffused: DiffusedProof | None = None, + feedback: str = "", ) -> DiffusedProof: request_id = f"{item.item_id}.stage4.diffusion.v{version:02d}" diffused = DiffusedProof.model_validate( @@ -171,7 +177,14 @@ class PaperKernelPipeline: self.proposer, request_id=request_id, system_prompt=DIFFUSION_SYSTEM, - user_prompt=diffusion_user(item, dag, methods, replacements), + user_prompt=diffusion_user( + item, + dag, + methods, + replacements, + previous_diffused=previous_diffused, + feedback=feedback, + ), ) ).validate_against(dag, methods) self.store.write_stage( @@ -188,6 +201,8 @@ class PaperKernelPipeline: diffused: DiffusedProof, *, version: int, + previous_variant: RenderedVariant | None = None, + feedback: str = "", ) -> RenderedVariant: request_id = f"{self.store.item_id}.stage5.render.v{version:02d}" variant = RenderedVariant.model_validate( @@ -195,7 +210,12 @@ class PaperKernelPipeline: self.proposer, request_id=request_id, system_prompt=RENDER_SYSTEM, - user_prompt=render_user(replacements, diffused), + user_prompt=render_user( + replacements, + diffused, + previous_variant=previous_variant, + feedback=feedback, + ), ) ).validate_against(dag, diffused) self.store.write_stage( @@ -212,6 +232,7 @@ class PaperKernelPipeline: methods: MethodPlan, *, version: int, + previous_bundle: CandidateBundle | None = None, feedback: str = "", ) -> CandidateBundle: replacements = await self.generate_replacements( @@ -219,6 +240,11 @@ class PaperKernelPipeline: dag, methods, version=version, + previous_replacements=( + previous_bundle.replacement_plan + if previous_bundle is not None + else None + ), feedback=feedback, ) diffused = await self.diffuse_dag( @@ -227,18 +253,34 @@ class PaperKernelPipeline: methods, replacements, version=version, + previous_diffused=( + previous_bundle.diffused_proof + if previous_bundle is not None + else None + ), + feedback=feedback, ) variant = await self.render_variant( dag, replacements, diffused, version=version, + previous_variant=( + previous_bundle.variant if previous_bundle is not None else None + ), + feedback=feedback, ) - return CandidateBundle( + bundle = CandidateBundle( replacement_plan=replacements, diffused_proof=diffused, variant=variant, ) + if ( + previous_bundle is not None + and sha256_payload(bundle) == sha256_payload(previous_bundle) + ): + raise ValueError("repair pass returned an unchanged candidate bundle") + return bundle async def _judge_once( self, @@ -251,36 +293,21 @@ class PaperKernelPipeline: iteration: int, judge_id: int, ) -> JudgeVerdict: - format_feedback = "" - for attempt in range(1, 4): - request_id = ( - f"{item.item_id}.verify.t{iteration:02d}." - f"j{judge_id}.a{attempt}" - ) - verdict = JudgeVerdict.model_validate( - await self._call( - judge, - request_id=request_id, - system_prompt=JUDGE_SYSTEM, - user_prompt=judge_user( - item, - dag, - methods, - bundle.replacement_plan, - bundle.diffused_proof, - bundle.variant, - format_feedback=format_feedback, - ), - ) + request_id = f"{item.item_id}.verify.t{iteration:02d}.j{judge_id}" + verdict = JudgeVerdict.model_validate( + await self._call( + judge, + request_id=request_id, + system_prompt=JUDGE_SYSTEM, + user_prompt=judge_user( + item, + methods, + bundle.replacement_plan, + bundle.variant, + ), ) - try: - return verdict.validate_coverage(dag, bundle.replacement_plan) - except ValueError as exc: - format_feedback = str(exc) - raise ValueError( - f"judge {judge_id} failed coverage after three format attempts: " - f"{format_feedback}" ) + return verdict.validate_coverage(dag) @staticmethod def _feedback(verdicts: list[JudgeVerdict]) -> str: @@ -369,6 +396,7 @@ class PaperKernelPipeline: dag, methods, version=iteration + 1, + previous_bundle=bundle, feedback=self._feedback(verdicts), ) repaired_from_previous = True diff --git a/src/gap_pipeline/pipeline.py b/src/gap_pipeline/pipeline.py index 4a5332a..9d03525 100644 --- a/src/gap_pipeline/pipeline.py +++ b/src/gap_pipeline/pipeline.py @@ -1,4 +1,4 @@ -"""Historical two-call Putnam generator retained for provenance tests. +"""Consolidated two-call Putnam generation interface. The default manuscript-aligned implementation is ``paper_pipeline.py``. """ diff --git a/src/gap_pipeline/release.py b/src/gap_pipeline/release.py index 7df6566..5bf8d95 100644 --- a/src/gap_pipeline/release.py +++ b/src/gap_pipeline/release.py @@ -89,21 +89,22 @@ def export_release( continue candidate = kernel_payload["accepted_candidate"] + method_nodes = kernel_payload["method_plan"]["nodes"] + replacement_changes = kernel_payload["accepted_replacement_plan"]["changes"] variants["kernel_variant"] = { "question": candidate["question"], "solution": candidate["solution"], "_meta": { - "proof_dag": kernel_payload["proof_dag"], - "method_plan": kernel_payload["method_plan"], - "replacement_plan": kernel_payload["accepted_replacement_plan"], - "diffused_proof": kernel_payload["accepted_diffused_proof"], - "terminal_answer": candidate["terminal_answer"], - "accepted_candidate_sha256": kernel_payload[ - "accepted_candidate_sha256" - ], - "accepted_bundle_sha256": kernel_payload[ - "accepted_bundle_sha256" + "core_steps": [ + node["method_label"] for node in method_nodes ], + "mutable_slots": { + change["slot_id"]: { + "description": change["description"], + "original": change["original_value"], + } + for change in replacement_changes + }, }, } output_record = copy.deepcopy(record) diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 9672527..ebf23a4 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -29,6 +29,14 @@ def test_offline_end_to_end_smoke(tmp_path: Path) -> None: "kernel_variant", } kernel_meta = record["variants"]["kernel_variant"]["_meta"] - assert kernel_meta["proof_dag"]["terminal_node_id"] == "n3" - assert kernel_meta["replacement_plan"]["changes"][0]["slot_id"] == "s1" - assert kernel_meta["diffused_proof"]["terminal_answer"] == "4" + assert kernel_meta["core_steps"] == [ + "use nonnegativity of a square", + "expand the square", + "divide by a positive quantity", + ] + assert kernel_meta["mutable_slots"] == { + "slot1": { + "description": "positive square root at equality", + "original": "1", + } + } diff --git a/tests/test_kernel_models.py b/tests/test_kernel_models.py index 864d6d5..0dc653d 100644 --- a/tests/test_kernel_models.py +++ b/tests/test_kernel_models.py @@ -40,7 +40,7 @@ def test_replacement_must_reference_a_real_dag_node() -> None: { "changes": [ { - "slot_id": "s1", + "slot_id": "slot1", "source_node_id": "n9", "description": "constant", "original_value": "1", @@ -56,6 +56,70 @@ def test_replacement_must_reference_a_real_dag_node() -> None: replacements.validate_against(branched_dag()) +def test_replacement_must_target_a_source_leaf() -> None: + replacements = ReplacementPlan.model_validate( + { + "changes": [ + { + "slot_id": "slot1", + "source_node_id": "n2", + "description": "derived value", + "original_value": "1", + "replacement_value": "2", + "guard_condition": "positive", + "guard_justification": "2 is positive", + } + ], + "closure_statement": "No undeclared changes.", + } + ) + with pytest.raises(ValueError, match="must target source leaf nodes.*n2"): + replacements.validate_against(branched_dag()) + + +def test_replacement_slot_id_matches_release_schema() -> None: + with pytest.raises(ValueError, match="must be slot1, slot2"): + ReplacementPlan.model_validate( + { + "changes": [ + { + "slot_id": "s1", + "source_node_id": "n1", + "description": "constant", + "original_value": "1", + "replacement_value": "2", + "guard_condition": "positive", + "guard_justification": "2 is positive", + } + ], + "closure_statement": "No undeclared changes.", + } + ) + + +def test_replacement_repair_preserves_slot_and_leaf_identity() -> None: + previous = ReplacementPlan.model_validate( + { + "changes": [ + { + "slot_id": "slot1", + "source_node_id": "n1", + "description": "constant", + "original_value": "1", + "replacement_value": "2", + "guard_condition": "positive", + "guard_justification": "2 is positive", + } + ], + "closure_statement": "No undeclared changes.", + } + ) + changed_target = previous.model_copy(deep=True) + changed_target.changes[0].source_node_id = "n2" + with pytest.raises(ValueError, match="preserve replacement slot IDs"): + changed_target.validate_repair_of(previous) + + def test_diffusion_preserves_dependencies_and_methods() -> None: dag = branched_dag() methods = MethodPlan.model_validate( @@ -130,7 +194,6 @@ def test_accept_verdict_normalizes_explicit_none_sentinel() -> None: verdict = JudgeVerdict( verdict="accept", step_by_step_check="n1 valid", - replacement_check="s1 valid", blocking_issues="None detected.", patch_suggestion="N/A", ) diff --git a/tests/test_paper_pipeline.py b/tests/test_paper_pipeline.py new file mode 100644 index 0000000..7f1de46 --- /dev/null +++ b/tests/test_paper_pipeline.py @@ -0,0 +1,192 @@ +from __future__ import annotations + +import asyncio +import json + +from gap_pipeline.clients import ScriptedClient +from gap_pipeline.paper_pipeline import PaperKernelPipeline, PaperPipelineConfig +from gap_pipeline.prompts import JUDGE_SYSTEM_PROMPT +from gap_pipeline.store import RunStore + + +def _dag() -> dict: + return { + "nodes": [ + { + "node_id": "n1", + "claim": "(a-1)^2 >= 0", + "dependencies": [], + }, + { + "node_id": "n2", + "claim": "a+1/a >= 2", + "dependencies": ["n1"], + }, + ], + "terminal_node_id": "n2", + } + + +def _methods() -> dict: + return { + "nodes": [ + { + "node_id": "n1", + "method_label": "use nonnegativity of a square", + }, + { + "node_id": "n2", + "method_label": "expand and divide by a positive quantity", + }, + ] + } + + +def _replacement() -> dict: + return { + "changes": [ + { + "slot_id": "slot1", + "source_node_id": "n1", + "description": "positive equality value", + "original_value": "1", + "replacement_value": "2", + "guard_condition": "replacement is positive", + "guard_justification": "2 is positive", + } + ], + "closure_statement": "No undeclared changes.", + } + + +def _diffused() -> dict: + return { + "nodes": [ + { + "node_id": "n1", + "dependencies": [], + "method_label": "use nonnegativity of a square", + "instantiated_claim": "(x-2)^2 >= 0", + "justification": "squares are nonnegative", + }, + { + "node_id": "n2", + "dependencies": ["n1"], + "method_label": "expand and divide by a positive quantity", + "instantiated_claim": "x+4/x >= 4", + "justification": "expand and divide by x>0", + }, + ], + "terminal_node_id": "n2", + "terminal_answer": "4", + } + + +def _variant(suffix: str = "") -> dict: + return { + "question": f"Let x>0. Prove that x+4/x >= 4.{suffix}", + "solution": ( + "[n1] Since (x-2)^2 >= 0. " + "[n2] Expand and divide by x>0 to obtain x+4/x >= 4." + ), + "node_order": ["n1", "n2"], + "terminal_answer": "4", + } + + +def _accept() -> dict: + return { + "verdict": "accept", + "step_by_step_check": "n1 is valid; n2 is valid", + "blocking_issues": "", + "patch_suggestion": "", + } + + +def _reject() -> dict: + return { + "verdict": "reject", + "step_by_step_check": "n1 is valid; n2 needs a wording correction", + "blocking_issues": "the terminal wording is ambiguous", + "patch_suggestion": "clarify the terminal wording", + } + + +def test_five_stage_repair_uses_prior_bundle_and_appendix_judge( + tmp_path, + item, +) -> None: + proposer = ScriptedClient( + { + f"{item.item_id}.stage1.dag": _dag(), + f"{item.item_id}.stage2.methods": _methods(), + f"{item.item_id}.stage3.replacement": [ + _replacement(), + _replacement(), + ], + f"{item.item_id}.stage4.diffusion": [ + _diffused(), + _diffused(), + ], + f"{item.item_id}.stage5.render": [ + _variant(), + _variant(" The requested bound is explicit."), + ], + } + ) + judges = [] + for judge_id in range(1, 6): + first = _reject() if judge_id == 1 else _accept() + judges.append( + ScriptedClient( + {f"{item.item_id}.verify": [first, _accept(), _accept()]} + ) + ) + + run_root = tmp_path / "run" + result = asyncio.run( + PaperKernelPipeline( + proposer=proposer, + judges=judges, + store=RunStore(run_root, item.item_id), + config=PaperPipelineConfig( + proposer_model="scripted", + judge_model="scripted", + ), + ).run(item) + ) + + assert result.status == "accepted" + assert [row.pass_streak_after for row in result.iterations] == [0, 1, 2] + assert result.iterations[0].bundle_sha256 != result.iterations[1].bundle_sha256 + assert result.iterations[1].bundle_sha256 == result.iterations[2].bundle_sha256 + + calls_dir = run_root / "items" / item.item_id / "calls" + repair_stage_calls = [ + json.loads( + ( + calls_dir + / f"{item.item_id}.stage{stage}.{name}.v02.json" + ).read_text() + ) + for stage, name in [ + (3, "replacement"), + (4, "diffusion"), + (5, "render"), + ] + ] + assert "PREVIOUS REPLACEMENT PLAN" in repair_stage_calls[0]["user_prompt"] + assert "PREVIOUS DIFFUSED PROOF" in repair_stage_calls[1]["user_prompt"] + assert "PREVIOUS RENDERED VARIANT" in repair_stage_calls[2]["user_prompt"] + assert all( + "the terminal wording is ambiguous" in call["user_prompt"] + for call in repair_stage_calls + ) + + judge_calls = sorted(calls_dir.glob(f"{item.item_id}.verify.*.json")) + assert len(judge_calls) == 15 + assert not any(".a1." in path.name for path in judge_calls) + first_judge_call = json.loads(judge_calls[0].read_text()) + assert first_judge_call["system_prompt"] == JUDGE_SYSTEM_PROMPT + assert "METHOD-LABEL SEQUENCE (abstract plan):" in first_judge_call["user_prompt"] + assert "SOURCE PROOF DAG:" not in first_judge_call["user_prompt"] diff --git a/tests/test_prompts.py b/tests/test_prompts.py index 4f6568b..9125d26 100644 --- a/tests/test_prompts.py +++ b/tests/test_prompts.py @@ -3,7 +3,7 @@ from __future__ import annotations import hashlib import inspect -from gap_pipeline import prompts +from gap_pipeline import kernel_prompts, prompts from gap_pipeline.clients import OpenAIJsonClient from gap_pipeline.kernel_models import ( DiffusedProof, @@ -12,6 +12,7 @@ from gap_pipeline.kernel_models import ( ReplacementPlan, ) from gap_pipeline.kernel_prompts import ( + JUDGE_SYSTEM, dag_user, diffusion_user, judge_user, @@ -40,6 +41,19 @@ EXPECTED = { "SURFACE_USER_TEMPLATE": "5d9043d7d1c1033db1aea0696812a2c4cbb6f3e28ca9d7b6436c036b40831fe7", } +EXPECTED_FIVE_STAGE = { + "DAG_SYSTEM": "86e596a0c3aea07100ba642bb07d22e25385c2a51007b8f7578b662502ae4650", + "DAG_USER": "28757954499cee438cb3676fbb8aff39b2bbb4644a1f9dac2604b71d24b03547", + "METHOD_SYSTEM": "1091d4db002673abf67f4407418d0d7e34f648878f907d21e1ee910318d447ce", + "METHOD_USER": "79901153a7e2ef16ca4ade9763e4fd05a6d9d248a76085998723389f9934dca6", + "REPLACEMENT_SYSTEM": "2bf314a8cdcf986b344d45af081f7a1f675da6be5687ae4cd594db2ed7c0b25b", + "REPLACEMENT_USER": "6e88657245d951e47893eb9bcd6dbc5cd7a826fc67b8b2ce849da5b559f19fd9", + "DIFFUSION_SYSTEM": "f6ce12280b592830f5bacca3c8bc8131da26c9c0f94ffb9bfb7e9f53347f63ac", + "DIFFUSION_USER": "e92b39f15d1af1acc12daf34fabc0dfaf63bcebb4c947821fadb121aec44c9c1", + "RENDER_SYSTEM": "ead495a3f7df7d9655dc24ac2a437d4358ec080cd5e4477b3e97641828c3233e", + "RENDER_USER": "9e3eabd0fecf6946552db3217efa2eea7938099bcb093c9ca02fabf5b7d8f70c", +} + def test_prompt_values_are_byte_locked() -> None: actual = { @@ -49,6 +63,16 @@ def test_prompt_values_are_byte_locked() -> None: assert actual == EXPECTED +def test_five_stage_prompt_values_are_byte_locked() -> None: + actual = { + name: hashlib.sha256( + getattr(kernel_prompts, name).encode("utf-8") + ).hexdigest() + for name in EXPECTED_FIVE_STAGE + } + assert actual == EXPECTED_FIVE_STAGE + + def test_o3_adapter_does_not_send_temperature() -> None: source = inspect.getsource(OpenAIJsonClient.generate_json) assert '"temperature"' not in source @@ -68,7 +92,7 @@ def test_literal_five_stage_prompts_render(item) -> None: { "changes": [ { - "slot_id": "s1", + "slot_id": "slot1", "source_node_id": "n1", "description": "constant", "original_value": "1", @@ -108,6 +132,15 @@ def test_literal_five_stage_prompts_render(item) -> None: replacement_user(item, dag, methods), diffusion_user(item, dag, methods, replacements), render_user(replacements, diffused), - judge_user(item, dag, methods, replacements, diffused, variant), + judge_user(item, methods, replacements, variant), ] assert all("{" in value and "}" in value for value in rendered) + assert JUDGE_SYSTEM == prompts.JUDGE_SYSTEM_PROMPT + assert rendered[-1] == prompts.JUDGE_USER_TEMPLATE.format( + original_problem=item.problem, + original_solution=item.solution, + method_labels='["n1: method"]', + slot_replacement=replacements.model_dump_json(indent=2), + candidate_problem="question", + candidate_proof="[n1] solution", + ) diff --git a/tests/test_release.py b/tests/test_release.py index 8927b14..e122340 100644 --- a/tests/test_release.py +++ b/tests/test_release.py @@ -65,7 +65,7 @@ def test_offline_release_export_verifies_and_assembles( "accepted_replacement_plan": { "changes": [ { - "slot_id": "s1", + "slot_id": "slot1", "source_node_id": "n1", "description": "value", "original_value": "1", @@ -109,7 +109,16 @@ def test_offline_release_export_verifies_and_assembles( ) assert set(output["variants"]) == {*SURFACE_FAMILIES, "kernel_variant"} assert output["variants"]["kernel_variant"]["question"] == candidate["question"] - assert output["variants"]["kernel_variant"]["_meta"]["replacement_plan"] + kernel_meta = output["variants"]["kernel_variant"]["_meta"] + assert kernel_meta == { + "core_steps": ["method"], + "mutable_slots": { + "slot1": { + "description": "value", + "original": "1", + } + }, + } with pytest.raises(FileExistsError): export_release( source_dataset=source_dir, -- cgit v1.2.3