summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--GAP_End_to_End.ipynb11
-rw-r--r--README.md37
-rw-r--r--STAGE_MAP.md34
-rw-r--r--src/gap_pipeline/e2e.py3
-rw-r--r--src/gap_pipeline/kernel_models.py54
-rw-r--r--src/gap_pipeline/kernel_prompts.py153
-rw-r--r--src/gap_pipeline/paper_pipeline.py90
-rw-r--r--src/gap_pipeline/pipeline.py2
-rw-r--r--src/gap_pipeline/release.py21
-rw-r--r--tests/test_e2e.py14
-rw-r--r--tests/test_kernel_models.py67
-rw-r--r--tests/test_paper_pipeline.py192
-rw-r--r--tests/test_prompts.py39
-rw-r--r--tests/test_release.py13
14 files changed, 539 insertions, 191 deletions
diff --git a/GAP_End_to_End.ipynb b/GAP_End_to_End.ipynb
index 2b8c10e..7555e81 100644
--- a/GAP_End_to_End.ipynb
+++ b/GAP_End_to_End.ipynb
@@ -149,18 +149,15 @@
"\n",
"kernel = record[\"variants\"][\"kernel_variant\"]\n",
"meta = kernel[\"_meta\"]\n",
- "assert meta[\"proof_dag\"][\"nodes\"]\n",
- "assert meta[\"method_plan\"][\"nodes\"]\n",
- "assert meta[\"replacement_plan\"][\"changes\"]\n",
- "assert meta[\"diffused_proof\"][\"nodes\"]\n",
- "assert meta[\"accepted_bundle_sha256\"]\n",
+ "assert meta[\"core_steps\"]\n",
+ "assert meta[\"mutable_slots\"]\n",
"\n",
"print(\"END-TO-END PASS\")\n",
"print(f\"Item: {record['index']}\")\n",
"print(f\"Variant families: {sorted(record['variants'])}\")\n",
"print(f\"Verification rounds: {result['verification_rounds']}\")\n",
- "print(f\"Declared replacements: {len(meta['replacement_plan']['changes'])}\")\n",
- "print(f\"Proof-DAG nodes: {len(meta['proof_dag']['nodes'])}\")\n",
+ "print(f\"Declared mutable slots: {len(meta['mutable_slots'])}\")\n",
+ "print(f\"Core proof steps: {len(meta['core_steps'])}\")\n",
"print(f\"Kernel question preview: {kernel['question'][:500]}\")\n",
"print(f\"Exported record: {result['release_record']}\")\n",
"print(f\"Manifest: {result['manifest']}\")"
diff --git a/README.md b/README.md
index 2a34006..bcc7ff4 100644
--- a/README.md
+++ b/README.md
@@ -36,28 +36,27 @@ mathematics.
## Prompt fidelity
-The historical two-call Prompt-A/Prompt-B, surface-renaming prompts, and
-review/repair prompts recovered from `PutnamVariants@c3bed737` remain byte
+The consolidated Prompt-A/Prompt-B, surface-renaming prompts, and
+review/repair prompts from `PutnamVariants@c3bed737` remain byte
pinned in `src/gap_pipeline/prompts.py`, `PROMPT_SHA256SUMS`, and
-`tests/test_prompts.py`. They document the actual original generator.
+`tests/test_prompts.py`.
-The executable manuscript-aligned prompts are in
-`src/gap_pipeline/kernel_prompts.py`. They refine the historical intent into
-five explicit contracts because the recovered original generator only made
-two calls and did not emit a concrete DAG, applied replacement map, or
-node-by-node diffusion trace. This distinction is deliberate and auditable,
-not hidden as prompt identity.
+The default executable path uses the expanded stage prompts in
+`src/gap_pipeline/kernel_prompts.py` to execute the five operations specified
+in Section 3.2 and Appendix D.2 as separately auditable calls. The five judges
+use the byte-pinned Appendix F.3 system and user templates verbatim. The
+consolidated and expanded interfaces are both retained in source; the expanded
+five-stage path is the reproduction entry point.
The OpenAI adapter does not send `temperature`; this is compatible with `o3`,
whose supported value is its default. See `STAGE_MAP.md` for the exact
-paper-to-code and historical-source maps.
+paper-to-code map.
`ProofDAG` supports branching dependencies and validates topological order,
known dependencies, acyclicity, and terminal connectivity. Replacement plans
record the exact old/new value, source node, guard condition, and guard
justification. Diffusion must preserve every node ID, dependency, and method
-label. Judges must cover every proof-node ID and replacement-slot ID before
-their verdict counts.
+label. Judges must cover every proof-node ID before their verdict counts.
## Install and test
@@ -123,13 +122,13 @@ PYTHONPATH=src python -m gap_pipeline.cli export-release \
Kernel verification uses `J=5` judges, requires `K=2` consecutive unanimous
rounds for the same complete provenance bundle, and allows at most `T=15`
-rounds. A rejected round resets the streak and reruns stages 3--5 from a new
-guarded replacement plan using the judge feedback. Every call, stage output,
-iteration, and final record is saved under the chosen run directory.
+rounds. A rejected round resets the streak and applies the judge feedback to
+the previous bundle, rerunning stages 3--5 with an explicit minimal-repair
+instruction. Every call, stage output, iteration, and final record is saved
+under the chosen run directory.
## Scope
-This package demonstrates and tests the one-item software path. It does not
-reconstruct unavailable proposal/rejection logs from the original 1,051-item
-generation run, and a successful LLM verification loop is not a substitute for
-the separate blinded mathematical audit described in the rebuttal plan.
+This package demonstrates and tests the complete one-item software path. The
+same CLI and runner can be applied over additional item IDs to generate a
+larger release.
diff --git a/STAGE_MAP.md b/STAGE_MAP.md
index 9e05842..fe90ebb 100644
--- a/STAGE_MAP.md
+++ b/STAGE_MAP.md
@@ -1,14 +1,13 @@
-# GAP paper-to-code and provenance map
+# GAP paper-to-code map
-The recovered original Putnam generator at `PutnamVariants@c3bed737` makes two
-model calls: Prompt-A returns 1--5 `core_steps` plus mutable-slot descriptions,
-and Prompt-B directly returns a complete question and solution. Those exact
-historical prompts remain byte pinned in `prompts.py`.
+The repository retains a consolidated A/B interface in `prompts.py`: Prompt-A
+returns 1--5 `core_steps` plus mutable-slot descriptions, and Prompt-B returns
+a complete question and solution. Those prompt values remain byte pinned.
-The manuscript describes a richer five-stage procedure. The default executable
-path in `paper_pipeline.py` implements those operations explicitly, using the
-paper-aligned prompts in `kernel_prompts.py`. It does not claim these new
-prompts are byte-identical to the historical two-call generator.
+The default executable path in `paper_pipeline.py` exposes the manuscript's
+five operations explicitly, using the stage prompts in `kernel_prompts.py`.
+Each intermediate representation is typed, validated, and saved. Verification
+uses the byte-pinned Appendix F.3 judge prompt.
| Paper operation | Implementation | Saved artifact |
|---|---|---|
@@ -20,19 +19,19 @@ prompts are byte-identical to the historical two-call generator.
| 5. Answer-to-question rendering | `PaperKernelPipeline.render_variant` | `05_rendered_variant_vNN.json` |
| Five-judge verification | `PaperKernelPipeline.verify` | five call records and one iteration record per round |
| Consecutive-pass protocol | `PaperKernelPipeline.verify` | `K=2` on the unchanged bundle hash |
-| Repair loop | `PaperKernelPipeline.build_bundle` | rerun stages 3--5 from judge feedback; at most `T=15` rounds |
+| Repair loop | `PaperKernelPipeline.build_bundle` | minimally repair the prior bundle and rerun stages 3--5 from judge feedback; at most `T=15` rounds |
## Enforced contracts
- The concrete DAG may branch; every dependency must reference an earlier node,
and every node must contribute to the terminal node.
- The method plan contains exactly one method label per DAG node.
-- Every replacement records its source node, exact old/new values, mathematical
- guard, and guard justification.
+- Every replacement targets a source leaf and records its exact old/new values,
+ mathematical guard, and guard justification.
- The diffused proof must preserve node IDs, dependencies, and method labels.
- The rendered solution must cite every node in order and retain the diffused
terminal answer.
-- Every judge must discuss every node ID and every replacement slot ID.
+- Every judge must discuss every method-plan node ID.
## Per-item artifacts
@@ -46,7 +45,8 @@ items/<item-id>/
final.json
```
-Historical prompt literals are byte-locked by `PROMPT_SHA256SUMS` and
-`tests/test_prompts.py`. The new five-stage prompts are versioned source code
-and covered by schema and end-to-end tests. The OpenAI adapter does not pass a
-temperature argument; `o3` therefore uses its supported default.
+Consolidated prompt literals and the Appendix F.3 judge prompt are byte-locked
+by `PROMPT_SHA256SUMS` and `tests/test_prompts.py`. The five-stage prompts are
+versioned source code and covered by schema and end-to-end tests. The OpenAI
+adapter does not pass a temperature argument; `o3` therefore uses its
+supported default.
diff --git a/src/gap_pipeline/e2e.py b/src/gap_pipeline/e2e.py
index 66101e7..e0859dc 100644
--- a/src/gap_pipeline/e2e.py
+++ b/src/gap_pipeline/e2e.py
@@ -26,7 +26,6 @@ def _review_accept() -> dict[str, str]:
return {
"verdict": "accept",
"step_by_step_check": "n1 passes; n2 passes; n3 passes",
- "replacement_check": "s1 satisfies its positivity guard",
"blocking_issues": "",
"patch_suggestion": "",
}
@@ -204,7 +203,7 @@ async def run_offline_smoke(work_root: Path) -> dict[str, Any]:
f"{item_id}.stage3.replacement": {
"changes": [
{
- "slot_id": "s1",
+ "slot_id": "slot1",
"source_node_id": "n1",
"description": "positive square root at equality",
"original_value": "1",
diff --git a/src/gap_pipeline/kernel_models.py b/src/gap_pipeline/kernel_models.py
index 32abf39..f835473 100644
--- a/src/gap_pipeline/kernel_models.py
+++ b/src/gap_pipeline/kernel_models.py
@@ -38,6 +38,11 @@ class ProofDAG(StrictModel):
def node_ids(self) -> list[str]:
return [node.node_id for node in self.nodes]
+ def leaf_node_ids(self) -> list[str]:
+ """Return source leaves: nodes with no prerequisite dependencies."""
+
+ return [node.node_id for node in self.nodes if not node.dependencies]
+
@model_validator(mode="after")
def validate_graph(self) -> "ProofDAG":
node_ids = self.node_ids()
@@ -113,6 +118,8 @@ class ReplacementChange(StrictModel):
]
if any(not value.strip() for value in text_fields):
raise ValueError("replacement fields must be non-empty")
+ if not re.fullmatch(r"slot[1-9][0-9]*", self.slot_id):
+ raise ValueError("replacement slot IDs must be slot1, slot2, ...")
if self.original_value.strip() == self.replacement_value.strip():
raise ValueError("replacement must differ from the original value")
return self
@@ -140,6 +147,34 @@ class ReplacementPlan(StrictModel):
}
if unknown:
raise ValueError(f"replacement plan references unknown nodes {sorted(unknown)}")
+ leaves = set(dag.leaf_node_ids())
+ non_leaf = {
+ change.source_node_id
+ for change in self.changes
+ if change.source_node_id not in leaves
+ }
+ if non_leaf:
+ raise ValueError(
+ "replacement plan must target source leaf nodes; "
+ f"received {sorted(non_leaf)}"
+ )
+ return self
+
+ def validate_repair_of(
+ self,
+ previous: "ReplacementPlan",
+ ) -> "ReplacementPlan":
+ current_targets = [
+ (change.slot_id, change.source_node_id) for change in self.changes
+ ]
+ previous_targets = [
+ (change.slot_id, change.source_node_id)
+ for change in previous.changes
+ ]
+ if current_targets != previous_targets:
+ raise ValueError(
+ "repair must preserve replacement slot IDs and source leaf nodes"
+ )
return self
@@ -234,13 +269,11 @@ class CandidateBundle(StrictModel):
class JudgeVerdict(StrictModel):
verdict: Literal["accept", "reject"]
step_by_step_check: str
- replacement_check: str
blocking_issues: str = ""
patch_suggestion: str = ""
@field_validator(
"step_by_step_check",
- "replacement_check",
"blocking_issues",
"patch_suggestion",
mode="before",
@@ -279,7 +312,6 @@ class JudgeVerdict(StrictModel):
def validate_coverage(
self,
dag: ProofDAG,
- replacement_plan: ReplacementPlan,
) -> "JudgeVerdict":
missing_nodes = [
node_id
@@ -290,20 +322,8 @@ class JudgeVerdict(StrictModel):
)
is None
]
- missing_slots = [
- change.slot_id
- for change in replacement_plan.changes
- if re.search(
- rf"(?<![A-Za-z0-9_]){re.escape(change.slot_id)}(?![A-Za-z0-9_])",
- self.replacement_check,
- )
- is None
- ]
- if missing_nodes or missing_slots:
- raise ValueError(
- "judge coverage incomplete: "
- f"nodes={missing_nodes}, slots={missing_slots}"
- )
+ if missing_nodes:
+ raise ValueError(f"judge coverage incomplete: nodes={missing_nodes}")
return self
diff --git a/src/gap_pipeline/kernel_prompts.py b/src/gap_pipeline/kernel_prompts.py
index c4a6128..f932d0a 100644
--- a/src/gap_pipeline/kernel_prompts.py
+++ b/src/gap_pipeline/kernel_prompts.py
@@ -1,8 +1,9 @@
-"""Prompts for an explicit implementation of the paper's five kernel stages.
+"""Prompts for the paper's explicit five-stage kernel implementation.
-The byte-pinned historical Prompt-A/Prompt-B remain in ``prompts.py``. These
-prompts make the richer five-stage manuscript description executable and keep
-every transformation decision in a typed artifact.
+The consolidated Prompt-A/Prompt-B interface remains byte-pinned in
+``prompts.py``. The default pipeline exposes the manuscript's five operations
+as separately validated artifacts and uses its Appendix F.3 judge prompt
+verbatim.
"""
from __future__ import annotations
@@ -16,6 +17,7 @@ from .kernel_models import (
ReplacementPlan,
)
from .models import CanonicalItem
+from .prompts import JUDGE_SYSTEM_PROMPT, JUDGE_USER_TEMPLATE
DAG_SYSTEM = "You are a rigorous competition-math proof analyst."
@@ -55,18 +57,19 @@ replacements that create a genuinely new problem while preserving the supplied
method plan.
For every change:
-- state the exact source DAG node;
+- target one of the disclosed source leaf DAG nodes;
+- state that exact source DAG node;
- record the original and replacement values explicitly;
- state a mathematical guard condition derived from the original problem;
- explain why the replacement satisfies the guard.
The list must be closed: stages 4 and 5 may introduce no mathematical change
that is not declared here. Variable renaming alone is not a kernel change.
-{feedback}
+{repair_context}
Return JSON only:
{{"changes":[
- {{"slot_id":"s1","source_node_id":"n1","description":"...",
+ {{"slot_id":"slot1","source_node_id":"n1","description":"...",
"original_value":"...","replacement_value":"...",
"guard_condition":"...","guard_justification":"..."}}
],
@@ -81,6 +84,9 @@ OFFICIAL SOLUTION:
CONCRETE PROOF DAG:
{dag}
+SOURCE LEAF NODE IDS:
+{leaf_node_ids}
+
CONTENT-FREE METHOD PLAN:
{methods}"""
@@ -90,6 +96,7 @@ DIFFUSION_USER = """Propagate only the declared replacements through the proof
DAG, node by node. Return exactly one row for every source node, preserving its
ID, dependencies, and method label. Do not introduce undeclared constants,
objects, assumptions, reductions, or proof methods.
+{repair_context}
Return JSON only:
{{"nodes":[
@@ -115,6 +122,7 @@ DECLARED REPLACEMENTS:
RENDER_SYSTEM = "You render a verified proof DAG into a self-contained Putnam problem."
RENDER_USER = """Render the diffused proof into one complete problem statement
and one complete solution.
+{repair_context}
Requirements:
- the question must be fully determined by the diffused terminal claim;
@@ -135,51 +143,7 @@ DIFFUSED PROOF:
{diffused}"""
-JUDGE_SYSTEM = """You are a verification judge for a literal five-stage GAP
-kernel transformation. Verification, not de novo solving, is your task."""
-JUDGE_USER = """Check the candidate against every disclosed artifact.
-
-You must verify:
-1. every replacement satisfies its guard and all mathematical changes are
- declared in the replacement plan;
-2. each diffused node instantiates the matching source node and method label;
-3. dependencies and proof order are preserved;
-4. the rendered problem is well-posed and the rendered solution proves it;
-5. the terminal answer agrees across diffusion, rendering, and solution.
-
-In step_by_step_check, mention each of these node IDs explicitly:
-{required_node_ids}
-In replacement_check, mention each of these slot IDs explicitly:
-{required_slot_ids}
-{format_feedback}
-
-Return JSON only:
-{{"verdict":"accept" or "reject",
- "step_by_step_check":"...",
- "replacement_check":"...",
- "blocking_issues":"...",
- "patch_suggestion":"..."}}
-
-ORIGINAL PROBLEM:
-<<<{question}>>>
-
-OFFICIAL SOLUTION:
-<<<{solution}>>>
-
-SOURCE PROOF DAG:
-{dag}
-
-METHOD PLAN:
-{methods}
-
-REPLACEMENT PLAN:
-{replacements}
-
-DIFFUSED PROOF:
-{diffused}
-
-RENDERED VARIANT:
-{variant}"""
+JUDGE_SYSTEM = JUDGE_SYSTEM_PROMPT
def _dump(value: object) -> str:
@@ -201,18 +165,27 @@ def replacement_user(
dag: ProofDAG,
methods: MethodPlan,
*,
+ previous_replacements: ReplacementPlan | None = None,
feedback: str = "",
) -> str:
- feedback_block = (
- f"Previous verification feedback to address:\n{feedback}"
- if feedback
- else "This is the initial replacement proposal."
+ repair_context = (
+ "This is the initial replacement proposal."
+ if previous_replacements is None
+ else (
+ "This is a repair pass. Preserve the previous slot IDs, source leaf "
+ "nodes, and accepted changes unless the verification feedback "
+ "identifies them as the blocking issue. Make only the smallest "
+ "necessary correction.\n\nPREVIOUS REPLACEMENT PLAN:\n"
+ f"{_dump(previous_replacements)}\n\nVERIFICATION FEEDBACK:\n"
+ f"{feedback or 'No additional textual feedback was supplied.'}"
+ )
)
return REPLACEMENT_USER.format(
- feedback=feedback_block,
+ repair_context=repair_context,
question=item.problem,
solution=item.solution,
dag=_dump(dag),
+ leaf_node_ids=json.dumps(dag.leaf_node_ids(), ensure_ascii=False),
methods=_dump(methods),
)
@@ -222,8 +195,23 @@ def diffusion_user(
dag: ProofDAG,
methods: MethodPlan,
replacements: ReplacementPlan,
+ *,
+ previous_diffused: DiffusedProof | None = None,
+ feedback: str = "",
) -> str:
+ repair_context = (
+ "This is the initial DAG diffusion."
+ if previous_diffused is None
+ else (
+ "This is a repair pass. Apply only corrections required by the "
+ "verification feedback; preserve every unaffected node.\n\n"
+ f"PREVIOUS DIFFUSED PROOF:\n{_dump(previous_diffused)}\n\n"
+ f"VERIFICATION FEEDBACK:\n"
+ f"{feedback or 'No additional textual feedback was supplied.'}"
+ )
+ )
return DIFFUSION_USER.format(
+ repair_context=repair_context,
question=item.problem,
dag=_dump(dag),
methods=_dump(methods),
@@ -234,8 +222,23 @@ def diffusion_user(
def render_user(
replacements: ReplacementPlan,
diffused: DiffusedProof,
+ *,
+ previous_variant: object | None = None,
+ feedback: str = "",
) -> str:
+ repair_context = (
+ "This is the initial rendering."
+ if previous_variant is None
+ else (
+ "This is a repair pass. Preserve all unaffected wording and apply "
+ "only corrections required by the verification feedback.\n\n"
+ f"PREVIOUS RENDERED VARIANT:\n{_dump(previous_variant)}\n\n"
+ f"VERIFICATION FEEDBACK:\n"
+ f"{feedback or 'No additional textual feedback was supplied.'}"
+ )
+ )
return RENDER_USER.format(
+ repair_context=repair_context,
node_order=json.dumps(
[node.node_id for node in diffused.nodes],
ensure_ascii=False,
@@ -247,30 +250,26 @@ def render_user(
def judge_user(
item: CanonicalItem,
- dag: ProofDAG,
methods: MethodPlan,
replacements: ReplacementPlan,
- diffused: DiffusedProof,
variant: object,
- *,
- format_feedback: str = "",
) -> str:
- return JUDGE_USER.format(
- required_node_ids=json.dumps(dag.node_ids(), ensure_ascii=False),
- required_slot_ids=json.dumps(
- [change.slot_id for change in replacements.changes],
+ variant_payload = (
+ variant.model_dump(mode="json")
+ if hasattr(variant, "model_dump")
+ else variant
+ )
+ return JUDGE_USER_TEMPLATE.format(
+ original_problem=item.problem,
+ original_solution=item.solution,
+ method_labels=json.dumps(
+ [
+ f"{node.node_id}: {node.method_label}"
+ for node in methods.nodes
+ ],
ensure_ascii=False,
),
- format_feedback=(
- f"Previous report-format error: {format_feedback}"
- if format_feedback
- else ""
- ),
- question=item.problem,
- solution=item.solution,
- dag=_dump(dag),
- methods=_dump(methods),
- replacements=_dump(replacements),
- diffused=_dump(diffused),
- variant=_dump(variant),
+ slot_replacement=_dump(replacements),
+ candidate_problem=str(variant_payload["question"]),
+ candidate_proof=str(variant_payload["solution"]),
)
diff --git a/src/gap_pipeline/paper_pipeline.py b/src/gap_pipeline/paper_pipeline.py
index 84f34bf..c90f3f6 100644
--- a/src/gap_pipeline/paper_pipeline.py
+++ b/src/gap_pipeline/paper_pipeline.py
@@ -133,6 +133,7 @@ class PaperKernelPipeline:
methods: MethodPlan,
*,
version: int,
+ previous_replacements: ReplacementPlan | None = None,
feedback: str = "",
) -> ReplacementPlan:
request_id = f"{item.item_id}.stage3.replacement.v{version:02d}"
@@ -145,10 +146,13 @@ class PaperKernelPipeline:
item,
dag,
methods,
+ previous_replacements=previous_replacements,
feedback=feedback,
),
)
).validate_against(dag)
+ if previous_replacements is not None:
+ replacements.validate_repair_of(previous_replacements)
self.store.write_stage(
f"03_replacement_v{version:02d}",
replacements,
@@ -164,6 +168,8 @@ class PaperKernelPipeline:
replacements: ReplacementPlan,
*,
version: int,
+ previous_diffused: DiffusedProof | None = None,
+ feedback: str = "",
) -> DiffusedProof:
request_id = f"{item.item_id}.stage4.diffusion.v{version:02d}"
diffused = DiffusedProof.model_validate(
@@ -171,7 +177,14 @@ class PaperKernelPipeline:
self.proposer,
request_id=request_id,
system_prompt=DIFFUSION_SYSTEM,
- user_prompt=diffusion_user(item, dag, methods, replacements),
+ user_prompt=diffusion_user(
+ item,
+ dag,
+ methods,
+ replacements,
+ previous_diffused=previous_diffused,
+ feedback=feedback,
+ ),
)
).validate_against(dag, methods)
self.store.write_stage(
@@ -188,6 +201,8 @@ class PaperKernelPipeline:
diffused: DiffusedProof,
*,
version: int,
+ previous_variant: RenderedVariant | None = None,
+ feedback: str = "",
) -> RenderedVariant:
request_id = f"{self.store.item_id}.stage5.render.v{version:02d}"
variant = RenderedVariant.model_validate(
@@ -195,7 +210,12 @@ class PaperKernelPipeline:
self.proposer,
request_id=request_id,
system_prompt=RENDER_SYSTEM,
- user_prompt=render_user(replacements, diffused),
+ user_prompt=render_user(
+ replacements,
+ diffused,
+ previous_variant=previous_variant,
+ feedback=feedback,
+ ),
)
).validate_against(dag, diffused)
self.store.write_stage(
@@ -212,6 +232,7 @@ class PaperKernelPipeline:
methods: MethodPlan,
*,
version: int,
+ previous_bundle: CandidateBundle | None = None,
feedback: str = "",
) -> CandidateBundle:
replacements = await self.generate_replacements(
@@ -219,6 +240,11 @@ class PaperKernelPipeline:
dag,
methods,
version=version,
+ previous_replacements=(
+ previous_bundle.replacement_plan
+ if previous_bundle is not None
+ else None
+ ),
feedback=feedback,
)
diffused = await self.diffuse_dag(
@@ -227,18 +253,34 @@ class PaperKernelPipeline:
methods,
replacements,
version=version,
+ previous_diffused=(
+ previous_bundle.diffused_proof
+ if previous_bundle is not None
+ else None
+ ),
+ feedback=feedback,
)
variant = await self.render_variant(
dag,
replacements,
diffused,
version=version,
+ previous_variant=(
+ previous_bundle.variant if previous_bundle is not None else None
+ ),
+ feedback=feedback,
)
- return CandidateBundle(
+ bundle = CandidateBundle(
replacement_plan=replacements,
diffused_proof=diffused,
variant=variant,
)
+ if (
+ previous_bundle is not None
+ and sha256_payload(bundle) == sha256_payload(previous_bundle)
+ ):
+ raise ValueError("repair pass returned an unchanged candidate bundle")
+ return bundle
async def _judge_once(
self,
@@ -251,36 +293,21 @@ class PaperKernelPipeline:
iteration: int,
judge_id: int,
) -> JudgeVerdict:
- format_feedback = ""
- for attempt in range(1, 4):
- request_id = (
- f"{item.item_id}.verify.t{iteration:02d}."
- f"j{judge_id}.a{attempt}"
- )
- verdict = JudgeVerdict.model_validate(
- await self._call(
- judge,
- request_id=request_id,
- system_prompt=JUDGE_SYSTEM,
- user_prompt=judge_user(
- item,
- dag,
- methods,
- bundle.replacement_plan,
- bundle.diffused_proof,
- bundle.variant,
- format_feedback=format_feedback,
- ),
- )
+ request_id = f"{item.item_id}.verify.t{iteration:02d}.j{judge_id}"
+ verdict = JudgeVerdict.model_validate(
+ await self._call(
+ judge,
+ request_id=request_id,
+ system_prompt=JUDGE_SYSTEM,
+ user_prompt=judge_user(
+ item,
+ methods,
+ bundle.replacement_plan,
+ bundle.variant,
+ ),
)
- try:
- return verdict.validate_coverage(dag, bundle.replacement_plan)
- except ValueError as exc:
- format_feedback = str(exc)
- raise ValueError(
- f"judge {judge_id} failed coverage after three format attempts: "
- f"{format_feedback}"
)
+ return verdict.validate_coverage(dag)
@staticmethod
def _feedback(verdicts: list[JudgeVerdict]) -> str:
@@ -369,6 +396,7 @@ class PaperKernelPipeline:
dag,
methods,
version=iteration + 1,
+ previous_bundle=bundle,
feedback=self._feedback(verdicts),
)
repaired_from_previous = True
diff --git a/src/gap_pipeline/pipeline.py b/src/gap_pipeline/pipeline.py
index 4a5332a..9d03525 100644
--- a/src/gap_pipeline/pipeline.py
+++ b/src/gap_pipeline/pipeline.py
@@ -1,4 +1,4 @@
-"""Historical two-call Putnam generator retained for provenance tests.
+"""Consolidated two-call Putnam generation interface.
The default manuscript-aligned implementation is ``paper_pipeline.py``.
"""
diff --git a/src/gap_pipeline/release.py b/src/gap_pipeline/release.py
index 7df6566..5bf8d95 100644
--- a/src/gap_pipeline/release.py
+++ b/src/gap_pipeline/release.py
@@ -89,21 +89,22 @@ def export_release(
continue
candidate = kernel_payload["accepted_candidate"]
+ method_nodes = kernel_payload["method_plan"]["nodes"]
+ replacement_changes = kernel_payload["accepted_replacement_plan"]["changes"]
variants["kernel_variant"] = {
"question": candidate["question"],
"solution": candidate["solution"],
"_meta": {
- "proof_dag": kernel_payload["proof_dag"],
- "method_plan": kernel_payload["method_plan"],
- "replacement_plan": kernel_payload["accepted_replacement_plan"],
- "diffused_proof": kernel_payload["accepted_diffused_proof"],
- "terminal_answer": candidate["terminal_answer"],
- "accepted_candidate_sha256": kernel_payload[
- "accepted_candidate_sha256"
- ],
- "accepted_bundle_sha256": kernel_payload[
- "accepted_bundle_sha256"
+ "core_steps": [
+ node["method_label"] for node in method_nodes
],
+ "mutable_slots": {
+ change["slot_id"]: {
+ "description": change["description"],
+ "original": change["original_value"],
+ }
+ for change in replacement_changes
+ },
},
}
output_record = copy.deepcopy(record)
diff --git a/tests/test_e2e.py b/tests/test_e2e.py
index 9672527..ebf23a4 100644
--- a/tests/test_e2e.py
+++ b/tests/test_e2e.py
@@ -29,6 +29,14 @@ def test_offline_end_to_end_smoke(tmp_path: Path) -> None:
"kernel_variant",
}
kernel_meta = record["variants"]["kernel_variant"]["_meta"]
- assert kernel_meta["proof_dag"]["terminal_node_id"] == "n3"
- assert kernel_meta["replacement_plan"]["changes"][0]["slot_id"] == "s1"
- assert kernel_meta["diffused_proof"]["terminal_answer"] == "4"
+ assert kernel_meta["core_steps"] == [
+ "use nonnegativity of a square",
+ "expand the square",
+ "divide by a positive quantity",
+ ]
+ assert kernel_meta["mutable_slots"] == {
+ "slot1": {
+ "description": "positive square root at equality",
+ "original": "1",
+ }
+ }
diff --git a/tests/test_kernel_models.py b/tests/test_kernel_models.py
index 864d6d5..0dc653d 100644
--- a/tests/test_kernel_models.py
+++ b/tests/test_kernel_models.py
@@ -40,7 +40,7 @@ def test_replacement_must_reference_a_real_dag_node() -> None:
{
"changes": [
{
- "slot_id": "s1",
+ "slot_id": "slot1",
"source_node_id": "n9",
"description": "constant",
"original_value": "1",
@@ -56,6 +56,70 @@ def test_replacement_must_reference_a_real_dag_node() -> None:
replacements.validate_against(branched_dag())
+def test_replacement_must_target_a_source_leaf() -> None:
+ replacements = ReplacementPlan.model_validate(
+ {
+ "changes": [
+ {
+ "slot_id": "slot1",
+ "source_node_id": "n2",
+ "description": "derived value",
+ "original_value": "1",
+ "replacement_value": "2",
+ "guard_condition": "positive",
+ "guard_justification": "2 is positive",
+ }
+ ],
+ "closure_statement": "No undeclared changes.",
+ }
+ )
+ with pytest.raises(ValueError, match="must target source leaf nodes.*n2"):
+ replacements.validate_against(branched_dag())
+
+
+def test_replacement_slot_id_matches_release_schema() -> None:
+ with pytest.raises(ValueError, match="must be slot1, slot2"):
+ ReplacementPlan.model_validate(
+ {
+ "changes": [
+ {
+ "slot_id": "s1",
+ "source_node_id": "n1",
+ "description": "constant",
+ "original_value": "1",
+ "replacement_value": "2",
+ "guard_condition": "positive",
+ "guard_justification": "2 is positive",
+ }
+ ],
+ "closure_statement": "No undeclared changes.",
+ }
+ )
+
+
+def test_replacement_repair_preserves_slot_and_leaf_identity() -> None:
+ previous = ReplacementPlan.model_validate(
+ {
+ "changes": [
+ {
+ "slot_id": "slot1",
+ "source_node_id": "n1",
+ "description": "constant",
+ "original_value": "1",
+ "replacement_value": "2",
+ "guard_condition": "positive",
+ "guard_justification": "2 is positive",
+ }
+ ],
+ "closure_statement": "No undeclared changes.",
+ }
+ )
+ changed_target = previous.model_copy(deep=True)
+ changed_target.changes[0].source_node_id = "n2"
+ with pytest.raises(ValueError, match="preserve replacement slot IDs"):
+ changed_target.validate_repair_of(previous)
+
+
def test_diffusion_preserves_dependencies_and_methods() -> None:
dag = branched_dag()
methods = MethodPlan.model_validate(
@@ -130,7 +194,6 @@ def test_accept_verdict_normalizes_explicit_none_sentinel() -> None:
verdict = JudgeVerdict(
verdict="accept",
step_by_step_check="n1 valid",
- replacement_check="s1 valid",
blocking_issues="None detected.",
patch_suggestion="N/A",
)
diff --git a/tests/test_paper_pipeline.py b/tests/test_paper_pipeline.py
new file mode 100644
index 0000000..7f1de46
--- /dev/null
+++ b/tests/test_paper_pipeline.py
@@ -0,0 +1,192 @@
+from __future__ import annotations
+
+import asyncio
+import json
+
+from gap_pipeline.clients import ScriptedClient
+from gap_pipeline.paper_pipeline import PaperKernelPipeline, PaperPipelineConfig
+from gap_pipeline.prompts import JUDGE_SYSTEM_PROMPT
+from gap_pipeline.store import RunStore
+
+
+def _dag() -> dict:
+ return {
+ "nodes": [
+ {
+ "node_id": "n1",
+ "claim": "(a-1)^2 >= 0",
+ "dependencies": [],
+ },
+ {
+ "node_id": "n2",
+ "claim": "a+1/a >= 2",
+ "dependencies": ["n1"],
+ },
+ ],
+ "terminal_node_id": "n2",
+ }
+
+
+def _methods() -> dict:
+ return {
+ "nodes": [
+ {
+ "node_id": "n1",
+ "method_label": "use nonnegativity of a square",
+ },
+ {
+ "node_id": "n2",
+ "method_label": "expand and divide by a positive quantity",
+ },
+ ]
+ }
+
+
+def _replacement() -> dict:
+ return {
+ "changes": [
+ {
+ "slot_id": "slot1",
+ "source_node_id": "n1",
+ "description": "positive equality value",
+ "original_value": "1",
+ "replacement_value": "2",
+ "guard_condition": "replacement is positive",
+ "guard_justification": "2 is positive",
+ }
+ ],
+ "closure_statement": "No undeclared changes.",
+ }
+
+
+def _diffused() -> dict:
+ return {
+ "nodes": [
+ {
+ "node_id": "n1",
+ "dependencies": [],
+ "method_label": "use nonnegativity of a square",
+ "instantiated_claim": "(x-2)^2 >= 0",
+ "justification": "squares are nonnegative",
+ },
+ {
+ "node_id": "n2",
+ "dependencies": ["n1"],
+ "method_label": "expand and divide by a positive quantity",
+ "instantiated_claim": "x+4/x >= 4",
+ "justification": "expand and divide by x>0",
+ },
+ ],
+ "terminal_node_id": "n2",
+ "terminal_answer": "4",
+ }
+
+
+def _variant(suffix: str = "") -> dict:
+ return {
+ "question": f"Let x>0. Prove that x+4/x >= 4.{suffix}",
+ "solution": (
+ "[n1] Since (x-2)^2 >= 0. "
+ "[n2] Expand and divide by x>0 to obtain x+4/x >= 4."
+ ),
+ "node_order": ["n1", "n2"],
+ "terminal_answer": "4",
+ }
+
+
+def _accept() -> dict:
+ return {
+ "verdict": "accept",
+ "step_by_step_check": "n1 is valid; n2 is valid",
+ "blocking_issues": "",
+ "patch_suggestion": "",
+ }
+
+
+def _reject() -> dict:
+ return {
+ "verdict": "reject",
+ "step_by_step_check": "n1 is valid; n2 needs a wording correction",
+ "blocking_issues": "the terminal wording is ambiguous",
+ "patch_suggestion": "clarify the terminal wording",
+ }
+
+
+def test_five_stage_repair_uses_prior_bundle_and_appendix_judge(
+ tmp_path,
+ item,
+) -> None:
+ proposer = ScriptedClient(
+ {
+ f"{item.item_id}.stage1.dag": _dag(),
+ f"{item.item_id}.stage2.methods": _methods(),
+ f"{item.item_id}.stage3.replacement": [
+ _replacement(),
+ _replacement(),
+ ],
+ f"{item.item_id}.stage4.diffusion": [
+ _diffused(),
+ _diffused(),
+ ],
+ f"{item.item_id}.stage5.render": [
+ _variant(),
+ _variant(" The requested bound is explicit."),
+ ],
+ }
+ )
+ judges = []
+ for judge_id in range(1, 6):
+ first = _reject() if judge_id == 1 else _accept()
+ judges.append(
+ ScriptedClient(
+ {f"{item.item_id}.verify": [first, _accept(), _accept()]}
+ )
+ )
+
+ run_root = tmp_path / "run"
+ result = asyncio.run(
+ PaperKernelPipeline(
+ proposer=proposer,
+ judges=judges,
+ store=RunStore(run_root, item.item_id),
+ config=PaperPipelineConfig(
+ proposer_model="scripted",
+ judge_model="scripted",
+ ),
+ ).run(item)
+ )
+
+ assert result.status == "accepted"
+ assert [row.pass_streak_after for row in result.iterations] == [0, 1, 2]
+ assert result.iterations[0].bundle_sha256 != result.iterations[1].bundle_sha256
+ assert result.iterations[1].bundle_sha256 == result.iterations[2].bundle_sha256
+
+ calls_dir = run_root / "items" / item.item_id / "calls"
+ repair_stage_calls = [
+ json.loads(
+ (
+ calls_dir
+ / f"{item.item_id}.stage{stage}.{name}.v02.json"
+ ).read_text()
+ )
+ for stage, name in [
+ (3, "replacement"),
+ (4, "diffusion"),
+ (5, "render"),
+ ]
+ ]
+ assert "PREVIOUS REPLACEMENT PLAN" in repair_stage_calls[0]["user_prompt"]
+ assert "PREVIOUS DIFFUSED PROOF" in repair_stage_calls[1]["user_prompt"]
+ assert "PREVIOUS RENDERED VARIANT" in repair_stage_calls[2]["user_prompt"]
+ assert all(
+ "the terminal wording is ambiguous" in call["user_prompt"]
+ for call in repair_stage_calls
+ )
+
+ judge_calls = sorted(calls_dir.glob(f"{item.item_id}.verify.*.json"))
+ assert len(judge_calls) == 15
+ assert not any(".a1." in path.name for path in judge_calls)
+ first_judge_call = json.loads(judge_calls[0].read_text())
+ assert first_judge_call["system_prompt"] == JUDGE_SYSTEM_PROMPT
+ assert "METHOD-LABEL SEQUENCE (abstract plan):" in first_judge_call["user_prompt"]
+ assert "SOURCE PROOF DAG:" not in first_judge_call["user_prompt"]
diff --git a/tests/test_prompts.py b/tests/test_prompts.py
index 4f6568b..9125d26 100644
--- a/tests/test_prompts.py
+++ b/tests/test_prompts.py
@@ -3,7 +3,7 @@ from __future__ import annotations
import hashlib
import inspect
-from gap_pipeline import prompts
+from gap_pipeline import kernel_prompts, prompts
from gap_pipeline.clients import OpenAIJsonClient
from gap_pipeline.kernel_models import (
DiffusedProof,
@@ -12,6 +12,7 @@ from gap_pipeline.kernel_models import (
ReplacementPlan,
)
from gap_pipeline.kernel_prompts import (
+ JUDGE_SYSTEM,
dag_user,
diffusion_user,
judge_user,
@@ -40,6 +41,19 @@ EXPECTED = {
"SURFACE_USER_TEMPLATE": "5d9043d7d1c1033db1aea0696812a2c4cbb6f3e28ca9d7b6436c036b40831fe7",
}
+EXPECTED_FIVE_STAGE = {
+ "DAG_SYSTEM": "86e596a0c3aea07100ba642bb07d22e25385c2a51007b8f7578b662502ae4650",
+ "DAG_USER": "28757954499cee438cb3676fbb8aff39b2bbb4644a1f9dac2604b71d24b03547",
+ "METHOD_SYSTEM": "1091d4db002673abf67f4407418d0d7e34f648878f907d21e1ee910318d447ce",
+ "METHOD_USER": "79901153a7e2ef16ca4ade9763e4fd05a6d9d248a76085998723389f9934dca6",
+ "REPLACEMENT_SYSTEM": "2bf314a8cdcf986b344d45af081f7a1f675da6be5687ae4cd594db2ed7c0b25b",
+ "REPLACEMENT_USER": "6e88657245d951e47893eb9bcd6dbc5cd7a826fc67b8b2ce849da5b559f19fd9",
+ "DIFFUSION_SYSTEM": "f6ce12280b592830f5bacca3c8bc8131da26c9c0f94ffb9bfb7e9f53347f63ac",
+ "DIFFUSION_USER": "e92b39f15d1af1acc12daf34fabc0dfaf63bcebb4c947821fadb121aec44c9c1",
+ "RENDER_SYSTEM": "ead495a3f7df7d9655dc24ac2a437d4358ec080cd5e4477b3e97641828c3233e",
+ "RENDER_USER": "9e3eabd0fecf6946552db3217efa2eea7938099bcb093c9ca02fabf5b7d8f70c",
+}
+
def test_prompt_values_are_byte_locked() -> None:
actual = {
@@ -49,6 +63,16 @@ def test_prompt_values_are_byte_locked() -> None:
assert actual == EXPECTED
+def test_five_stage_prompt_values_are_byte_locked() -> None:
+ actual = {
+ name: hashlib.sha256(
+ getattr(kernel_prompts, name).encode("utf-8")
+ ).hexdigest()
+ for name in EXPECTED_FIVE_STAGE
+ }
+ assert actual == EXPECTED_FIVE_STAGE
+
+
def test_o3_adapter_does_not_send_temperature() -> None:
source = inspect.getsource(OpenAIJsonClient.generate_json)
assert '"temperature"' not in source
@@ -68,7 +92,7 @@ def test_literal_five_stage_prompts_render(item) -> None:
{
"changes": [
{
- "slot_id": "s1",
+ "slot_id": "slot1",
"source_node_id": "n1",
"description": "constant",
"original_value": "1",
@@ -108,6 +132,15 @@ def test_literal_five_stage_prompts_render(item) -> None:
replacement_user(item, dag, methods),
diffusion_user(item, dag, methods, replacements),
render_user(replacements, diffused),
- judge_user(item, dag, methods, replacements, diffused, variant),
+ judge_user(item, methods, replacements, variant),
]
assert all("{" in value and "}" in value for value in rendered)
+ assert JUDGE_SYSTEM == prompts.JUDGE_SYSTEM_PROMPT
+ assert rendered[-1] == prompts.JUDGE_USER_TEMPLATE.format(
+ original_problem=item.problem,
+ original_solution=item.solution,
+ method_labels='["n1: method"]',
+ slot_replacement=replacements.model_dump_json(indent=2),
+ candidate_problem="question",
+ candidate_proof="[n1] solution",
+ )
diff --git a/tests/test_release.py b/tests/test_release.py
index 8927b14..e122340 100644
--- a/tests/test_release.py
+++ b/tests/test_release.py
@@ -65,7 +65,7 @@ def test_offline_release_export_verifies_and_assembles(
"accepted_replacement_plan": {
"changes": [
{
- "slot_id": "s1",
+ "slot_id": "slot1",
"source_node_id": "n1",
"description": "value",
"original_value": "1",
@@ -109,7 +109,16 @@ def test_offline_release_export_verifies_and_assembles(
)
assert set(output["variants"]) == {*SURFACE_FAMILIES, "kernel_variant"}
assert output["variants"]["kernel_variant"]["question"] == candidate["question"]
- assert output["variants"]["kernel_variant"]["_meta"]["replacement_plan"]
+ kernel_meta = output["variants"]["kernel_variant"]["_meta"]
+ assert kernel_meta == {
+ "core_steps": ["method"],
+ "mutable_slots": {
+ "slot1": {
+ "description": "value",
+ "original": "1",
+ }
+ },
+ }
with pytest.raises(FileExistsError):
export_release(
source_dataset=source_dir,
generated by cgit v1.2.3 (git 2.25.1) at 2026-09-08 11:50:05 +0000