diff options
| -rw-r--r-- | .gitignore | 4 | ||||
| -rw-r--r-- | GAP_End_to_End.ipynb | 361 | ||||
| -rw-r--r-- | README.md | 82 | ||||
| -rw-r--r-- | STAGE_MAP.md | 55 | ||||
| -rw-r--r-- | src/gap_pipeline/cli.py | 6 | ||||
| -rw-r--r-- | src/gap_pipeline/e2e.py | 104 | ||||
| -rw-r--r-- | src/gap_pipeline/kernel_models.py | 348 | ||||
| -rw-r--r-- | src/gap_pipeline/kernel_prompts.py | 276 | ||||
| -rw-r--r-- | src/gap_pipeline/models.py | 17 | ||||
| -rw-r--r-- | src/gap_pipeline/offline.py | 14 | ||||
| -rw-r--r-- | src/gap_pipeline/paper_pipeline.py | 395 | ||||
| -rw-r--r-- | src/gap_pipeline/pipeline.py | 5 | ||||
| -rw-r--r-- | src/gap_pipeline/release.py | 13 | ||||
| -rw-r--r-- | src/gap_pipeline/surface.py | 29 | ||||
| -rw-r--r-- | tests/test_e2e.py | 4 | ||||
| -rw-r--r-- | tests/test_kernel_models.py | 138 | ||||
| -rw-r--r-- | tests/test_models.py | 20 | ||||
| -rw-r--r-- | tests/test_offline.py | 68 | ||||
| -rw-r--r-- | tests/test_prompts.py | 73 | ||||
| -rw-r--r-- | tests/test_release.py | 47 | ||||
| -rw-r--r-- | tests/test_surface.py | 37 |
21 files changed, 1835 insertions, 261 deletions
@@ -5,6 +5,10 @@ __pycache__/ build/ dist/ *.egg-info/ +.env +.env.* +!.env.example +env runs/ surface-runs/ kernel-runs/ diff --git a/GAP_End_to_End.ipynb b/GAP_End_to_End.ipynb index bdb519f..2b8c10e 100644 --- a/GAP_End_to_End.ipynb +++ b/GAP_End_to_End.ipynb @@ -1,174 +1,193 @@ { - "cells": [ - { - "cell_type": "markdown", - "id": "gap-title", - "metadata": {}, - "source": [ - "# GAP end-to-end reproduction\n", - "\n", - "Run all cells to generate four surface variants and one verified kernel variant for a Putnam problem, then export a machine-readable GAP record.\n", - "\n", - "The live path uses `o3` for generation and five independent verification calls per round. At least two unanimous rounds are required, so a live run makes multiple API calls and may take several minutes. The API key is read with `getpass` and is never written into the notebook or run artifacts." - ] + "cells": [ + { + "cell_type": "markdown", + "id": "gap-title", + "metadata": {}, + "source": [ + "# GAP end-to-end reproduction\n", + "\n", + "Run all cells to generate four surface variants and one verified kernel variant for a Putnam problem, then export a machine-readable GAP record.\n", + "\n", + "The live kernel path executes the paper's five stages explicitly: concrete proof-DAG construction, content-free method abstraction, guarded replacement with recorded old/new values, node-by-node DAG diffusion, and answer-to-question rendering. It then uses five independent `o3` verification calls per round. At least two unanimous rounds on the unchanged full-provenance bundle are required, so a live run makes multiple API calls and may take several minutes.\n", + "\n", + "The API key is loaded from the process environment, a gitignored `.env` file, or a hidden `getpass` prompt. It is never written into the notebook or run artifacts." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "install-package", + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "import json\n", + "import os\n", + "import subprocess\n", + "import sys\n", + "from datetime import datetime, timezone\n", + "\n", + "candidates = [Path.cwd(), Path.cwd().parent]\n", + "PACKAGE_ROOT = next((p.resolve() for p in candidates if (p / \"pyproject.toml\").exists()), None)\n", + "if PACKAGE_ROOT is None:\n", + " raise RuntimeError(\"Open this notebook from the GAP package root or its parent directory.\")\n", + "\n", + "subprocess.run(\n", + " [sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"-e\", f\"{PACKAGE_ROOT}[api,test]\"],\n", + " check=True,\n", + ")\n", + "src_path = str(PACKAGE_ROOT / \"src\")\n", + "if src_path not in sys.path:\n", + " sys.path.insert(0, src_path)\n", + "subprocess.run(\n", + " [sys.executable, \"-m\", \"pytest\", \"-q\", str(PACKAGE_ROOT / \"tests\" / \"test_prompts.py\")],\n", + " cwd=PACKAGE_ROOT,\n", + " check=True,\n", + ")\n", + "print(f\"GAP package root: {PACKAGE_ROOT}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "configure-run", + "metadata": {}, + "outputs": [], + "source": [ + "# Configuration. Set GAP_OFFLINE_SMOKE=1 to exercise the full pipeline without an API call.\n", + "OFFLINE_SMOKE = os.environ.get(\"GAP_OFFLINE_SMOKE\", \"0\") == \"1\"\n", + "MODEL = os.environ.get(\"GAP_MODEL\", \"o3\")\n", + "ITEM_ID = os.environ.get(\"GAP_ITEM_ID\", \"1998-B-1\")\n", + "\n", + "dataset_candidates = [\n", + " Path(os.environ[\"PUTNAMGAP_DATASET\"]).expanduser() if os.environ.get(\"PUTNAMGAP_DATASET\") else None,\n", + " PACKAGE_ROOT / \"examples\" / \"sample_data\",\n", + " PACKAGE_ROOT.parent / \"putnamsup\" / \"PutnamGAP\",\n", + "]\n", + "DATASET_DIR = next((p.resolve() for p in dataset_candidates if p is not None and p.exists()), None)\n", + "if not OFFLINE_SMOKE and DATASET_DIR is None:\n", + " raise RuntimeError(\"Set PUTNAMGAP_DATASET to the directory containing the PutnamGAP JSON files.\")\n", + "\n", + "stamp = datetime.now(timezone.utc).strftime(\"%Y%m%dT%H%M%SZ\")\n", + "OUTPUT_BASE = Path(os.environ.get(\"GAP_NOTEBOOK_OUTPUT_BASE\", PACKAGE_ROOT / \"notebook_runs\")).expanduser()\n", + "WORK_ROOT = OUTPUT_BASE / f\"{ITEM_ID}-{stamp}\"\n", + "print({\"offline_smoke\": OFFLINE_SMOKE, \"model\": MODEL, \"item_id\": ITEM_ID, \"dataset\": str(DATASET_DIR), \"work_root\": str(WORK_ROOT)})" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "load-api-key", + "metadata": {}, + "outputs": [], + "source": [ + "api_key = None\n", + "if not OFFLINE_SMOKE:\n", + " api_key = os.environ.get(\"OPENAI_API_KEY\")\n", + " if not api_key:\n", + " env_path = PACKAGE_ROOT / \".env\"\n", + " if env_path.exists():\n", + " for raw_line in env_path.read_text(encoding=\"utf-8\").splitlines():\n", + " line = raw_line.strip()\n", + " if line and not line.startswith(\"#\") and line.startswith(\"OPENAI_API_KEY=\"):\n", + " api_key = line.split(\"=\", 1)[1].strip().strip('\"').strip(\"'\")\n", + " break\n", + " if not api_key:\n", + " from getpass import getpass\n", + " api_key = getpass(\"OPENAI_API_KEY (input hidden): \").strip()\n", + " if not api_key:\n", + " raise RuntimeError(\"An OpenAI API key is required for the live run.\")\n", + " print(\"API key loaded in memory; it will not be displayed or saved.\")\n", + "else:\n", + " print(\"Offline scripted smoke mode: no API key required.\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "run-pipeline", + "metadata": {}, + "outputs": [], + "source": [ + "from gap_pipeline.e2e import run_live_item, run_offline_smoke\n", + "\n", + "if OFFLINE_SMOKE:\n", + " result = await run_offline_smoke(WORK_ROOT)\n", + "else:\n", + " result = await run_live_item(\n", + " dataset_dir=DATASET_DIR,\n", + " item_id=ITEM_ID,\n", + " work_root=WORK_ROOT,\n", + " model=MODEL,\n", + " api_key=api_key,\n", + " )\n", + "\n", + "print(json.dumps(result, indent=2, ensure_ascii=False))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "verify-output", + "metadata": {}, + "outputs": [], + "source": [ + "# End-to-end assertions and a compact preview of the exported record.\n", + "record = json.loads(Path(result[\"release_record\"]).read_text(encoding=\"utf-8\"))\n", + "expected_families = {\n", + " \"descriptive_long\",\n", + " \"descriptive_long_confusing\",\n", + " \"descriptive_long_misleading\",\n", + " \"garbled_string\",\n", + " \"kernel_variant\",\n", + "}\n", + "assert set(record[\"variants\"]) == expected_families\n", + "assert result[\"kernel_status\"] == \"accepted\"\n", + "assert result[\"verification_rounds\"] >= 2\n", + "assert result[\"export_status\"] == \"complete\"\n", + "\n", + "kernel = record[\"variants\"][\"kernel_variant\"]\n", + "meta = kernel[\"_meta\"]\n", + "assert meta[\"proof_dag\"][\"nodes\"]\n", + "assert meta[\"method_plan\"][\"nodes\"]\n", + "assert meta[\"replacement_plan\"][\"changes\"]\n", + "assert meta[\"diffused_proof\"][\"nodes\"]\n", + "assert meta[\"accepted_bundle_sha256\"]\n", + "\n", + "print(\"END-TO-END PASS\")\n", + "print(f\"Item: {record['index']}\")\n", + "print(f\"Variant families: {sorted(record['variants'])}\")\n", + "print(f\"Verification rounds: {result['verification_rounds']}\")\n", + "print(f\"Declared replacements: {len(meta['replacement_plan']['changes'])}\")\n", + "print(f\"Proof-DAG nodes: {len(meta['proof_dag']['nodes'])}\")\n", + "print(f\"Kernel question preview: {kernel['question'][:500]}\")\n", + "print(f\"Exported record: {result['release_record']}\")\n", + "print(f\"Manifest: {result['manifest']}\")" + ] + }, + { + "cell_type": "markdown", + "id": "full-dataset-note", + "metadata": {}, + "source": [ + "## Run another released item\n", + "\n", + "The notebook intentionally runs one item so the complete path can be checked at modest cost. Set `PUTNAMGAP_DATASET` and `GAP_ITEM_ID` before running to choose another source item; repeat the same runner over IDs for a larger generation job." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.11" + } }, - { - "cell_type": "code", - "id": "install-package", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from pathlib import Path\n", - "import json\n", - "import os\n", - "import subprocess\n", - "import sys\n", - "from datetime import datetime, timezone\n", - "\n", - "candidates = [Path.cwd(), Path.cwd().parent]\n", - "PACKAGE_ROOT = next((p.resolve() for p in candidates if (p / \"pyproject.toml\").exists()), None)\n", - "if PACKAGE_ROOT is None:\n", - " raise RuntimeError(\"Open this notebook from the GAP package root or its parent directory.\")\n", - "\n", - "subprocess.run(\n", - " [sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"-e\", f\"{PACKAGE_ROOT}[api,test]\"],\n", - " check=True,\n", - ")\n", - "src_path = str(PACKAGE_ROOT / \"src\")\n", - "if src_path not in sys.path:\n", - " sys.path.insert(0, src_path)\n", - "subprocess.run(\n", - " [sys.executable, \"-m\", \"pytest\", \"-q\", str(PACKAGE_ROOT / \"tests\" / \"test_prompts.py\")],\n", - " cwd=PACKAGE_ROOT,\n", - " check=True,\n", - ")\n", - "print(f\"GAP package root: {PACKAGE_ROOT}\")" - ] - }, - { - "cell_type": "code", - "id": "configure-run", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Configuration. Set GAP_OFFLINE_SMOKE=1 to exercise the full pipeline without an API call.\n", - "OFFLINE_SMOKE = os.environ.get(\"GAP_OFFLINE_SMOKE\", \"0\") == \"1\"\n", - "MODEL = os.environ.get(\"GAP_MODEL\", \"o3\")\n", - "ITEM_ID = os.environ.get(\"GAP_ITEM_ID\", \"1998-B-1\")\n", - "\n", - "dataset_candidates = [\n", - " Path(os.environ[\"PUTNAMGAP_DATASET\"]).expanduser() if os.environ.get(\"PUTNAMGAP_DATASET\") else None,\n", - " PACKAGE_ROOT / \"examples\" / \"sample_data\",\n", - " PACKAGE_ROOT.parent / \"putnamsup\" / \"PutnamGAP\",\n", - "]\n", - "DATASET_DIR = next((p.resolve() for p in dataset_candidates if p is not None and p.exists()), None)\n", - "if not OFFLINE_SMOKE and DATASET_DIR is None:\n", - " raise RuntimeError(\"Set PUTNAMGAP_DATASET to the directory containing the PutnamGAP JSON files.\")\n", - "\n", - "stamp = datetime.now(timezone.utc).strftime(\"%Y%m%dT%H%M%SZ\")\n", - "OUTPUT_BASE = Path(os.environ.get(\"GAP_NOTEBOOK_OUTPUT_BASE\", PACKAGE_ROOT / \"notebook_runs\")).expanduser()\n", - "WORK_ROOT = OUTPUT_BASE / f\"{ITEM_ID}-{stamp}\"\n", - "print({\"offline_smoke\": OFFLINE_SMOKE, \"model\": MODEL, \"item_id\": ITEM_ID, \"dataset\": str(DATASET_DIR), \"work_root\": str(WORK_ROOT)})" - ] - }, - { - "cell_type": "code", - "id": "load-api-key", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "api_key = None\n", - "if not OFFLINE_SMOKE:\n", - " api_key = os.environ.get(\"OPENAI_API_KEY\")\n", - " if not api_key:\n", - " from getpass import getpass\n", - " api_key = getpass(\"OPENAI_API_KEY (input hidden): \").strip()\n", - " if not api_key:\n", - " raise RuntimeError(\"An OpenAI API key is required for the live run.\")\n", - " print(\"API key loaded in memory; it will not be displayed or saved.\")\n", - "else:\n", - " print(\"Offline scripted smoke mode: no API key required.\")" - ] - }, - { - "cell_type": "code", - "id": "run-pipeline", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from gap_pipeline.e2e import run_live_item, run_offline_smoke\n", - "\n", - "if OFFLINE_SMOKE:\n", - " result = await run_offline_smoke(WORK_ROOT)\n", - "else:\n", - " result = await run_live_item(\n", - " dataset_dir=DATASET_DIR,\n", - " item_id=ITEM_ID,\n", - " work_root=WORK_ROOT,\n", - " model=MODEL,\n", - " api_key=api_key,\n", - " )\n", - "\n", - "print(json.dumps(result, indent=2, ensure_ascii=False))" - ] - }, - { - "cell_type": "code", - "id": "verify-output", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# End-to-end assertions and a compact preview of the exported record.\n", - "record = json.loads(Path(result[\"release_record\"]).read_text(encoding=\"utf-8\"))\n", - "expected_families = {\n", - " \"descriptive_long\",\n", - " \"descriptive_long_confusing\",\n", - " \"descriptive_long_misleading\",\n", - " \"garbled_string\",\n", - " \"kernel_variant\",\n", - "}\n", - "assert set(record[\"variants\"]) == expected_families\n", - "assert result[\"kernel_status\"] == \"accepted\"\n", - "assert result[\"verification_rounds\"] >= 2\n", - "assert result[\"export_status\"] == \"complete\"\n", - "\n", - "kernel = record[\"variants\"][\"kernel_variant\"]\n", - "print(\"END-TO-END PASS\")\n", - "print(f\"Item: {record['index']}\")\n", - "print(f\"Variant families: {sorted(record['variants'])}\")\n", - "print(f\"Verification rounds: {result['verification_rounds']}\")\n", - "print(f\"Kernel question preview: {kernel['question'][:500]}\")\n", - "print(f\"Exported record: {result['release_record']}\")\n", - "print(f\"Manifest: {result['manifest']}\")" - ] - }, - { - "cell_type": "markdown", - "id": "full-dataset-note", - "metadata": {}, - "source": [ - "## Run another released item\n", - "\n", - "The notebook intentionally runs one item so the complete path can be checked at modest cost. Set `PUTNAMGAP_DATASET` and `GAP_ITEM_ID` before running to choose another source item; repeat the same runner over IDs for a larger generation job." - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python", - "version": "3.11" - } - }, - "nbformat": 4, - "nbformat_minor": 5 + "nbformat": 4, + "nbformat_minor": 5 } @@ -1,7 +1,9 @@ # GAP minimal reproduction package -This repository provides a one-item end-to-end reproduction of GAP: four -surface-renaming families and one verified kernel variant. +This repository provides a one-item end-to-end reference implementation of +GAP: four surface-renaming families and one verified kernel variant. The +default kernel path executes the manuscript's five conceptual operations as +five typed, separately saved model calls. ## One-click reproduction @@ -10,14 +12,17 @@ Open `GAP_End_to_End.ipynb` and choose **Run All**. The notebook: 1. installs the package; 2. loads one canonical Putnam problem; 3. generates the four surface variants; -4. extracts the kernel proof plan and mutable slots; -5. generates a new question and complete solution with the same plan; -6. runs five judges until the unchanged candidate receives two consecutive +4. constructs a concrete proof DAG whose nodes are intermediate claims; +5. abstracts one content-free method label per DAG node; +6. records guarded old/new replacements and propagates them node by node; +7. renders the diffused terminal claim into a new question and solution; +8. runs five judges until the unchanged full-provenance bundle receives two consecutive unanimous rounds; -7. exports and validates one machine-readable GAP record. +9. exports and validates one machine-readable GAP record. -The default model is `o3`. Set `OPENAI_API_KEY` before starting Jupyter, or -enter it in the notebook's hidden prompt. The key is never saved. +The default model is `o3`. Set `OPENAI_API_KEY`, place +`OPENAI_API_KEY=...` in the gitignored `.env` file, or enter it in the +notebook's hidden prompt. The key is never written to run artifacts. For a no-API software check: @@ -31,25 +36,28 @@ mathematics. ## Prompt fidelity -Generation, surface-renaming, judge, and repair prompts are copied verbatim -from the original author source and the prompt listing in the paper. Their -UTF-8 SHA-256 digests are pinned in `PROMPT_SHA256SUMS` and enforced by -`tests/test_prompts.py`. +The historical two-call Prompt-A/Prompt-B, surface-renaming prompts, and +review/repair prompts recovered from `PutnamVariants@c3bed737` remain byte +pinned in `src/gap_pipeline/prompts.py`, `PROMPT_SHA256SUMS`, and +`tests/test_prompts.py`. They document the actual original generator. -The OpenAI adapter does not send `temperature`; this is compatible with `o3`, -whose supported value is its default. +The executable manuscript-aligned prompts are in +`src/gap_pipeline/kernel_prompts.py`. They refine the historical intent into +five explicit contracts because the recovered original generator only made +two calls and did not emit a concrete DAG, applied replacement map, or +node-by-node diffusion trace. This distinction is deliberate and auditable, +not hidden as prompt identity. -The conceptual five stages are represented explicitly in saved artifacts. -The original implementation batches stages 1–3 into Prompt-A and stages 4–5 -into Prompt-B; the wrapper does not change those prompts. See `STAGE_MAP.md`. +The OpenAI adapter does not send `temperature`; this is compatible with `o3`, +whose supported value is its default. See `STAGE_MAP.md` for the exact +paper-to-code and historical-source maps. -Prompt-A's ordered core steps instantiate a path-structured proof-plan DAG: -each step is a typed node and each edge records the dependency on the preceding -step. `ProofPlanDAG` validates unique node IDs, known dependencies, acyclicity, -connectivity to the terminal node, and the extracted order. This path structure -is the precise graph induced by an ordered minimal proof chain. Judges receive -the same method-label sequence with stable node IDs and must report a check for -every node before their verdict is counted. +`ProofDAG` supports branching dependencies and validates topological order, +known dependencies, acyclicity, and terminal connectivity. Replacement plans +record the exact old/new value, source node, guard condition, and guard +justification. Diffusion must preserve every node ID, dependency, and method +label. Judges must cover every proof-node ID and replacement-slot ID before +their verdict counts. ## Install and test @@ -65,6 +73,14 @@ PYTHONPATH=src pytest PYTHONPATH=src python -m gap_pipeline.cli --help ``` +PowerShell equivalents: + +```powershell +$env:PYTHONPATH = "src" +python -m pytest +python -m gap_pipeline.cli --help +``` + ## Live one-item commands Generate the kernel variant: @@ -79,6 +95,9 @@ PYTHONPATH=src python -m gap_pipeline.cli generate-kernel \ --judge-model o3 ``` +In PowerShell, use `$env:OPENAI_API_KEY = "..."` and backticks for line +continuation, or use the gitignored `.env` file through the notebook. + Generate all surface variants: ```bash @@ -103,7 +122,14 @@ PYTHONPATH=src python -m gap_pipeline.cli export-release \ ## Verification protocol Kernel verification uses `J=5` judges, requires `K=2` consecutive unanimous -rounds for the same candidate, and allows at most `T=15` rounds. A rejected -round resets the streak and triggers a complete question-and-solution repair. -Every call, stage output, iteration, and final record is saved under the chosen -run directory. +rounds for the same complete provenance bundle, and allows at most `T=15` +rounds. A rejected round resets the streak and reruns stages 3--5 from a new +guarded replacement plan using the judge feedback. Every call, stage output, +iteration, and final record is saved under the chosen run directory. + +## Scope + +This package demonstrates and tests the one-item software path. It does not +reconstruct unavailable proposal/rejection logs from the original 1,051-item +generation run, and a successful LLM verification loop is not a substitute for +the separate blinded mathematical audit described in the rebuttal plan. diff --git a/STAGE_MAP.md b/STAGE_MAP.md index ee77f96..9e05842 100644 --- a/STAGE_MAP.md +++ b/STAGE_MAP.md @@ -1,28 +1,38 @@ -# GAP paper-to-code map +# GAP paper-to-code and provenance map -The original implementation batches adjacent conceptual stages into two model -calls. Prompt-A returns the ordered proof-plan nodes and mutable slots; Prompt-B -re-instantiates that plan and returns the regenerated proof and question. -The package writes each conceptual output separately so every paper stage is -visible in an end-to-end run without changing either prompt. +The recovered original Putnam generator at `PutnamVariants@c3bed737` makes two +model calls: Prompt-A returns 1--5 `core_steps` plus mutable-slot descriptions, +and Prompt-B directly returns a complete question and solution. Those exact +historical prompts remain byte pinned in `prompts.py`. -The ordered nodes form a path-structured DAG, the exact graph induced by a -minimal sequential proof plan. `ProofPlanDAG` validates node identity, -dependencies, acyclicity, terminal connectivity, and order. The five judges -receive stable node IDs with the method-label sequence, and a verdict is valid -only when its step-by-step check covers every node. +The manuscript describes a richer five-stage procedure. The default executable +path in `paper_pipeline.py` implements those operations explicitly, using the +paper-aligned prompts in `kernel_prompts.py`. It does not claim these new +prompts are byte-identical to the historical two-call generator. | Paper operation | Implementation | Saved artifact | |---|---|---| | Surface rename | `SurfacePipeline.run_family` | `surface_<family>_variant.json` | -| 1. Reference solution to proof structure | `KernelPipeline.extract_plan` + `ProofPlanDAG` validation | `01_proof_dag.json` | -| 2. Content-free method plan | `KernelPipeline.extract_plan` | `02_method_plan.json` | -| 3. Mutable-slot identification | `KernelPipeline.extract_plan` | `03_mutable_slots.json` | -| 4. Proof regeneration | `KernelPipeline.generate_candidate` | `04_regenerated_proof.json` | -| 5. Problem rendering | `KernelPipeline.generate_candidate` | `05_variant_question.json` | -| Five-judge verification | `KernelPipeline.verify_candidate` | five call records and one iteration record per round | -| Consecutive-pass protocol | `KernelPipeline.verify_candidate` | `K=2`; any rejection resets the streak | -| Repair loop | `KernelPipeline._repair` | complete corrected question and solution; at most `T=15` rounds | +| 1. Reference solution to concrete proof DAG | `PaperKernelPipeline.construct_dag` + `ProofDAG` validation | `01_proof_dag.json` | +| 2. Content-free method plan | `PaperKernelPipeline.summarize_methods` | `02_method_plan.json` | +| 3. Guarded replacement generation | `PaperKernelPipeline.generate_replacements` | `03_replacement_vNN.json` | +| 4. Node-by-node DAG diffusion | `PaperKernelPipeline.diffuse_dag` | `04_diffused_proof_vNN.json` | +| 5. Answer-to-question rendering | `PaperKernelPipeline.render_variant` | `05_rendered_variant_vNN.json` | +| Five-judge verification | `PaperKernelPipeline.verify` | five call records and one iteration record per round | +| Consecutive-pass protocol | `PaperKernelPipeline.verify` | `K=2` on the unchanged bundle hash | +| Repair loop | `PaperKernelPipeline.build_bundle` | rerun stages 3--5 from judge feedback; at most `T=15` rounds | + +## Enforced contracts + +- The concrete DAG may branch; every dependency must reference an earlier node, + and every node must contribute to the terminal node. +- The method plan contains exactly one method label per DAG node. +- Every replacement records its source node, exact old/new values, mathematical + guard, and guard justification. +- The diffused proof must preserve node IDs, dependencies, and method labels. +- The rendered solution must cite every node in order and retain the diffused + terminal answer. +- Every judge must discuss every node ID and every replacement slot ID. ## Per-item artifacts @@ -36,6 +46,7 @@ items/<item-id>/ final.json ``` -Prompt literals are byte-locked by `PROMPT_SHA256SUMS` and -`tests/test_prompts.py`. The OpenAI adapter does not pass a temperature -argument; `o3` therefore uses its supported default. +Historical prompt literals are byte-locked by `PROMPT_SHA256SUMS` and +`tests/test_prompts.py`. The new five-stage prompts are versioned source code +and covered by schema and end-to-end tests. The OpenAI adapter does not pass a +temperature argument; `o3` therefore uses its supported default. diff --git a/src/gap_pipeline/cli.py b/src/gap_pipeline/cli.py index 105cc97..e79e3e1 100644 --- a/src/gap_pipeline/cli.py +++ b/src/gap_pipeline/cli.py @@ -16,7 +16,7 @@ from .offline import ( summarize_run, validate_public_dataset, ) -from .pipeline import KernelPipeline, PipelineConfig +from .paper_pipeline import PaperKernelPipeline, PaperPipelineConfig from .release import export_release from .store import RunStore from .surface import SurfacePipeline @@ -36,7 +36,7 @@ async def generate_kernel(args: argparse.Namespace) -> None: if args.item_id not in records: raise SystemExit(f"item ID {args.item_id!r} not found in {args.dataset}") item = CanonicalItem.from_public_record(records[args.item_id]) - config = PipelineConfig( + config = PaperPipelineConfig( proposer_model=args.proposer_model, judge_model=args.judge_model, ) @@ -48,7 +48,7 @@ async def generate_kernel(args: argparse.Namespace) -> None: ) for judge_id in range(1, 6) ] - pipeline = KernelPipeline( + pipeline = PaperKernelPipeline( proposer=proposer, judges=judges, store=RunStore(args.run_dir, item.item_id), diff --git a/src/gap_pipeline/e2e.py b/src/gap_pipeline/e2e.py index ded1381..66101e7 100644 --- a/src/gap_pipeline/e2e.py +++ b/src/gap_pipeline/e2e.py @@ -11,7 +11,7 @@ from typing import Any from .clients import OpenAIJsonClient, ScriptedClient from .models import CanonicalItem from .offline import load_dataset -from .pipeline import KernelPipeline, PipelineConfig +from .paper_pipeline import PaperKernelPipeline, PaperPipelineConfig from .release import export_release from .store import RunStore from .surface import SurfacePipeline @@ -25,7 +25,8 @@ def _ensure_fresh(path: Path) -> None: def _review_accept() -> dict[str, str]: return { "verdict": "accept", - "step_by_step_check": "n1 passes; n2 passes", + "step_by_step_check": "n1 passes; n2 passes; n3 passes", + "replacement_check": "s1 satisfies its positivity guard", "blocking_issues": "", "patch_suggestion": "", } @@ -84,8 +85,8 @@ async def run_live_item( surface_store, ).run_all(item) - config = PipelineConfig(proposer_model=model, judge_model=model) - kernel = await KernelPipeline( + config = PaperPipelineConfig(proposer_model=model, judge_model=model) + kernel = await PaperKernelPipeline( proposer=OpenAIJsonClient(model, api_key=api_key), judges=[OpenAIJsonClient(model, api_key=api_key) for _ in range(5)], store=RunStore(kernel_root, item.item_id), @@ -164,24 +165,91 @@ async def run_offline_smoke(work_root: Path) -> dict[str, Any]: proposer = ScriptedClient( { - f"{item_id}.plan": { - "core_steps": [ - "use nonnegativity of a square", - "expand and divide by a positive quantity", + f"{item_id}.stage1.dag": { + "nodes": [ + { + "node_id": "n1", + "claim": "(a-1)^2 >= 0", + "dependencies": [], + }, + { + "node_id": "n2", + "claim": "a^2-2a+1 >= 0", + "dependencies": ["n1"], + }, + { + "node_id": "n3", + "claim": "a+1/a >= 2", + "dependencies": ["n2"], + }, ], - "mutable_slots": { - "slot1": { - "description": "the positive reference value", - "original": "1", + "terminal_node_id": "n3", + }, + f"{item_id}.stage2.methods": { + "nodes": [ + { + "node_id": "n1", + "method_label": "use nonnegativity of a square", + }, + { + "node_id": "n2", + "method_label": "expand the square", + }, + { + "node_id": "n3", + "method_label": "divide by a positive quantity", + }, + ] + }, + f"{item_id}.stage3.replacement": { + "changes": [ + { + "slot_id": "s1", + "source_node_id": "n1", + "description": "positive square root at equality", + "original_value": "1", + "replacement_value": "2", + "guard_condition": "replacement is positive", + "guard_justification": "2 is positive", } - }, + ], + "closure_statement": "The equality value is the only change.", + }, + f"{item_id}.stage4.diffusion": { + "nodes": [ + { + "node_id": "n1", + "dependencies": [], + "method_label": "use nonnegativity of a square", + "instantiated_claim": "(x-2)^2 >= 0", + "justification": "squares are nonnegative", + }, + { + "node_id": "n2", + "dependencies": ["n1"], + "method_label": "expand the square", + "instantiated_claim": "x^2-4x+4 >= 0", + "justification": "expand n1", + }, + { + "node_id": "n3", + "dependencies": ["n2"], + "method_label": "divide by a positive quantity", + "instantiated_claim": "x+4/x >= 4", + "justification": "divide n2 by x>0", + }, + ], + "terminal_node_id": "n3", + "terminal_answer": "4", }, - f"{item_id}.candidate": { + f"{item_id}.stage5.render": { "question": "Let x>0. Prove that x+4/x >= 4.", "solution": ( - "Since (x-2)^2 >= 0, expansion and division by x>0 " - "give x+4/x >= 4." + "[n1] Since (x-2)^2 >= 0. [n2] Expanding gives " + "x^2-4x+4 >= 0. [n3] Divide by x>0 to get x+4/x >= 4." ), + "node_order": ["n1", "n2", "n3"], + "terminal_answer": "4", }, } ) @@ -191,11 +259,11 @@ async def run_offline_smoke(work_root: Path) -> dict[str, Any]: ) for _ in range(5) ] - kernel = await KernelPipeline( + kernel = await PaperKernelPipeline( proposer=proposer, judges=judges, store=RunStore(kernel_root, item_id), - config=PipelineConfig( + config=PaperPipelineConfig( proposer_model="scripted", judge_model="scripted", ), diff --git a/src/gap_pipeline/kernel_models.py b/src/gap_pipeline/kernel_models.py new file mode 100644 index 0000000..32abf39 --- /dev/null +++ b/src/gap_pipeline/kernel_models.py @@ -0,0 +1,348 @@ +"""Typed contracts for the literal five-stage GAP kernel pipeline.""" + +from __future__ import annotations + +import json +import re +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + +from .models import SCHEMA_VERSION + + +class StrictModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + +class ProofNode(StrictModel): + node_id: str + claim: str + dependencies: list[str] = Field(default_factory=list) + + @model_validator(mode="after") + def validate_content(self) -> "ProofNode": + if not re.fullmatch(r"n[1-9][0-9]*", self.node_id): + raise ValueError("proof node IDs must be n1, n2, ...") + if not self.claim.strip(): + raise ValueError("proof-node claim must be non-empty") + if len(self.dependencies) != len(set(self.dependencies)): + raise ValueError(f"{self.node_id} has duplicate dependencies") + return self + + +class ProofDAG(StrictModel): + nodes: list[ProofNode] = Field(min_length=1) + terminal_node_id: str + + def node_ids(self) -> list[str]: + return [node.node_id for node in self.nodes] + + @model_validator(mode="after") + def validate_graph(self) -> "ProofDAG": + node_ids = self.node_ids() + expected = [f"n{index}" for index in range(1, len(node_ids) + 1)] + if node_ids != expected: + raise ValueError("proof DAG nodes must be topologically ordered n1 through nN") + known: set[str] = set() + for node in self.nodes: + if node.node_id in node.dependencies: + raise ValueError(f"{node.node_id} cannot depend on itself") + unknown = set(node.dependencies) - known + if unknown: + raise ValueError( + f"{node.node_id} has non-prior dependencies {sorted(unknown)}" + ) + known.add(node.node_id) + if self.terminal_node_id not in known: + raise ValueError("terminal node is not present in the DAG") + + by_id = {node.node_id: node for node in self.nodes} + ancestors: set[str] = set() + pending = [self.terminal_node_id] + while pending: + node_id = pending.pop() + if node_id in ancestors: + continue + ancestors.add(node_id) + pending.extend(by_id[node_id].dependencies) + if ancestors != known: + raise ValueError("every proof node must contribute to the terminal node") + return self + + +class MethodNode(StrictModel): + node_id: str + method_label: str + + @model_validator(mode="after") + def validate_content(self) -> "MethodNode": + if not self.method_label.strip(): + raise ValueError("method label must be non-empty") + return self + + +class MethodPlan(StrictModel): + nodes: list[MethodNode] = Field(min_length=1) + + def validate_against(self, dag: ProofDAG) -> "MethodPlan": + if [node.node_id for node in self.nodes] != dag.node_ids(): + raise ValueError("method-plan IDs must exactly match proof-DAG IDs") + return self + + +class ReplacementChange(StrictModel): + slot_id: str + source_node_id: str + description: str + original_value: str + replacement_value: str + guard_condition: str + guard_justification: str + + @model_validator(mode="after") + def validate_change(self) -> "ReplacementChange": + text_fields = [ + self.slot_id, + self.source_node_id, + self.description, + self.original_value, + self.replacement_value, + self.guard_condition, + self.guard_justification, + ] + if any(not value.strip() for value in text_fields): + raise ValueError("replacement fields must be non-empty") + if self.original_value.strip() == self.replacement_value.strip(): + raise ValueError("replacement must differ from the original value") + return self + + +class ReplacementPlan(StrictModel): + changes: list[ReplacementChange] = Field(min_length=1) + closure_statement: str + + @model_validator(mode="after") + def validate_content(self) -> "ReplacementPlan": + slot_ids = [change.slot_id for change in self.changes] + if len(slot_ids) != len(set(slot_ids)): + raise ValueError("replacement slot IDs must be unique") + if not self.closure_statement.strip(): + raise ValueError("replacement plan must state its closure guarantee") + return self + + def validate_against(self, dag: ProofDAG) -> "ReplacementPlan": + known = set(dag.node_ids()) + unknown = { + change.source_node_id + for change in self.changes + if change.source_node_id not in known + } + if unknown: + raise ValueError(f"replacement plan references unknown nodes {sorted(unknown)}") + return self + + +class DiffusedProofNode(StrictModel): + node_id: str + dependencies: list[str] = Field(default_factory=list) + method_label: str + instantiated_claim: str + justification: str + + @model_validator(mode="after") + def validate_content(self) -> "DiffusedProofNode": + if not self.method_label.strip(): + raise ValueError("diffused method label must be non-empty") + if not self.instantiated_claim.strip() or not self.justification.strip(): + raise ValueError("diffused claim and justification must be non-empty") + return self + + +class DiffusedProof(StrictModel): + nodes: list[DiffusedProofNode] = Field(min_length=1) + terminal_node_id: str + terminal_answer: str + + def validate_against( + self, + dag: ProofDAG, + method_plan: MethodPlan, + ) -> "DiffusedProof": + method_plan.validate_against(dag) + if [node.node_id for node in self.nodes] != dag.node_ids(): + raise ValueError("diffused proof must contain exactly one row per DAG node") + dag_by_id = {node.node_id: node for node in dag.nodes} + methods = {node.node_id: node.method_label for node in method_plan.nodes} + for node in self.nodes: + if node.dependencies != dag_by_id[node.node_id].dependencies: + raise ValueError( + f"{node.node_id} dependencies changed during DAG diffusion" + ) + if node.method_label != methods[node.node_id]: + raise ValueError( + f"{node.node_id} method label changed during DAG diffusion" + ) + if self.terminal_node_id != dag.terminal_node_id: + raise ValueError("diffused terminal node must match the source DAG") + if not self.terminal_answer.strip(): + raise ValueError("diffused proof must expose a terminal answer") + return self + + +class RenderedVariant(StrictModel): + question: str + solution: str + node_order: list[str] = Field(min_length=1) + terminal_answer: str + + @model_validator(mode="after") + def validate_content(self) -> "RenderedVariant": + if not self.question.strip() or not self.solution.strip(): + raise ValueError("rendered question and solution must be non-empty") + if not self.terminal_answer.strip(): + raise ValueError("rendered terminal answer must be non-empty") + return self + + def validate_against( + self, + dag: ProofDAG, + diffused_proof: DiffusedProof, + ) -> "RenderedVariant": + if self.node_order != dag.node_ids(): + raise ValueError("rendered solution must cite every proof node in order") + missing_markers = [ + node_id + for node_id in self.node_order + if f"[{node_id}]" not in self.solution + ] + if missing_markers: + raise ValueError( + f"rendered solution is missing node markers {missing_markers}" + ) + if self.terminal_answer.strip() != diffused_proof.terminal_answer.strip(): + raise ValueError("rendered terminal answer differs from diffused proof") + return self + + +class CandidateBundle(StrictModel): + replacement_plan: ReplacementPlan + diffused_proof: DiffusedProof + variant: RenderedVariant + + +class JudgeVerdict(StrictModel): + verdict: Literal["accept", "reject"] + step_by_step_check: str + replacement_check: str + blocking_issues: str = "" + patch_suggestion: str = "" + + @field_validator( + "step_by_step_check", + "replacement_check", + "blocking_issues", + "patch_suggestion", + mode="before", + ) + @classmethod + def normalize_text_fields(cls, value: Any) -> str: + if value is None: + return "" + if isinstance(value, str): + return value + return json.dumps(value, ensure_ascii=False, sort_keys=True) + + @field_validator("blocking_issues", "patch_suggestion", mode="after") + @classmethod + def normalize_absence_sentinels(cls, value: str) -> str: + normalized = value.strip().lower().rstrip(".") + if normalized in { + "", + "none", + "none detected", + "n/a", + "no issues", + "no blocking issues", + }: + return "" + return value + + @model_validator(mode="after") + def validate_verdict(self) -> "JudgeVerdict": + if self.verdict == "accept" and self.blocking_issues.strip(): + raise ValueError("accept verdict cannot contain blocking issues") + if self.verdict == "reject" and not self.blocking_issues.strip(): + raise ValueError("reject verdict must identify a blocking issue") + return self + + def validate_coverage( + self, + dag: ProofDAG, + replacement_plan: ReplacementPlan, + ) -> "JudgeVerdict": + missing_nodes = [ + node_id + for node_id in dag.node_ids() + if re.search( + rf"(?<![A-Za-z0-9_]){re.escape(node_id)}(?![A-Za-z0-9_])", + self.step_by_step_check, + ) + is None + ] + missing_slots = [ + change.slot_id + for change in replacement_plan.changes + if re.search( + rf"(?<![A-Za-z0-9_]){re.escape(change.slot_id)}(?![A-Za-z0-9_])", + self.replacement_check, + ) + is None + ] + if missing_nodes or missing_slots: + raise ValueError( + "judge coverage incomplete: " + f"nodes={missing_nodes}, slots={missing_slots}" + ) + return self + + +class VerificationIteration(StrictModel): + iteration: int + bundle_sha256: str + candidate_sha256: str + verdicts: list[JudgeVerdict] + unanimous: bool + pass_streak_after: int + repaired_from_previous: bool = False + + +class KernelRunResult(StrictModel): + item_id: str + status: Literal["accepted", "rejected"] + proof_dag: ProofDAG + method_plan: MethodPlan + accepted_replacement_plan: ReplacementPlan | None = None + accepted_diffused_proof: DiffusedProof | None = None + accepted_candidate: RenderedVariant | None = None + iterations: list[VerificationIteration] + rejection_reason: str = "" + accepted_candidate_sha256: str | None = None + accepted_bundle_sha256: str | None = None + schema_version: str = f"{SCHEMA_VERSION}-literal-five-stage" + + @model_validator(mode="after") + def validate_terminal_state(self) -> "KernelRunResult": + if self.status == "accepted": + required = [ + self.accepted_replacement_plan, + self.accepted_diffused_proof, + self.accepted_candidate, + self.accepted_candidate_sha256, + self.accepted_bundle_sha256, + ] + if any(value is None for value in required): + raise ValueError("accepted run must include complete stage provenance") + elif not self.rejection_reason: + raise ValueError("rejected run must state a reason") + return self diff --git a/src/gap_pipeline/kernel_prompts.py b/src/gap_pipeline/kernel_prompts.py new file mode 100644 index 0000000..c4a6128 --- /dev/null +++ b/src/gap_pipeline/kernel_prompts.py @@ -0,0 +1,276 @@ +"""Prompts for an explicit implementation of the paper's five kernel stages. + +The byte-pinned historical Prompt-A/Prompt-B remain in ``prompts.py``. These +prompts make the richer five-stage manuscript description executable and keep +every transformation decision in a typed artifact. +""" + +from __future__ import annotations + +import json + +from .kernel_models import ( + DiffusedProof, + MethodPlan, + ProofDAG, + ReplacementPlan, +) +from .models import CanonicalItem + + +DAG_SYSTEM = "You are a rigorous competition-math proof analyst." +DAG_USER = """Parse the official solution into a concrete proof DAG. + +Each node must be an intermediate mathematical claim, not a method name. +Each dependency must be a local entailment used to establish that node. +List nodes in topological order with IDs n1, n2, ... . Every node must +contribute to one terminal node. + +Return JSON only: +{{"nodes":[{{"node_id":"n1","claim":"...","dependencies":[]}}], + "terminal_node_id":"nN"}} + +ORIGINAL PROBLEM: +<<<{question}>>> + +OFFICIAL SOLUTION: +<<<{solution}>>>""" + + +METHOD_SYSTEM = "You abstract concrete proofs into content-free method plans." +METHOD_USER = """For every concrete proof-DAG node, provide one content-free +method label. Preserve node IDs and order exactly. A method label describes the +operation, not the source constants, variable names, or final numerical answer. + +Return JSON only: +{{"nodes":[{{"node_id":"n1","method_label":"..."}}]}} + +PROOF DAG: +{dag}""" + + +REPLACEMENT_SYSTEM = "You design guarded, proof-plan-preserving math replacements." +REPLACEMENT_USER = """Choose one or more substantive numerical or structural +replacements that create a genuinely new problem while preserving the supplied +method plan. + +For every change: +- state the exact source DAG node; +- record the original and replacement values explicitly; +- state a mathematical guard condition derived from the original problem; +- explain why the replacement satisfies the guard. + +The list must be closed: stages 4 and 5 may introduce no mathematical change +that is not declared here. Variable renaming alone is not a kernel change. +{feedback} + +Return JSON only: +{{"changes":[ + {{"slot_id":"s1","source_node_id":"n1","description":"...", + "original_value":"...","replacement_value":"...", + "guard_condition":"...","guard_justification":"..."}} + ], + "closure_statement":"All intended mathematical changes are listed above."}} + +ORIGINAL PROBLEM: +<<<{question}>>> + +OFFICIAL SOLUTION: +<<<{solution}>>> + +CONCRETE PROOF DAG: +{dag} + +CONTENT-FREE METHOD PLAN: +{methods}""" + + +DIFFUSION_SYSTEM = "You re-instantiate a proof DAG under declared guarded replacements." +DIFFUSION_USER = """Propagate only the declared replacements through the proof +DAG, node by node. Return exactly one row for every source node, preserving its +ID, dependencies, and method label. Do not introduce undeclared constants, +objects, assumptions, reductions, or proof methods. + +Return JSON only: +{{"nodes":[ + {{"node_id":"n1","dependencies":[],"method_label":"...", + "instantiated_claim":"...","justification":"..."}} + ], + "terminal_node_id":"nN", + "terminal_answer":"..."}} + +ORIGINAL PROBLEM: +<<<{question}>>> + +SOURCE PROOF DAG: +{dag} + +METHOD PLAN: +{methods} + +DECLARED REPLACEMENTS: +{replacements}""" + + +RENDER_SYSTEM = "You render a verified proof DAG into a self-contained Putnam problem." +RENDER_USER = """Render the diffused proof into one complete problem statement +and one complete solution. + +Requirements: +- the question must be fully determined by the diffused terminal claim; +- the solution must follow the diffused nodes in order and visibly mark each + paragraph with [n1], [n2], ...; +- node_order must equal exactly {node_order}; +- do not add any mathematical change beyond the declared replacements; +- copy the terminal answer exactly. + +Return JSON only: +{{"question":"...","solution":"...","node_order":{node_order}, + "terminal_answer":"..."}} + +DECLARED REPLACEMENTS: +{replacements} + +DIFFUSED PROOF: +{diffused}""" + + +JUDGE_SYSTEM = """You are a verification judge for a literal five-stage GAP +kernel transformation. Verification, not de novo solving, is your task.""" +JUDGE_USER = """Check the candidate against every disclosed artifact. + +You must verify: +1. every replacement satisfies its guard and all mathematical changes are + declared in the replacement plan; +2. each diffused node instantiates the matching source node and method label; +3. dependencies and proof order are preserved; +4. the rendered problem is well-posed and the rendered solution proves it; +5. the terminal answer agrees across diffusion, rendering, and solution. + +In step_by_step_check, mention each of these node IDs explicitly: +{required_node_ids} +In replacement_check, mention each of these slot IDs explicitly: +{required_slot_ids} +{format_feedback} + +Return JSON only: +{{"verdict":"accept" or "reject", + "step_by_step_check":"...", + "replacement_check":"...", + "blocking_issues":"...", + "patch_suggestion":"..."}} + +ORIGINAL PROBLEM: +<<<{question}>>> + +OFFICIAL SOLUTION: +<<<{solution}>>> + +SOURCE PROOF DAG: +{dag} + +METHOD PLAN: +{methods} + +REPLACEMENT PLAN: +{replacements} + +DIFFUSED PROOF: +{diffused} + +RENDERED VARIANT: +{variant}""" + + +def _dump(value: object) -> str: + if hasattr(value, "model_dump"): + value = value.model_dump(mode="json") + return json.dumps(value, ensure_ascii=False, indent=2) + + +def dag_user(item: CanonicalItem) -> str: + return DAG_USER.format(question=item.problem, solution=item.solution) + + +def method_user(dag: ProofDAG) -> str: + return METHOD_USER.format(dag=_dump(dag)) + + +def replacement_user( + item: CanonicalItem, + dag: ProofDAG, + methods: MethodPlan, + *, + feedback: str = "", +) -> str: + feedback_block = ( + f"Previous verification feedback to address:\n{feedback}" + if feedback + else "This is the initial replacement proposal." + ) + return REPLACEMENT_USER.format( + feedback=feedback_block, + question=item.problem, + solution=item.solution, + dag=_dump(dag), + methods=_dump(methods), + ) + + +def diffusion_user( + item: CanonicalItem, + dag: ProofDAG, + methods: MethodPlan, + replacements: ReplacementPlan, +) -> str: + return DIFFUSION_USER.format( + question=item.problem, + dag=_dump(dag), + methods=_dump(methods), + replacements=_dump(replacements), + ) + + +def render_user( + replacements: ReplacementPlan, + diffused: DiffusedProof, +) -> str: + return RENDER_USER.format( + node_order=json.dumps( + [node.node_id for node in diffused.nodes], + ensure_ascii=False, + ), + replacements=_dump(replacements), + diffused=_dump(diffused), + ) + + +def judge_user( + item: CanonicalItem, + dag: ProofDAG, + methods: MethodPlan, + replacements: ReplacementPlan, + diffused: DiffusedProof, + variant: object, + *, + format_feedback: str = "", +) -> str: + return JUDGE_USER.format( + required_node_ids=json.dumps(dag.node_ids(), ensure_ascii=False), + required_slot_ids=json.dumps( + [change.slot_id for change in replacements.changes], + ensure_ascii=False, + ), + format_feedback=( + f"Previous report-format error: {format_feedback}" + if format_feedback + else "" + ), + question=item.problem, + solution=item.solution, + dag=_dump(dag), + methods=_dump(methods), + replacements=_dump(replacements), + diffused=_dump(diffused), + variant=_dump(variant), + ) diff --git a/src/gap_pipeline/models.py b/src/gap_pipeline/models.py index 6fab32f..9b0f214 100644 --- a/src/gap_pipeline/models.py +++ b/src/gap_pipeline/models.py @@ -2,11 +2,12 @@ from __future__ import annotations +import json import re from datetime import datetime, timezone from typing import Any, Literal -from pydantic import BaseModel, ConfigDict, Field, model_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator SCHEMA_VERSION = "gap-prompt-faithful-v1" @@ -194,6 +195,20 @@ class JudgeVerdict(StrictModel): blocking_issues: str = "" patch_suggestion: str = "" + @field_validator( + "step_by_step_check", + "blocking_issues", + "patch_suggestion", + mode="before", + ) + @classmethod + def normalize_text_fields(cls, value: Any) -> str: + if value is None: + return "" + if isinstance(value, str): + return value + return json.dumps(value, ensure_ascii=False, sort_keys=True) + @model_validator(mode="after") def validate_verdict(self) -> "JudgeVerdict": if self.verdict == "accept" and self.blocking_issues.strip(): diff --git a/src/gap_pipeline/offline.py b/src/gap_pipeline/offline.py index 2be8922..e1c4927 100644 --- a/src/gap_pipeline/offline.py +++ b/src/gap_pipeline/offline.py @@ -198,7 +198,7 @@ def summarize_run(run_dir: Path) -> dict[str, Any]: first_round_failed += 1 if status == "accepted": accepted_ids.append(item_id) - if any(row.get("repaired", False) for row in history): + if any(not row.get("unanimous", False) for row in history): repaired_accepts += 1 else: rejected_ids.append(item_id) @@ -234,12 +234,12 @@ def align_run_to_release(run_dir: Path, dataset_dir: Path) -> dict[str, Any]: rows.append({"item_id": item_id, "status": "missing_from_release"}) continue released_kernel = release[item_id]["variants"]["kernel_variant"] - question_exact = candidate["problem"] == released_kernel["question"] - solution_exact = candidate["proof"] == released_kernel["solution"] - question_normalized = normalize_text(candidate["problem"]) == normalize_text( + question_exact = candidate["question"] == released_kernel["question"] + solution_exact = candidate["solution"] == released_kernel["solution"] + question_normalized = normalize_text(candidate["question"]) == normalize_text( released_kernel["question"] ) - solution_normalized = normalize_text(candidate["proof"]) == normalize_text( + solution_normalized = normalize_text(candidate["solution"]) == normalize_text( released_kernel["solution"] ) rows.append( @@ -258,8 +258,8 @@ def align_run_to_release(run_dir: Path, dataset_dir: Path) -> dict[str, Any]: "solution_normalized": solution_normalized, "candidate_sha256": sha256_payload( { - "question": candidate["problem"], - "solution": candidate["proof"], + "question": candidate["question"], + "solution": candidate["solution"], } ), "release_sha256": sha256_payload( diff --git a/src/gap_pipeline/paper_pipeline.py b/src/gap_pipeline/paper_pipeline.py new file mode 100644 index 0000000..84f34bf --- /dev/null +++ b/src/gap_pipeline/paper_pipeline.py @@ -0,0 +1,395 @@ +"""Literal five-stage GAP kernel pipeline matching the manuscript operations.""" + +from __future__ import annotations + +import asyncio + +from pydantic import BaseModel, ConfigDict, model_validator + +from .clients import JsonLLM +from .kernel_models import ( + CandidateBundle, + DiffusedProof, + JudgeVerdict, + KernelRunResult, + MethodPlan, + ProofDAG, + RenderedVariant, + ReplacementPlan, + VerificationIteration, +) +from .kernel_prompts import ( + DAG_SYSTEM, + DIFFUSION_SYSTEM, + JUDGE_SYSTEM, + METHOD_SYSTEM, + RENDER_SYSTEM, + REPLACEMENT_SYSTEM, + dag_user, + diffusion_user, + judge_user, + method_user, + render_user, + replacement_user, +) +from .models import CanonicalItem, ModelCallRecord +from .store import RunStore, sha256_payload + + +class PaperPipelineConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + + protocol_name: str = "gap-literal-five-stage-J5-K2-T15" + proposer_model: str + judge_model: str + judge_count: int = 5 + streak_length: int = 2 + max_iterations: int = 15 + + @model_validator(mode="after") + def enforce_protocol(self) -> "PaperPipelineConfig": + if (self.judge_count, self.streak_length, self.max_iterations) != (5, 2, 15): + raise ValueError("the GAP protocol is fixed at J=5, K=2, T=15") + return self + + +class PaperKernelPipeline: + """Execute all five paper stages as typed, separately auditable calls.""" + + def __init__( + self, + *, + proposer: JsonLLM, + judges: list[JsonLLM], + store: RunStore, + config: PaperPipelineConfig, + ) -> None: + if len(judges) != config.judge_count: + raise ValueError( + f"expected {config.judge_count} judges, received {len(judges)}" + ) + self.proposer = proposer + self.judges = judges + self.store = store + self.config = config + + async def _call( + self, + client: JsonLLM, + *, + request_id: str, + system_prompt: str, + user_prompt: str, + ) -> dict: + response = await client.generate_json( + system_prompt=system_prompt, + user_prompt=user_prompt, + request_id=request_id, + ) + self.store.write_call( + ModelCallRecord( + request_id=request_id, + model=response.model, + system_prompt=system_prompt, + user_prompt=user_prompt, + response_data=response.data, + raw_text=response.raw_text, + provider_response_id=response.response_id, + usage=response.usage, + ) + ) + return response.data + + async def construct_dag(self, item: CanonicalItem) -> ProofDAG: + request_id = f"{item.item_id}.stage1.dag" + dag = ProofDAG.model_validate( + await self._call( + self.proposer, + request_id=request_id, + system_prompt=DAG_SYSTEM, + user_prompt=dag_user(item), + ) + ) + self.store.write_stage("01_proof_dag", dag, request_id=request_id) + return dag + + async def summarize_methods(self, dag: ProofDAG) -> MethodPlan: + request_id = f"{self.store.item_id}.stage2.methods" + methods = MethodPlan.model_validate( + await self._call( + self.proposer, + request_id=request_id, + system_prompt=METHOD_SYSTEM, + user_prompt=method_user(dag), + ) + ).validate_against(dag) + self.store.write_stage("02_method_plan", methods, request_id=request_id) + return methods + + async def generate_replacements( + self, + item: CanonicalItem, + dag: ProofDAG, + methods: MethodPlan, + *, + version: int, + feedback: str = "", + ) -> ReplacementPlan: + request_id = f"{item.item_id}.stage3.replacement.v{version:02d}" + replacements = ReplacementPlan.model_validate( + await self._call( + self.proposer, + request_id=request_id, + system_prompt=REPLACEMENT_SYSTEM, + user_prompt=replacement_user( + item, + dag, + methods, + feedback=feedback, + ), + ) + ).validate_against(dag) + self.store.write_stage( + f"03_replacement_v{version:02d}", + replacements, + request_id=request_id, + ) + return replacements + + async def diffuse_dag( + self, + item: CanonicalItem, + dag: ProofDAG, + methods: MethodPlan, + replacements: ReplacementPlan, + *, + version: int, + ) -> DiffusedProof: + request_id = f"{item.item_id}.stage4.diffusion.v{version:02d}" + diffused = DiffusedProof.model_validate( + await self._call( + self.proposer, + request_id=request_id, + system_prompt=DIFFUSION_SYSTEM, + user_prompt=diffusion_user(item, dag, methods, replacements), + ) + ).validate_against(dag, methods) + self.store.write_stage( + f"04_diffused_proof_v{version:02d}", + diffused, + request_id=request_id, + ) + return diffused + + async def render_variant( + self, + dag: ProofDAG, + replacements: ReplacementPlan, + diffused: DiffusedProof, + *, + version: int, + ) -> RenderedVariant: + request_id = f"{self.store.item_id}.stage5.render.v{version:02d}" + variant = RenderedVariant.model_validate( + await self._call( + self.proposer, + request_id=request_id, + system_prompt=RENDER_SYSTEM, + user_prompt=render_user(replacements, diffused), + ) + ).validate_against(dag, diffused) + self.store.write_stage( + f"05_rendered_variant_v{version:02d}", + variant, + request_id=request_id, + ) + return variant + + async def build_bundle( + self, + item: CanonicalItem, + dag: ProofDAG, + methods: MethodPlan, + *, + version: int, + feedback: str = "", + ) -> CandidateBundle: + replacements = await self.generate_replacements( + item, + dag, + methods, + version=version, + feedback=feedback, + ) + diffused = await self.diffuse_dag( + item, + dag, + methods, + replacements, + version=version, + ) + variant = await self.render_variant( + dag, + replacements, + diffused, + version=version, + ) + return CandidateBundle( + replacement_plan=replacements, + diffused_proof=diffused, + variant=variant, + ) + + async def _judge_once( + self, + judge: JsonLLM, + *, + item: CanonicalItem, + dag: ProofDAG, + methods: MethodPlan, + bundle: CandidateBundle, + iteration: int, + judge_id: int, + ) -> JudgeVerdict: + format_feedback = "" + for attempt in range(1, 4): + request_id = ( + f"{item.item_id}.verify.t{iteration:02d}." + f"j{judge_id}.a{attempt}" + ) + verdict = JudgeVerdict.model_validate( + await self._call( + judge, + request_id=request_id, + system_prompt=JUDGE_SYSTEM, + user_prompt=judge_user( + item, + dag, + methods, + bundle.replacement_plan, + bundle.diffused_proof, + bundle.variant, + format_feedback=format_feedback, + ), + ) + ) + try: + return verdict.validate_coverage(dag, bundle.replacement_plan) + except ValueError as exc: + format_feedback = str(exc) + raise ValueError( + f"judge {judge_id} failed coverage after three format attempts: " + f"{format_feedback}" + ) + + @staticmethod + def _feedback(verdicts: list[JudgeVerdict]) -> str: + rows = [] + for index, verdict in enumerate(verdicts, start=1): + if verdict.verdict == "reject": + rows.append( + f"judge {index}: {verdict.blocking_issues}; " + f"suggested repair: {verdict.patch_suggestion}" + ) + return "\n".join(rows) + + async def verify( + self, + item: CanonicalItem, + dag: ProofDAG, + methods: MethodPlan, + initial_bundle: CandidateBundle, + ) -> KernelRunResult: + bundle = initial_bundle + iterations: list[VerificationIteration] = [] + pass_streak = 0 + streak_sha: str | None = None + repaired_from_previous = False + + for iteration in range(1, self.config.max_iterations + 1): + bundle_sha = sha256_payload(bundle) + candidate_sha = sha256_payload(bundle.variant) + verdicts = list( + await asyncio.gather( + *( + self._judge_once( + judge, + item=item, + dag=dag, + methods=methods, + bundle=bundle, + iteration=iteration, + judge_id=judge_id, + ) + for judge_id, judge in enumerate(self.judges, start=1) + ) + ) + ) + unanimous = all(verdict.verdict == "accept" for verdict in verdicts) + if unanimous: + if streak_sha not in {None, bundle_sha}: + raise AssertionError("pass streak crossed provenance versions") + streak_sha = bundle_sha + pass_streak += 1 + else: + pass_streak = 0 + streak_sha = None + + record = VerificationIteration( + iteration=iteration, + bundle_sha256=bundle_sha, + candidate_sha256=candidate_sha, + verdicts=verdicts, + unanimous=unanimous, + pass_streak_after=pass_streak, + repaired_from_previous=repaired_from_previous, + ) + iterations.append(record) + self.store.write_iteration(iteration, record) + + if pass_streak == self.config.streak_length: + result = KernelRunResult( + item_id=item.item_id, + status="accepted", + proof_dag=dag, + method_plan=methods, + accepted_replacement_plan=bundle.replacement_plan, + accepted_diffused_proof=bundle.diffused_proof, + accepted_candidate=bundle.variant, + iterations=iterations, + accepted_candidate_sha256=candidate_sha, + accepted_bundle_sha256=bundle_sha, + ) + self.store.write_final(result) + return result + + if not unanimous and iteration < self.config.max_iterations: + bundle = await self.build_bundle( + item, + dag, + methods, + version=iteration + 1, + feedback=self._feedback(verdicts), + ) + repaired_from_previous = True + else: + repaired_from_previous = False + + result = KernelRunResult( + item_id=item.item_id, + status="rejected", + proof_dag=dag, + method_plan=methods, + iterations=iterations, + rejection_reason="no two consecutive unanimous rounds within T=15", + ) + self.store.write_final(result) + return result + + async def run(self, item: CanonicalItem) -> KernelRunResult: + self.store.write_input(item) + self.store.write_config(self.config) + dag = await self.construct_dag(item) + methods = await self.summarize_methods(dag) + bundle = await self.build_bundle(item, dag, methods, version=1) + return await self.verify(item, dag, methods, bundle) diff --git a/src/gap_pipeline/pipeline.py b/src/gap_pipeline/pipeline.py index 76ae722..4a5332a 100644 --- a/src/gap_pipeline/pipeline.py +++ b/src/gap_pipeline/pipeline.py @@ -1,4 +1,7 @@ -"""Prompt-faithful kernel generation and the J=5, K=2, T=15 loop.""" +"""Historical two-call Putnam generator retained for provenance tests. + +The default manuscript-aligned implementation is ``paper_pipeline.py``. +""" from __future__ import annotations diff --git a/src/gap_pipeline/release.py b/src/gap_pipeline/release.py index 9afa117..7df6566 100644 --- a/src/gap_pipeline/release.py +++ b/src/gap_pipeline/release.py @@ -92,6 +92,19 @@ def export_release( variants["kernel_variant"] = { "question": candidate["question"], "solution": candidate["solution"], + "_meta": { + "proof_dag": kernel_payload["proof_dag"], + "method_plan": kernel_payload["method_plan"], + "replacement_plan": kernel_payload["accepted_replacement_plan"], + "diffused_proof": kernel_payload["accepted_diffused_proof"], + "terminal_answer": candidate["terminal_answer"], + "accepted_candidate_sha256": kernel_payload[ + "accepted_candidate_sha256" + ], + "accepted_bundle_sha256": kernel_payload[ + "accepted_bundle_sha256" + ], + }, } output_record = copy.deepcopy(record) output_record["variants"] = variants diff --git a/src/gap_pipeline/surface.py b/src/gap_pipeline/surface.py index 914d847..aae34a8 100644 --- a/src/gap_pipeline/surface.py +++ b/src/gap_pipeline/surface.py @@ -29,6 +29,24 @@ SURFACE_FAMILIES: tuple[SurfaceFamily, ...] = ( IDENTIFIER_RE = re.compile(r"^[a-z]{8,}$") +def apply_rename_map(text: str, rename_map: dict[str, str]) -> str: + """Apply symbol renames without replacing letters inside other tokens.""" + + rendered = text + for source in sorted(rename_map, key=len, reverse=True): + pattern = re.escape(source) + if source[0].isalnum(): + pattern = rf"(?<![A-Za-z0-9_]){pattern}" + if source[-1].isalnum(): + pattern = rf"{pattern}(?![A-Za-z0-9_])" + rendered = re.sub( + pattern, + lambda _match, replacement=rename_map[source]: replacement, + rendered, + ) + return rendered + + @dataclass(frozen=True) class SurfaceVariant: family: SurfaceFamily @@ -69,6 +87,13 @@ def validate_surface_variant( ) if not question.strip() or not solution.strip(): raise ValueError("surface question and solution must be non-empty") + expected_question = apply_rename_map(item.problem, rename_map) + expected_solution = apply_rename_map(item.solution, rename_map) + if question != expected_question or solution != expected_solution: + raise ValueError( + "surface text must be the exact deterministic rename of the " + "canonical question and solution" + ) class SurfacePipeline: @@ -105,8 +130,8 @@ class SurfacePipeline: str(old): str(new) for old, new in dict(response.data["map"]).items() } - question = str(response.data["question"]) - solution = str(response.data["solution"]) + question = apply_rename_map(item.problem, rename_map) + solution = apply_rename_map(item.solution, rename_map) validate_surface_variant( item, rename_map=rename_map, diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 9da1190..9672527 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -28,3 +28,7 @@ def test_offline_end_to_end_smoke(tmp_path: Path) -> None: "garbled_string", "kernel_variant", } + kernel_meta = record["variants"]["kernel_variant"]["_meta"] + assert kernel_meta["proof_dag"]["terminal_node_id"] == "n3" + assert kernel_meta["replacement_plan"]["changes"][0]["slot_id"] == "s1" + assert kernel_meta["diffused_proof"]["terminal_answer"] == "4" diff --git a/tests/test_kernel_models.py b/tests/test_kernel_models.py new file mode 100644 index 0000000..864d6d5 --- /dev/null +++ b/tests/test_kernel_models.py @@ -0,0 +1,138 @@ +from __future__ import annotations + +import pytest + +from gap_pipeline.kernel_models import ( + DiffusedProof, + JudgeVerdict, + MethodPlan, + ProofDAG, + RenderedVariant, + ReplacementPlan, +) + + +def branched_dag() -> ProofDAG: + return ProofDAG.model_validate( + { + "nodes": [ + {"node_id": "n1", "claim": "first fact", "dependencies": []}, + {"node_id": "n2", "claim": "left branch", "dependencies": ["n1"]}, + {"node_id": "n3", "claim": "right branch", "dependencies": ["n1"]}, + { + "node_id": "n4", + "claim": "combine branches", + "dependencies": ["n2", "n3"], + }, + ], + "terminal_node_id": "n4", + } + ) + + +def test_proof_dag_supports_real_branching() -> None: + dag = branched_dag() + assert dag.nodes[-1].dependencies == ["n2", "n3"] + + +def test_replacement_must_reference_a_real_dag_node() -> None: + replacements = ReplacementPlan.model_validate( + { + "changes": [ + { + "slot_id": "s1", + "source_node_id": "n9", + "description": "constant", + "original_value": "1", + "replacement_value": "2", + "guard_condition": "positive", + "guard_justification": "2 is positive", + } + ], + "closure_statement": "No undeclared changes.", + } + ) + with pytest.raises(ValueError, match="unknown nodes"): + replacements.validate_against(branched_dag()) + + +def test_diffusion_preserves_dependencies_and_methods() -> None: + dag = branched_dag() + methods = MethodPlan.model_validate( + { + "nodes": [ + {"node_id": node.node_id, "method_label": f"method {node.node_id}"} + for node in dag.nodes + ] + } + ) + diffused = DiffusedProof.model_validate( + { + "nodes": [ + { + "node_id": node.node_id, + "dependencies": node.dependencies, + "method_label": f"method {node.node_id}", + "instantiated_claim": f"new {node.claim}", + "justification": "valid re-instantiation", + } + for node in dag.nodes + ], + "terminal_node_id": "n4", + "terminal_answer": "answer", + } + ) + assert diffused.validate_against(dag, methods) is diffused + + broken = diffused.model_copy(deep=True) + broken.nodes[-1].dependencies = ["n3"] + with pytest.raises(ValueError, match="dependencies changed"): + broken.validate_against(dag, methods) + + +def test_rendered_solution_must_expose_every_node() -> None: + dag = branched_dag() + methods = MethodPlan.model_validate( + { + "nodes": [ + {"node_id": node.node_id, "method_label": f"method {node.node_id}"} + for node in dag.nodes + ] + } + ) + diffused = DiffusedProof.model_validate( + { + "nodes": [ + { + "node_id": node.node_id, + "dependencies": node.dependencies, + "method_label": f"method {node.node_id}", + "instantiated_claim": f"new {node.claim}", + "justification": "valid", + } + for node in dag.nodes + ], + "terminal_node_id": "n4", + "terminal_answer": "answer", + } + ).validate_against(dag, methods) + variant = RenderedVariant( + question="New problem", + solution="[n1] first [n2] left [n3] right", + node_order=dag.node_ids(), + terminal_answer="answer", + ) + with pytest.raises(ValueError, match="missing node markers.*n4"): + variant.validate_against(dag, diffused) + + +def test_accept_verdict_normalizes_explicit_none_sentinel() -> None: + verdict = JudgeVerdict( + verdict="accept", + step_by_step_check="n1 valid", + replacement_check="s1 valid", + blocking_issues="None detected.", + patch_suggestion="N/A", + ) + assert verdict.blocking_issues == "" + assert verdict.patch_suggestion == "" diff --git a/tests/test_models.py b/tests/test_models.py index d19b536..b329ecd 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -111,6 +111,26 @@ def test_judge_verdict_requires_every_dag_node(plan_dict: dict) -> None: ).validate_coverage(dag) +def test_judge_verdict_normalizes_structured_text_fields(plan_dict: dict) -> None: + dag = ProofPlanDAG.from_plan(KernelPlan.model_validate(plan_dict)) + verdict = JudgeVerdict.model_validate( + { + "verdict": "accept", + "step_by_step_check": { + "n1": "correctly instantiated", + "n2": "correctly instantiated", + }, + "blocking_issues": None, + "patch_suggestion": None, + } + ) + + assert '"n1"' in verdict.step_by_step_check + assert verdict.blocking_issues == "" + assert verdict.patch_suggestion == "" + assert verdict.validate_coverage(dag) is verdict + + def test_dag_labels_must_match_prompt_a_plan(plan_dict: dict) -> None: plan = KernelPlan.model_validate(plan_dict) dag = ProofPlanDAG.from_plan(plan) diff --git a/tests/test_offline.py b/tests/test_offline.py index 04b7f46..9851826 100644 --- a/tests/test_offline.py +++ b/tests/test_offline.py @@ -1,6 +1,72 @@ -from gap_pipeline.offline import normalize_latex_symbol +import json + +from gap_pipeline.offline import ( + align_run_to_release, + normalize_latex_symbol, + summarize_run, +) def test_latex_symbol_aliases_normalize_together() -> None: assert normalize_latex_symbol(r"x_{n}") == normalize_latex_symbol("x_n") assert normalize_latex_symbol(r"\\phi") == normalize_latex_symbol(r"\phi") + + +def test_run_summary_counts_acceptance_after_repair(tmp_path) -> None: + item_dir = tmp_path / "items" / "demo" + item_dir.mkdir(parents=True) + (item_dir / "final.json").write_text( + json.dumps( + { + "item_id": "demo", + "status": "accepted", + "iterations": [ + {"unanimous": False}, + {"unanimous": True}, + {"unanimous": True}, + ], + } + ), + encoding="utf-8", + ) + + summary = summarize_run(tmp_path) + assert summary["accepted_after_repair"] == 1 + + +def test_align_release_uses_current_candidate_schema(tmp_path) -> None: + run_dir = tmp_path / "run" + item_dir = run_dir / "items" / "demo" + item_dir.mkdir(parents=True) + (item_dir / "final.json").write_text( + json.dumps( + { + "item_id": "demo", + "status": "accepted", + "accepted_candidate": { + "question": "New question", + "solution": "New solution", + }, + } + ), + encoding="utf-8", + ) + dataset_dir = tmp_path / "dataset" + dataset_dir.mkdir() + (dataset_dir / "demo.json").write_text( + json.dumps( + { + "index": "demo", + "variants": { + "kernel_variant": { + "question": "New question", + "solution": "New solution", + } + }, + } + ), + encoding="utf-8", + ) + + alignment = align_run_to_release(run_dir, dataset_dir) + assert alignment["status_counts"] == {"exact_match": 1} diff --git a/tests/test_prompts.py b/tests/test_prompts.py index a2bf892..4f6568b 100644 --- a/tests/test_prompts.py +++ b/tests/test_prompts.py @@ -5,6 +5,20 @@ import inspect from gap_pipeline import prompts from gap_pipeline.clients import OpenAIJsonClient +from gap_pipeline.kernel_models import ( + DiffusedProof, + MethodPlan, + ProofDAG, + ReplacementPlan, +) +from gap_pipeline.kernel_prompts import ( + dag_user, + diffusion_user, + judge_user, + method_user, + render_user, + replacement_user, +) EXPECTED = { @@ -38,3 +52,62 @@ def test_prompt_values_are_byte_locked() -> None: def test_o3_adapter_does_not_send_temperature() -> None: source = inspect.getsource(OpenAIJsonClient.generate_json) assert '"temperature"' not in source + + +def test_literal_five_stage_prompts_render(item) -> None: + dag = ProofDAG.model_validate( + { + "nodes": [{"node_id": "n1", "claim": "claim", "dependencies": []}], + "terminal_node_id": "n1", + } + ) + methods = MethodPlan.model_validate( + {"nodes": [{"node_id": "n1", "method_label": "method"}]} + ) + replacements = ReplacementPlan.model_validate( + { + "changes": [ + { + "slot_id": "s1", + "source_node_id": "n1", + "description": "constant", + "original_value": "1", + "replacement_value": "2", + "guard_condition": "positive", + "guard_justification": "2 is positive", + } + ], + "closure_statement": "No undeclared changes.", + } + ) + diffused = DiffusedProof.model_validate( + { + "nodes": [ + { + "node_id": "n1", + "dependencies": [], + "method_label": "method", + "instantiated_claim": "new claim", + "justification": "valid", + } + ], + "terminal_node_id": "n1", + "terminal_answer": "answer", + } + ) + variant = { + "question": "question", + "solution": "[n1] solution", + "node_order": ["n1"], + "terminal_answer": "answer", + } + + rendered = [ + dag_user(item), + method_user(dag), + replacement_user(item, dag, methods), + diffusion_user(item, dag, methods, replacements), + render_user(replacements, diffused), + judge_user(item, dag, methods, replacements, diffused, variant), + ] + assert all("{" in value and "}" in value for value in rendered) diff --git a/tests/test_release.py b/tests/test_release.py index b825478..8927b14 100644 --- a/tests/test_release.py +++ b/tests/test_release.py @@ -4,7 +4,6 @@ import json import pytest -from gap_pipeline.models import KernelCandidate from gap_pipeline.release import export_release from gap_pipeline.store import RunStore, sha256_payload from gap_pipeline.surface import SURFACE_FAMILIES @@ -47,13 +46,52 @@ def test_offline_release_export_verifies_and_assembles( kernel_root = tmp_path / "kernel-runs" kernel_store = RunStore(kernel_root, item.item_id) - candidate = KernelCandidate.model_validate(candidate_dict) + candidate = { + **candidate_dict, + "node_order": ["n1"], + "terminal_answer": "4", + } kernel_store.write_final( { "item_id": item.item_id, "status": "accepted", - "accepted_candidate": candidate.model_dump(mode="json"), + "proof_dag": { + "nodes": [{"node_id": "n1", "claim": "claim", "dependencies": []}], + "terminal_node_id": "n1", + }, + "method_plan": { + "nodes": [{"node_id": "n1", "method_label": "method"}] + }, + "accepted_replacement_plan": { + "changes": [ + { + "slot_id": "s1", + "source_node_id": "n1", + "description": "value", + "original_value": "1", + "replacement_value": "2", + "guard_condition": "positive", + "guard_justification": "2 is positive", + } + ], + "closure_statement": "closed", + }, + "accepted_diffused_proof": { + "nodes": [ + { + "node_id": "n1", + "dependencies": [], + "method_label": "method", + "instantiated_claim": "claim", + "justification": "reason", + } + ], + "terminal_node_id": "n1", + "terminal_answer": "4", + }, + "accepted_candidate": candidate, "accepted_candidate_sha256": sha256_payload(candidate), + "accepted_bundle_sha256": "bundle-sha", } ) @@ -70,7 +108,8 @@ def test_offline_release_export_verifies_and_assembles( (output_root / "records" / f"{item.item_id}.json").read_text() ) assert set(output["variants"]) == {*SURFACE_FAMILIES, "kernel_variant"} - assert output["variants"]["kernel_variant"]["question"] == candidate.question + assert output["variants"]["kernel_variant"]["question"] == candidate["question"] + assert output["variants"]["kernel_variant"]["_meta"]["replacement_plan"] with pytest.raises(FileExistsError): export_release( source_dataset=source_dir, diff --git a/tests/test_surface.py b/tests/test_surface.py index 4d92de3..5f38d57 100644 --- a/tests/test_surface.py +++ b/tests/test_surface.py @@ -6,7 +6,11 @@ import pytest from gap_pipeline.clients import ScriptedClient from gap_pipeline.store import RunStore -from gap_pipeline.surface import SurfacePipeline, validate_surface_variant +from gap_pipeline.surface import ( + SurfacePipeline, + apply_rename_map, + validate_surface_variant, +) def test_surface_pipeline_uses_full_original_contract(tmp_path, item) -> None: @@ -25,8 +29,15 @@ def test_surface_pipeline_uses_full_original_contract(tmp_path, item) -> None: ).run_family(item, "descriptive_long") ) assert variant.rename_map == {"a": "positivequantity"} - assert variant.question == response["question"] - assert variant.solution == response["solution"] + assert variant.question == apply_rename_map( + item.problem, {"a": "positivequantity"} + ) + assert variant.solution == apply_rename_map( + item.solution, {"a": "positivequantity"} + ) + assert "that" in variant.question + assert "expand" in variant.solution + assert "obtain" in variant.solution def test_missing_symbol_and_nonbijection_are_rejected(item) -> None: @@ -56,3 +67,23 @@ def test_original_identifier_contract_is_enforced(item) -> None: question="question", solution="solution", ) + + +def test_surface_text_must_be_only_a_token_safe_rename(item) -> None: + rename_map = {"a": "positivequantity"} + question = apply_rename_map(item.problem, rename_map) + solution = apply_rename_map(item.solution, rename_map) + validate_surface_variant( + item, + rename_map=rename_map, + question=question, + solution=solution, + ) + + with pytest.raises(ValueError, match="exact deterministic rename"): + validate_surface_variant( + item, + rename_map=rename_map, + question=question + " Extra text.", + solution=solution, + ) |