diff options
Diffstat (limited to 'src/gap_pipeline/e2e.py')
| -rw-r--r-- | src/gap_pipeline/e2e.py | 194 |
1 files changed, 57 insertions, 137 deletions
diff --git a/src/gap_pipeline/e2e.py b/src/gap_pipeline/e2e.py index 093068f..971da3e 100644 --- a/src/gap_pipeline/e2e.py +++ b/src/gap_pipeline/e2e.py @@ -1,4 +1,4 @@ -"""End-to-end helpers used by the reproducibility notebook.""" +"""End-to-end helpers used by the reproduction notebook.""" from __future__ import annotations @@ -14,7 +14,7 @@ from .offline import load_dataset from .pipeline import KernelPipeline, PipelineConfig from .release import export_release from .store import RunStore -from .surface import SURFACE_FAMILIES, SurfacePipeline +from .surface import SurfacePipeline def _ensure_fresh(path: Path) -> None: @@ -22,113 +22,28 @@ def _ensure_fresh(path: Path) -> None: raise FileExistsError(f"refusing to reuse non-empty output directory {path}") -def _accept_verdict() -> dict[str, Any]: +def _review_accept() -> dict[str, str]: return { "verdict": "accept", - "step_by_step_check": "n1 and n2 instantiate the fixed method plan", + "step_by_step_check": "both method labels are instantiated", "blocking_issues": "", "patch_suggestion": "", } -def _offline_fixture() -> dict[str, Any]: - plan = { - "steps": [ - { - "node_id": "n1", - "method_label": "use nonnegativity of a square", - "input_roles": ["positive scalar"], - "output_role": "nonnegative expression", - "invariants": ["scalar is real"], - }, - { - "node_id": "n2", - "method_label": "expand and divide by a positive quantity", - "input_roles": ["nonnegative expression", "positive scalar"], - "output_role": "target inequality", - "invariants": ["divisor is positive"], - }, - ], - "terminal_node_id": "n2", - } - replacement = { - "target_node_id": "n1", - "original_object": "(a-1)^2", - "replacement_object": "(x-2)^2", - "guard_conditions": ["x>0"], - "guard_evidence": ["positive real x satisfies the division guard"], - "expected_downstream_changes": ["expand around 2", "divide by x"], - "rationale": "the square-nonnegativity plan is unchanged", - } - diffused = { - "node_instantiations": [ - { - "node_id": "n1", - "method_label": plan["steps"][0]["method_label"], - "instantiated_claim": "(x-2)^2 >= 0", - "instantiated_derivation": "a square is nonnegative", - "dependencies": [], - }, - { - "node_id": "n2", - "method_label": plan["steps"][1]["method_label"], - "instantiated_claim": "x+4/x >= 4", - "instantiated_derivation": "expand and divide by positive x", - "dependencies": ["n1"], - }, - ], - "regenerated_proof": ( - "Since (x-2)^2 >= 0, expansion and division by x>0 give " - "x+4/x >= 4." - ), - "terminal_answer": "x+4/x >= 4", - } - candidate = { - "problem": "Let x>0. Prove that x+4/x >= 4.", - "proof": diffused["regenerated_proof"], - "terminal_answer": diffused["terminal_answer"], - "node_instantiations": diffused["node_instantiations"], - "replacement": replacement, - } +def _offline_record() -> dict[str, Any]: return { - "record": { - "index": "demo-A-1", - "question": r"Let \(a>0\). Prove that \(a+1/a\ge 2\).", - "solution": ( - r"Since \((a-1)^2\ge0\), expand and divide by \(a>0\) " - r"to obtain \(a+1/a\ge2\)." - ), - "vars": ["a"], - "params": [], - "sci_consts": [], - "problem_type": "proof", - "variants": {}, - }, - "dag": { - "nodes": [ - { - "node_id": "n1", - "claim": "(a-1)^2 >= 0", - "derivation": "a square is nonnegative", - "dependencies": [], - "source_span": "Since (a-1)^2 >= 0", - "mathematical_objects": ["a", "(a-1)^2"], - }, - { - "node_id": "n2", - "claim": "a+1/a >= 2", - "derivation": "expand and divide by positive a", - "dependencies": ["n1"], - "source_span": "divide by a>0", - "mathematical_objects": ["a"], - }, - ], - "terminal_node_id": "n2", - }, - "plan": plan, - "replacement": replacement, - "diffused": diffused, - "candidate": candidate, + "index": "demo-A-1", + "question": r"Let \(a>0\). Prove that \(a+1/a\ge 2\).", + "solution": ( + r"Since \((a-1)^2\ge0\), expand and divide by \(a>0\) " + r"to obtain \(a+1/a\ge2\)." + ), + "vars": ["a"], + "params": [], + "sci_consts": [], + "problem_type": "proof", + "variants": {}, } @@ -140,7 +55,7 @@ async def run_live_item( model: str = "o3", api_key: str | None = None, ) -> dict[str, Any]: - """Generate all five GAP variants for one Putnam item and export them.""" + """Generate four surface variants and one verified kernel variant.""" if not (api_key or os.getenv("OPENAI_API_KEY")): raise RuntimeError( @@ -162,29 +77,23 @@ async def run_live_item( surface_store = RunStore(surface_root, item.item_id) surface_store.write_input(item) surface_store.write_config( - { - "protocol_name": "gap-surface-v1", - "proposer_model": model, - } + {"protocol_name": "gap-surface-original-prompts", "model": model} ) - surface_pipeline = SurfacePipeline( + surfaces = await SurfacePipeline( OpenAIJsonClient(model, api_key=api_key), surface_store, - ) - surfaces = await surface_pipeline.run_all(item) + ).run_all(item) config = PipelineConfig(proposer_model=model, judge_model=model) - kernel_pipeline = KernelPipeline( + kernel = await KernelPipeline( proposer=OpenAIJsonClient(model, api_key=api_key), judges=[OpenAIJsonClient(model, api_key=api_key) for _ in range(5)], store=RunStore(kernel_root, item.item_id), config=config, - ) - kernel = await kernel_pipeline.run(item) + ).run(item) if kernel.status != "accepted": raise RuntimeError( - f"kernel candidate was rejected after {len(kernel.iterations)} rounds; " - f"inspect {kernel_root / 'items' / item.item_id}" + f"kernel candidate was rejected after {len(kernel.iterations)} rounds" ) manifest = export_release( @@ -211,7 +120,7 @@ async def run_live_item( async def run_offline_smoke(work_root: Path) -> dict[str, Any]: - """Exercise the same end-to-end code path with deterministic responses.""" + """Exercise the end-to-end wiring with deterministic model responses.""" _ensure_fresh(work_root) work_root.mkdir(parents=True, exist_ok=True) @@ -221,8 +130,7 @@ async def run_offline_smoke(work_root: Path) -> dict[str, Any]: kernel_root = work_root / "kernel-runs" release_root = work_root / "release" - fixture = _offline_fixture() - record = fixture["record"] + record = _offline_record() item_id = str(record["index"]) (source_root / f"{item_id}.json").write_text( json.dumps(record, ensure_ascii=False, indent=2) + "\n", @@ -230,21 +138,24 @@ async def run_offline_smoke(work_root: Path) -> dict[str, Any]: ) item = CanonicalItem.from_public_record(record) + names = { + "descriptive_long": "positivequantity", + "descriptive_long_confusing": "walnutvioletterrace", + "descriptive_long_misleading": "primefieldorder", + "garbled_string": "qzxwvtnphjgrksla", + } surface_responses = { - f"{item_id}.surface.{family}.a": { - "replacement": { - "descriptive_long": "positiveQuantity", - "descriptive_long_confusing": "walnutVioletTerrace", - "descriptive_long_misleading": "primeFieldOrder", - "garbled_string": "xcQ7h2ZfRw9v", - }[family] + f"{item_id}.surface.{family}": { + "map": {"a": name}, + "question": record["question"].replace("a", name), + "solution": record["solution"].replace("a", name), } - for family in SURFACE_FAMILIES + for family, name in names.items() } surface_store = RunStore(surface_root, item_id) surface_store.write_input(item) surface_store.write_config( - {"protocol_name": "gap-surface-smoke", "proposer_model": "scripted"} + {"protocol_name": "gap-surface-smoke", "model": "scripted"} ) surfaces = await SurfacePipeline( ScriptedClient(surface_responses), @@ -253,21 +164,30 @@ async def run_offline_smoke(work_root: Path) -> dict[str, Any]: proposer = ScriptedClient( { - f"{item_id}.stage1": fixture["dag"], - f"{item_id}.stage2": fixture["plan"], - f"{item_id}.stage3": fixture["replacement"], - f"{item_id}.stage4": fixture["diffused"], - f"{item_id}.stage5": fixture["candidate"], + f"{item_id}.plan": { + "core_steps": [ + "use nonnegativity of a square", + "expand and divide by a positive quantity", + ], + "mutable_slots": { + "slot1": { + "description": "the positive reference value", + "original": "1", + } + }, + }, + f"{item_id}.candidate": { + "question": "Let x>0. Prove that x+4/x >= 4.", + "solution": ( + "Since (x-2)^2 >= 0, expansion and division by x>0 " + "give x+4/x >= 4." + ), + }, } ) judges = [ ScriptedClient( - { - f"{item_id}.verify": [ - _accept_verdict(), - _accept_verdict(), - ] - } + {f"{item_id}.verify": [_review_accept(), _review_accept()]} ) for _ in range(5) ] |