diff options
Diffstat (limited to 'src/gap_pipeline/prompts.py')
| -rw-r--r-- | src/gap_pipeline/prompts.py | 513 |
1 files changed, 224 insertions, 289 deletions
diff --git a/src/gap_pipeline/prompts.py b/src/gap_pipeline/prompts.py index 667a47a..2ba87cf 100644 --- a/src/gap_pipeline/prompts.py +++ b/src/gap_pipeline/prompts.py @@ -1,338 +1,273 @@ -"""Prompts and strict JSON contracts for the GAP pipeline.""" +"""Verbatim prompts recovered from the original GAP/Putnam source. + +Do not edit prompt literals in this file. ``tests/test_prompts.py`` pins their +SHA-256 digests against the recovered source files. +""" from __future__ import annotations import json -from .models import ( - CanonicalItem, - KernelCandidate, - MethodPlan, - ProofDAG, - ReplacementSpec, -) - - -DAG_SYSTEM = """You are a mathematical proof-structure parser. -Convert a supplied reference solution into a directed acyclic graph. Each node -must be one locally checkable intermediate claim. Edges point from prerequisites -to claims that use them. Preserve every essential proof step and do not invent -new mathematics. Return JSON only.""" - - -def dag_user(item: CanonicalItem) -> str: - return f"""ITEM ID: {item.item_id} - -PROBLEM: -{item.problem} - -REFERENCE SOLUTION: -{item.solution} - -Return: -{{ - "nodes": [ - {{ - "node_id": "n1", - "claim": "concrete intermediate mathematical claim", - "derivation": "why this claim follows", - "dependencies": [], - "source_span": "corresponding reference-solution text", - "mathematical_objects": ["objects used at this node"] - }} - ], - "terminal_node_id": "node containing the requested conclusion" -}} - -Requirements: -- every dependency must refer to an earlier logical prerequisite; -- the graph must be acyclic and connected to the terminal conclusion; -- leaf nodes must expose the initial mathematical objects that could be - replaced to produce a genuinely new setting; -- retain all essential cases, guards, and endpoint checks.""" - - -METHOD_SYSTEM = """You abstract concrete proof steps into a reusable proof plan. -For every DAG node, produce a content-light method label that says what operation -or lemma is used without copying the node's particular constants or object names. -Keep exactly the DAG's topological order and node IDs. Return JSON only.""" - - -def method_user(dag: ProofDAG) -> str: - return f"""PROOF DAG: -{dag.model_dump_json(indent=2)} - -Return: -{{ - "steps": [ - {{ - "node_id": "n1", - "method_label": "content-free operation or lemma", - "input_roles": ["abstract role consumed by this step"], - "output_role": "abstract role produced by this step", - "invariants": ["conditions that must remain true"] - }} - ], - "terminal_node_id": "{dag.terminal_node_id}" -}} - -Do not merge, omit, reorder, or add nodes.""" - - -REPLACEMENT_SYSTEM = """You design one guarded replacement at a leaf of a proof -DAG. The replacement must create a genuinely different mathematical setting -while allowing the exact method-label plan to remain executable. Prefer a -replacement that does not trivialize the problem. State every guard and cite -concrete evidence from the source inequalities or assumptions. Return JSON only.""" - - -def replacement_user( - item: CanonicalItem, - dag: ProofDAG, - plan: MethodPlan, - *, - difficulty_instruction: str, -) -> str: - return f"""ORIGINAL PROBLEM: -{item.problem} - -ORIGINAL SOLUTION: -{item.solution} - -PROOF DAG: -{dag.model_dump_json(indent=2)} - -METHOD PLAN: -{plan.model_dump_json(indent=2)} - -DIFFICULTY INSTRUCTION: -{difficulty_instruction} - -Return: -{{ - "target_node_id": "a leaf node ID", - "original_object": "leaf object being replaced", - "replacement_object": "new object or setting", - "guard_conditions": ["condition required for every downstream step"], - "guard_evidence": ["source-derived evidence that the guard is satisfiable"], - "expected_downstream_changes": ["concrete changes expected during diffusion"], - "rationale": "why the same plan remains valid and the task is not trivialized" -}}""" - - -DIFFUSION_SYSTEM = """You re-instantiate a fixed method plan after one guarded -leaf replacement. Propagate the replacement through every dependent DAG node. -You must keep exactly the original node order, dependency structure, and method -labels. All algebra, cases, domains, and endpoint checks must be valid in the -new setting. Return JSON only.""" - +from .models import CanonicalItem, KernelCandidate, KernelPlan -def diffusion_user( - item: CanonicalItem, - dag: ProofDAG, - plan: MethodPlan, - replacement: ReplacementSpec, -) -> str: - return f"""ORIGINAL PROBLEM: -{item.problem} - -ORIGINAL REFERENCE SOLUTION: -{item.solution} - -PROOF DAG: -{dag.model_dump_json(indent=2)} -FIXED METHOD PLAN: -{plan.model_dump_json(indent=2)} +# Source: PutnamVariants@c3bed737370df2dbf73afd66bf6e86d4ece82d68 +# scripts/o3_kernel_variant.py +KERNEL_PLAN_SYSTEM = "You are an IMO medalist & pedagogue." +KERNEL_PLAN_PROMPT = """ +You are a competition-math expert. -GUARDED REPLACEMENT: -{replacement.model_dump_json(indent=2)} +(1) Read the Putnam problem and its official solution below. +(2) List the MINIMAL chain of lemmas / techniques essential to the solution. +(3) Identify every numerical or structural element that could be changed + *without* altering that chain of reasoning. Denote them as MUTABLE_SLOTS. -Return: +Return **one JSON object only**: {{ - "node_instantiations": [ - {{ - "node_id": "same node ID", - "method_label": "exact corresponding method label", - "instantiated_claim": "new concrete claim", - "instantiated_derivation": "valid derivation in the new setting", - "dependencies": ["same dependency IDs"] - }} - ], - "regenerated_proof": "complete rigorous proof in the new setting", - "terminal_answer": "terminal conclusion established by that proof" + "core_steps": ["..."], // 1–5 concise phrases + "mutable_slots": {{ + "slot1": {{"description": "...", "original": "..."}}, + "slot2": {{"description": "...", "original": "..."}} + }} }} -The sequence and labels must match exactly. Do not write a problem statement -yet; work forward from the replacement to a correct terminal answer.""" - - -QUESTION_SYSTEM = """You reverse-engineer a self-contained competition -mathematics problem from a fully regenerated proof and terminal answer. State -exactly the assumptions used by the proof, ask exactly for its terminal -conclusion, and do not expose the solution strategy. Return JSON only.""" +PROBLEM: +<<<{question}>>> +SOLUTION: +<<<{solution}>>> +""" -def question_user( - item: CanonicalItem, - dag: ProofDAG, - plan: MethodPlan, - replacement: ReplacementSpec, - diffused_payload: dict, -) -> str: - return f"""SOURCE PROBLEM (style reference only): -{item.problem} +KERNEL_GENERATE_SYSTEM = "You are a creative yet rigorous math professor." +KERNEL_GENERATE_PROMPT = """ +We previously extracted: -FIXED METHOD PLAN: -{plan.model_dump_json(indent=2)} +CORE_STEPS = {core} +MUTABLE_SLOTS = {slots} -REPLACEMENT: -{replacement.model_dump_json(indent=2)} +Here is the ORIGINAL problem for reference: +<<<{orig_q}>>> -REGENERATED PROOF: -{json.dumps(diffused_payload, ensure_ascii=False, indent=2)} +And its OFFICIAL solution: +<<<{orig_s}>>> -Return: +Create ONE *new* Putnam-level problem that + • still requires exactly the chain CORE_STEPS to solve, + • alters *every* MUTABLE_SLOT in a significant way. +Return JSON only: {{ - "problem": "self-contained candidate problem", - "proof": "the complete regenerated proof", - "terminal_answer": "answer requested by the candidate problem", - "node_instantiations": {json.dumps(diffused_payload.get("node_instantiations", []), ensure_ascii=False)}, - "replacement": {replacement.model_dump_json()} + "question": "...", // full statement (LaTeX-friendly) + "solution": "..." // complete proof with new data }} - -The problem must be genuinely different from the source and must neither omit -an assumption used by the proof nor add an assumption that trivializes it.""" +""" -# Four-field verifier JSON contract used by the GAP pipeline. -JUDGE_SYSTEM = """You are a verification judge for a kernel-variant generation -pipeline. You must decide whether a CANDIDATE variant problem and its CANDIDATE -proof are mathematically equivalent to the ORIGINAL problem under the given -METHOD-LABEL sequence (the abstract proof plan). +# Source: paper Appendix F.3, Listing 4. +JUDGE_SYSTEM_PROMPT = """You are a verification judge for a kernel-variant generation pipeline. You must decide whether a CANDIDATE variant problem and its CANDIDATE proof are mathematically equivalent to the ORIGINAL problem under the given METHOD-LABEL sequence (the abstract proof plan). Your job is verification, not solving. You receive: - the ORIGINAL problem statement and reference solution, -- the abstract METHOD-LABEL sequence, +- the abstract METHOD-LABEL sequence (a list of content-free steps), - the SLOT replacement that was applied, -- the CANDIDATE variant statement and regenerated proof. - -Check step by step that: -1. Every candidate step instantiates the corresponding method label. -2. The candidate proof is valid, with no unjustified leap. -3. The candidate problem is well posed and its requested terminal answer - matches the regenerated proof. -4. The candidate is genuinely different but uses the same plan. - -Return one JSON object only.""" +- the CANDIDATE variant statement and the CANDIDATE regenerated proof. +You must check, step by step, that: +1. Every CANDIDATE step instantiates the corresponding METHOD label with the new operands. +2. The CANDIDATE proof is mathematically valid: each step follows from the previous one, with no unjustified leap. +3. The CANDIDATE problem statement is well-posed and has a unique terminal answer matching the regenerated proof. +4. The CANDIDATE variant is genuinely different from the ORIGINAL (the slot replacement actually changed the instance) but uses the same plan. -def judge_user( - item: CanonicalItem, - plan: MethodPlan, - candidate: KernelCandidate, - *, - judge_id: int, - iteration: int, -) -> str: - return f"""JUDGE REPLICATE: {judge_id} -ITERATION: {iteration} +Output a single JSON object with exactly these fields.""" -ORIGINAL PROBLEM: -{item.problem} +JUDGE_USER_TEMPLATE = """ORIGINAL PROBLEM: +{original_problem} ORIGINAL REFERENCE SOLUTION: -{item.solution} +{original_solution} -METHOD-LABEL SEQUENCE: -{plan.model_dump_json(indent=2)} +METHOD-LABEL SEQUENCE (abstract plan): +{method_labels} SLOT REPLACEMENT: -{candidate.replacement.model_dump_json(indent=2)} +{slot_replacement} CANDIDATE VARIANT PROBLEM: -{candidate.problem} +{candidate_problem} CANDIDATE REGENERATED PROOF: -{candidate.proof} - -CANDIDATE NODE INSTANTIATIONS: -{json.dumps([node.model_dump() for node in candidate.node_instantiations], ensure_ascii=False, indent=2)} +{candidate_proof} Return: -{{ - "verdict": "accept or reject", - "step_by_step_check": "for every METHOD label, state whether the candidate step instantiates it correctly", - "blocking_issues": "logical gap, computation error, ill-posed statement, or plan deviation; empty if accepted", - "patch_suggestion": "minimal correction if rejected; empty if accepted" -}} - -The step-by-step check must explicitly cover every method-plan node.""" - - -REPAIR_SYSTEM = """You repair a rejected kernel candidate. Apply only the -minimal changes needed to address all blocking issues while preserving the -guarded replacement, DAG node order, dependencies, and exact method-label -sequence. Return the complete corrected candidate as JSON only.""" +{{"verdict": "accept" or "reject", + "step_by_step_check": "for each METHOD label, state whether the CANDIDATE step instantiates it correctly", + "blocking_issues": "list any logical gap, computation error, ill-posed statement, or plan deviation", + "patch_suggestion": "if reject, propose a minimal patch (a corrected proof step or a corrected slot value); leave empty if accept"}}""" + +FIX_SYSTEM_PROMPT = """You are a mathematical expert tasked with fixing kernel variant problems. +Based on the review feedback, correct the identified issues while maintaining the problem's essence. + +Guidelines: +- Fix mathematical errors while preserving the problem's structure +- Ensure the corrected version is well-posed and solvable +- Keep solutions detailed and pedagogically clear +- Maintain similar difficulty level to the original problem + +Provide COMPLETE corrected versions, not just patches.""" + +FIX_USER_TEMPLATE = """Based on the review feedback, please fix this kernel variant: + +CURRENT PROBLEM: +{kv_question} + +CURRENT SOLUTION: +{kv_solution} + +REVIEW FEEDBACK: +Problem Issues: {problem_issues} +Solution Issues: {solution_issues} + +ORIGINAL PROBLEM (for reference): +{orig_question} + +ORIGINAL SOLUTION (for reference): +{orig_solution} + +Please provide corrected versions. Return JSON with: +{{"corrected_question": "complete corrected problem statement", + "corrected_solution": "complete corrected solution",\x20 + "changes_made": "summary of key changes made"}}""" + + +# Source: PutnamVariants@c3bed737370df2dbf73afd66bf6e86d4ece82d68 +# scripts/o3_rename_vars.py +SURFACE_SYSTEM_BASE = "You are a meticulous LaTeX editor." +SURFACE_TASK_COMMON = """ +Given: + • A Putnam problem statement and its official solution (LaTeX-like); + • Two symbol lists: vars (unknowns) and params (given constants). + +Rename every symbol in *vars* and *params* with a unique English identifier: + – all-lowercase letters, ≥8 chars, no underscore/space; + – same original symbol → same new name everywhere; + – different symbols → different new names; + – NEVER touch sci_consts (\\pi,e,i,…) or numeric constants; + – do NOT alter any other text or LaTeX markup. +""" +SURFACE_TASK_DESCRIPTIVE = """ +Each new identifier **should describe the symbol's mathematical role**. +""" +SURFACE_TASK_CONFUSING = """ +Each new identifier **should *not* match the symbol's role**; choose plausible +but misleading nouns so the name sounds related but not matching the true meaning, such as replace Area with Radius. +""" +SURFACE_TASK_MISLEADING = """ +Each new identifier **should describe the *opposite* concept**. Pick names that +directly contradict the symbol's actual meaning, e.g. rename a parallel vector +to orthogonalvector. +""" +SURFACE_TASK_GARBLED = """ +Each new identifier **should look like random gibberish**: at least eight +lowercase letters with no apparent meaning such as qzxwvtnp or hjgrksla. +""" +SURFACE_RETURN_SPEC = """ +Return exactly **one JSON object** and nothing else: +{"map":{"old":"new",...},"question":"...","solution":"..."} +""" +SURFACE_USER_TEMPLATE = """Problem: +<<< +{question} +>>> + +Solution: +<<< +{solution} +>>> + +vars = {vars} +params = {params} +""" + +SURFACE_TASKS = { + "descriptive_long": SURFACE_TASK_DESCRIPTIVE, + "descriptive_long_confusing": SURFACE_TASK_CONFUSING, + "descriptive_long_misleading": SURFACE_TASK_MISLEADING, + "garbled_string": SURFACE_TASK_GARBLED, +} + + +def kernel_plan_user(item: CanonicalItem) -> str: + return KERNEL_PLAN_PROMPT.format( + question=item.problem, + solution=item.solution, + ) + + +def kernel_generate_user(item: CanonicalItem, plan: KernelPlan) -> str: + return KERNEL_GENERATE_PROMPT.format( + core=json.dumps(plan.core_steps, ensure_ascii=False), + slots=json.dumps( + { + key: value.model_dump(mode="json") + for key, value in plan.mutable_slots.items() + }, + ensure_ascii=False, + ), + orig_q=item.problem, + orig_s=item.solution, + ) -def repair_user( +def judge_user( item: CanonicalItem, - dag: ProofDAG, - plan: MethodPlan, + plan: KernelPlan, candidate: KernelCandidate, - issues: list[dict], ) -> str: - return f"""ORIGINAL PROBLEM: -{item.problem} - -PROOF DAG: -{dag.model_dump_json(indent=2)} - -FIXED METHOD PLAN: -{plan.model_dump_json(indent=2)} - -CURRENT CANDIDATE: -{candidate.model_dump_json(indent=2)} - -JUDGE FEEDBACK: -{json.dumps(issues, ensure_ascii=False, indent=2)} - -Return the complete corrected object with fields: -problem, proof, terminal_answer, node_instantiations, replacement. -Do not change the replacement unless a judge proves it violates a guard; if it -must change, retain the same target leaf and explain the corrected guards in -the replacement rationale.""" - - -SURFACE_NAME_SYSTEM = """You propose identifier replacements for a mathematical -problem. A replacement must be a single ASCII identifier beginning with a -letter, must not collide with supplied identifiers, and must fit the requested -family. Return JSON only.""" + return JUDGE_USER_TEMPLATE.format( + original_problem=item.problem, + original_solution=item.solution, + method_labels=json.dumps(plan.core_steps, ensure_ascii=False), + slot_replacement=json.dumps( + { + key: value.model_dump(mode="json") + for key, value in plan.mutable_slots.items() + }, + ensure_ascii=False, + ), + candidate_problem=candidate.question, + candidate_proof=candidate.solution, + ) -def surface_name_user( +def fix_user( + item: CanonicalItem, + candidate: KernelCandidate, *, - symbol: str, - role: str, - family: str, - context: str, - forbidden: list[str], + problem_issues: str, + solution_issues: str, ) -> str: - descriptions = { - "descriptive_long": "a semantically helpful descriptive identifier", - "descriptive_long_confusing": "2-5 unrelated common words concatenated", - "descriptive_long_misleading": ( - "mathematical jargon suggesting a different role" - ), - "garbled_string": "a 4-16 character semantically meaningless string", - } - return f"""SYMBOL: {symbol} -ROLE: {role} -FAMILY: {family} ({descriptions[family]}) -CONTEXT: -{context} -FORBIDDEN IDENTIFIERS: -{json.dumps(forbidden)} - -Return {{"replacement": "one valid identifier", "rationale": "brief reason"}}.""" + return FIX_USER_TEMPLATE.format( + kv_question=candidate.question, + kv_solution=candidate.solution, + problem_issues=problem_issues, + solution_issues=solution_issues, + orig_question=item.problem, + orig_solution=item.solution, + ) + + +def surface_system(family: str) -> str: + return ( + SURFACE_SYSTEM_BASE + + SURFACE_TASK_COMMON + + SURFACE_TASKS[family] + + SURFACE_RETURN_SPEC + ) + + +def surface_user(item: CanonicalItem) -> str: + return SURFACE_USER_TEMPLATE.format( + question=item.problem, + solution=item.solution, + vars=item.variables, + params=item.parameters, + ) |