From d4eb26780a8a8c70ca75812af0c3c6a295c0797c Mon Sep 17 00:00:00 2001 From: Anonymous Authors Date: Fri, 24 Jul 2026 13:52:11 -0500 Subject: Restore original GAP prompts and lock prompt bytes --- tests/conftest.py | 103 +++++------------------------ tests/test_models.py | 81 +++++++---------------- tests/test_pipeline.py | 176 ++++++++++++++++++++----------------------------- tests/test_prompts.py | 40 +++++++++++ tests/test_release.py | 2 +- tests/test_surface.py | 79 +++++++++++++--------- 6 files changed, 199 insertions(+), 282 deletions(-) create mode 100644 tests/test_prompts.py (limited to 'tests') diff --git a/tests/conftest.py b/tests/conftest.py index 80ccc2e..ab3003b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -21,106 +21,35 @@ def item() -> CanonicalItem: ) -@pytest.fixture -def dag_dict() -> dict: - return { - "nodes": [ - { - "node_id": "n1", - "claim": "(a-1)^2 >= 0", - "derivation": "a square is nonnegative", - "dependencies": [], - "source_span": "Since (a-1)^2 >= 0", - "mathematical_objects": ["a", "(a-1)^2"], - }, - { - "node_id": "n2", - "claim": "a+1/a >= 2", - "derivation": "expand and divide by positive a", - "dependencies": ["n1"], - "source_span": "Divide by a>0", - "mathematical_objects": ["a"], - }, - ], - "terminal_node_id": "n2", - } - - @pytest.fixture def plan_dict() -> dict: return { - "steps": [ - { - "node_id": "n1", - "method_label": "use nonnegativity of a square", - "input_roles": ["positive scalar"], - "output_role": "nonnegative expression", - "invariants": ["scalar is real"], - }, - { - "node_id": "n2", - "method_label": "expand and divide by a positive quantity", - "input_roles": ["nonnegative expression", "positive scalar"], - "output_role": "target inequality", - "invariants": ["divisor is positive"], - }, + "core_steps": [ + "use nonnegativity of a square", + "expand and divide by a positive quantity", ], - "terminal_node_id": "n2", + "mutable_slots": { + "slot1": { + "description": "the positive reference value", + "original": "1", + } + }, } @pytest.fixture -def replacement_dict() -> dict: +def candidate_dict() -> dict: return { - "target_node_id": "n1", - "original_object": "(a-1)^2", - "replacement_object": "(x-2)^2", - "guard_conditions": ["x>0"], - "guard_evidence": ["choose x in the positive reals"], - "expected_downstream_changes": ["expand around 2", "divide by x"], - "rationale": "the same square-nonnegativity plan applies", - } - - -@pytest.fixture -def diffused_dict(plan_dict: dict) -> dict: - return { - "node_instantiations": [ - { - "node_id": "n1", - "method_label": plan_dict["steps"][0]["method_label"], - "instantiated_claim": "(x-2)^2 >= 0", - "instantiated_derivation": "a square is nonnegative", - "dependencies": [], - }, - { - "node_id": "n2", - "method_label": plan_dict["steps"][1]["method_label"], - "instantiated_claim": "x+4/x >= 4", - "instantiated_derivation": "expand and divide by positive x", - "dependencies": ["n1"], - }, - ], - "regenerated_proof": ( - "Since (x-2)^2 >= 0, expand and divide by x>0 to get x+4/x >= 4." + "question": "Let x>0. Prove that x+4/x >= 4.", + "solution": ( + "Since (x-2)^2 >= 0, expand and divide by x>0 " + "to get x+4/x >= 4." ), - "terminal_answer": "x+4/x >= 4", - } - - -@pytest.fixture -def candidate_dict(replacement_dict: dict, diffused_dict: dict) -> dict: - return { - "problem": "Let x>0. Prove that x+4/x >= 4.", - "proof": diffused_dict["regenerated_proof"], - "terminal_answer": diffused_dict["terminal_answer"], - "node_instantiations": diffused_dict["node_instantiations"], - "replacement": replacement_dict, } def changed_candidate(candidate: dict, suffix: str) -> dict: value = copy.deepcopy(candidate) - value["problem"] = value["problem"] + f" [{suffix}]" - value["proof"] = value["proof"] + f" Repair {suffix}." + value["question"] = value["question"] + f" [{suffix}]" + value["solution"] = value["solution"] + f" Repair {suffix}." return value diff --git a/tests/test_models.py b/tests/test_models.py index 2c678a0..71dc854 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -3,69 +3,38 @@ from __future__ import annotations import pytest from pydantic import ValidationError -from gap_pipeline.models import ( - DiffusedProof, - MethodPlan, - ProofDAG, - ReplacementSpec, -) +from gap_pipeline.models import JudgeVerdict, KernelCandidate, KernelPlan -def test_valid_dag_and_plan(dag_dict: dict, plan_dict: dict) -> None: - dag = ProofDAG.model_validate(dag_dict) - assert [node.node_id for node in dag.topological_nodes()] == ["n1", "n2"] - assert dag.leaf_node_ids() == ["n1"] - plan = MethodPlan.model_validate(plan_dict) - plan.validate_against(dag) +def test_valid_plan_and_candidate(plan_dict: dict, candidate_dict: dict) -> None: + plan = KernelPlan.model_validate(plan_dict) + candidate = KernelCandidate.model_validate(candidate_dict) + assert len(plan.core_steps) == 2 + assert candidate.question.startswith("Let x") -def test_cycle_is_rejected(dag_dict: dict) -> None: - dag_dict["nodes"][0]["dependencies"] = ["n2"] - with pytest.raises(ValidationError, match="cycle"): - ProofDAG.model_validate(dag_dict) +def test_plan_requires_slots(plan_dict: dict) -> None: + plan_dict["mutable_slots"] = {} + with pytest.raises(ValidationError, match="mutable slot"): + KernelPlan.model_validate(plan_dict) -def test_node_disconnected_from_terminal_is_rejected(dag_dict: dict) -> None: - dag_dict["nodes"].append( - { - "node_id": "orphan", - "claim": "unused claim", - "derivation": "unused derivation", - "dependencies": [], - "source_span": "", - "mathematical_objects": [], - } - ) - with pytest.raises(ValidationError, match="disconnected"): - ProofDAG.model_validate(dag_dict) +def test_plan_rejects_more_than_five_core_steps(plan_dict: dict) -> None: + plan_dict["core_steps"] = [f"step {index}" for index in range(6)] + with pytest.raises(ValidationError): + KernelPlan.model_validate(plan_dict) -def test_method_reordering_is_rejected(dag_dict: dict, plan_dict: dict) -> None: - dag = ProofDAG.model_validate(dag_dict) - plan_dict["steps"].reverse() - plan = MethodPlan.model_validate(plan_dict) - with pytest.raises(ValueError, match="topological order"): - plan.validate_against(dag) +def test_candidate_requires_question_and_solution(candidate_dict: dict) -> None: + candidate_dict["solution"] = "" + with pytest.raises(ValidationError, match="non-empty"): + KernelCandidate.model_validate(candidate_dict) -def test_replacement_must_target_leaf( - dag_dict: dict, replacement_dict: dict -) -> None: - dag = ProofDAG.model_validate(dag_dict) - replacement_dict["target_node_id"] = "n2" - replacement = ReplacementSpec.model_validate(replacement_dict) - with pytest.raises(ValueError, match="not a DAG leaf"): - replacement.validate_against(dag) - - -def test_diffusion_cannot_change_dag_dependencies( - dag_dict: dict, - plan_dict: dict, - diffused_dict: dict, -) -> None: - dag = ProofDAG.model_validate(dag_dict) - plan = MethodPlan.model_validate(plan_dict) - diffused_dict["node_instantiations"][1]["dependencies"] = [] - diffused = DiffusedProof.model_validate(diffused_dict) - with pytest.raises(ValueError, match="changes dependencies"): - diffused.validate_against(dag, plan) +def test_accept_verdict_cannot_report_blocking_issue() -> None: + with pytest.raises(ValidationError, match="cannot contain"): + JudgeVerdict( + verdict="accept", + step_by_step_check="checked", + blocking_issues="an error", + ) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index ea1077a..864e967 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -10,62 +10,56 @@ from gap_pipeline.store import RunStore from conftest import changed_candidate -def accept_verdict() -> dict: +def accept_review() -> dict[str, str]: return { "verdict": "accept", - "step_by_step_check": "n1 passes; n2 passes", + "step_by_step_check": "every method label is instantiated", "blocking_issues": "", "patch_suggestion": "", } -def reject_verdict(iteration: int) -> dict: +def reject_review(iteration: int) -> dict[str, str]: return { "verdict": "reject", - "step_by_step_check": f"n1 passes; n2 fails at iteration {iteration}", - "blocking_issues": "the terminal algebra needs correction", - "patch_suggestion": "correct the terminal algebra without changing the plan", + "step_by_step_check": f"terminal step fails at iteration {iteration}", + "blocking_issues": "terminal algebra is incorrect", + "patch_suggestion": "correct the terminal algebra", } def proposer_responses( item_id: str, - dag_dict: dict, plan_dict: dict, - replacement_dict: dict, - diffused_dict: dict, candidate_dict: dict, ) -> dict: return { - f"{item_id}.stage1": dag_dict, - f"{item_id}.stage2": plan_dict, - f"{item_id}.stage3": replacement_dict, - f"{item_id}.stage4": diffused_dict, - f"{item_id}.stage5": candidate_dict, + f"{item_id}.plan": plan_dict, + f"{item_id}.candidate": candidate_dict, + } + + +def repair_response(candidate: dict) -> dict[str, str]: + return { + "corrected_question": candidate["question"], + "corrected_solution": candidate["solution"], + "changes_made": "corrected the terminal algebra", } def test_two_consecutive_unanimous_passes_accept( tmp_path, item, - dag_dict, plan_dict, - replacement_dict, - diffused_dict, candidate_dict, ) -> None: proposer = ScriptedClient( - proposer_responses( - item.item_id, - dag_dict, - plan_dict, - replacement_dict, - diffused_dict, - candidate_dict, - ) + proposer_responses(item.item_id, plan_dict, candidate_dict) ) judges = [ - ScriptedClient({f"{item.item_id}.verify": [accept_verdict(), accept_verdict()]}) + ScriptedClient( + {f"{item.item_id}.verify": [accept_review(), accept_review()]} + ) for _ in range(5) ] pipeline = KernelPipeline( @@ -78,138 +72,105 @@ def test_two_consecutive_unanimous_passes_accept( assert result.status == "accepted" assert len(result.iterations) == 2 assert [row.pass_streak_after for row in result.iterations] == [1, 2] - assert not any(row.repaired for row in result.iterations) final_path = tmp_path / "items" / item.item_id / "final.json" final = json.loads(final_path.read_text()) - assert final["human_audit_selected"] in {True, False} - assert len(list((final_path.parent / "calls").glob("*.json"))) == 15 + assert final["accepted_candidate_sha256"] + assert len(list((final_path.parent / "calls").glob("*.json"))) == 12 def test_rejection_resets_streak_and_repairs( tmp_path, item, - dag_dict, plan_dict, - replacement_dict, - diffused_dict, candidate_dict, ) -> None: repaired = changed_candidate(candidate_dict, "fixed") - responses = proposer_responses( - item.item_id, - dag_dict, - plan_dict, - replacement_dict, - diffused_dict, - candidate_dict, - ) - responses[f"{item.item_id}.repair"] = repaired + responses = proposer_responses(item.item_id, plan_dict, candidate_dict) + responses[f"{item.item_id}.repair"] = repair_response(repaired) proposer = ScriptedClient(responses) judges = [] for judge_id in range(1, 6): - first = reject_verdict(1) if judge_id == 1 else accept_verdict() + first = reject_review(1) if judge_id == 1 else accept_review() judges.append( ScriptedClient( { f"{item.item_id}.verify": [ first, - accept_verdict(), - accept_verdict(), + accept_review(), + accept_review(), ] } ) ) - pipeline = KernelPipeline( - proposer=proposer, - judges=judges, - store=RunStore(tmp_path, item.item_id), - config=PipelineConfig(proposer_model="scripted", judge_model="scripted"), + result = asyncio.run( + KernelPipeline( + proposer=proposer, + judges=judges, + store=RunStore(tmp_path, item.item_id), + config=PipelineConfig( + proposer_model="scripted", + judge_model="scripted", + ), + ).run(item) ) - result = asyncio.run(pipeline.run(item)) assert result.status == "accepted" - assert len(result.iterations) == 3 - assert result.iterations[0].repaired assert [row.pass_streak_after for row in result.iterations] == [0, 1, 2] - assert ( - result.iterations[0].repaired_candidate_sha256 - == result.iterations[1].candidate_sha256 - == result.iterations[2].candidate_sha256 - ) + assert result.iterations[0].candidate_sha256 != result.iterations[1].candidate_sha256 + assert result.iterations[1].candidate_sha256 == result.iterations[2].candidate_sha256 -def test_rejection_after_one_pass_resets_streak_on_new_candidate( +def test_rejection_after_one_pass_resets_streak( tmp_path, item, - dag_dict, plan_dict, - replacement_dict, - diffused_dict, candidate_dict, ) -> None: repaired = changed_candidate(candidate_dict, "after-broken-streak") - responses = proposer_responses( - item.item_id, - dag_dict, - plan_dict, - replacement_dict, - diffused_dict, - candidate_dict, - ) - responses[f"{item.item_id}.repair"] = repaired + responses = proposer_responses(item.item_id, plan_dict, candidate_dict) + responses[f"{item.item_id}.repair"] = repair_response(repaired) proposer = ScriptedClient(responses) judges = [] for judge_id in range(1, 6): - second = reject_verdict(2) if judge_id == 3 else accept_verdict() + second = reject_review(2) if judge_id == 3 else accept_review() judges.append( ScriptedClient( { f"{item.item_id}.verify": [ - accept_verdict(), + accept_review(), second, - accept_verdict(), - accept_verdict(), + accept_review(), + accept_review(), ] } ) ) - pipeline = KernelPipeline( - proposer=proposer, - judges=judges, - store=RunStore(tmp_path, item.item_id), - config=PipelineConfig(proposer_model="scripted", judge_model="scripted"), + result = asyncio.run( + KernelPipeline( + proposer=proposer, + judges=judges, + store=RunStore(tmp_path, item.item_id), + config=PipelineConfig( + proposer_model="scripted", + judge_model="scripted", + ), + ).run(item) ) - result = asyncio.run(pipeline.run(item)) assert result.status == "accepted" assert [row.pass_streak_after for row in result.iterations] == [1, 0, 1, 2] assert result.iterations[0].candidate_sha256 == result.iterations[1].candidate_sha256 - assert result.iterations[1].repaired_candidate_sha256 != result.iterations[1].candidate_sha256 - assert ( - result.iterations[1].repaired_candidate_sha256 - == result.iterations[2].candidate_sha256 - == result.iterations[3].candidate_sha256 - ) + assert result.iterations[1].candidate_sha256 != result.iterations[2].candidate_sha256 def test_fifteen_failed_rounds_reject( tmp_path, item, - dag_dict, plan_dict, - replacement_dict, - diffused_dict, candidate_dict, ) -> None: - responses = proposer_responses( - item.item_id, - dag_dict, - plan_dict, - replacement_dict, - diffused_dict, - candidate_dict, - ) + responses = proposer_responses(item.item_id, plan_dict, candidate_dict) responses[f"{item.item_id}.repair"] = [ - changed_candidate(candidate_dict, f"repair-{iteration}") + repair_response(changed_candidate(candidate_dict, f"repair-{iteration}")) for iteration in range(1, 15) ] proposer = ScriptedClient(responses) @@ -217,20 +178,23 @@ def test_fifteen_failed_rounds_reject( ScriptedClient( { f"{item.item_id}.verify": [ - reject_verdict(iteration) for iteration in range(1, 16) + reject_review(iteration) for iteration in range(1, 16) ] } ) for _ in range(5) ] - pipeline = KernelPipeline( - proposer=proposer, - judges=judges, - store=RunStore(tmp_path, item.item_id), - config=PipelineConfig(proposer_model="scripted", judge_model="scripted"), + result = asyncio.run( + KernelPipeline( + proposer=proposer, + judges=judges, + store=RunStore(tmp_path, item.item_id), + config=PipelineConfig( + proposer_model="scripted", + judge_model="scripted", + ), + ).run(item) ) - result = asyncio.run(pipeline.run(item)) assert result.status == "rejected" assert len(result.iterations) == 15 - assert sum(row.repaired for row in result.iterations) == 14 assert all(row.pass_streak_after == 0 for row in result.iterations) diff --git a/tests/test_prompts.py b/tests/test_prompts.py new file mode 100644 index 0000000..a2bf892 --- /dev/null +++ b/tests/test_prompts.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +import hashlib +import inspect + +from gap_pipeline import prompts +from gap_pipeline.clients import OpenAIJsonClient + + +EXPECTED = { + "KERNEL_PLAN_SYSTEM": "0c817ede22fd407411ee588a02cf7a14ddcbb3d607aa4db6b5f81619609bcfef", + "KERNEL_PLAN_PROMPT": "4aafd1c25d67c5a433c8ad48dcf347cfe53436e81cfde1883e044ca3c17d855d", + "KERNEL_GENERATE_SYSTEM": "db0c902e15c9830a3c3860509bdb8a17b412d6b646b0f7e3278f32669046212b", + "KERNEL_GENERATE_PROMPT": "89c8a05dcda1b866b7835167c41184de9176ea058922e2cb250325825f79db92", + "JUDGE_SYSTEM_PROMPT": "50d5ee3ccbdf5196afea1eea60e61e74a324c89f14086b0ec6a8e14cb4ad0f30", + "JUDGE_USER_TEMPLATE": "59338f0a2522bda73006e7f62243578596cb8113da90fd8bfe5767d5a2bf4c2b", + "FIX_SYSTEM_PROMPT": "1c028afeac4bfbf92dd56cfce7b004b0823ab721f6f71e776bb90a5c652c1ed8", + "FIX_USER_TEMPLATE": "f12126acdc1518e872cd61f6baf2eedd49f1ad85eca69438e78128293959a8e9", + "SURFACE_SYSTEM_BASE": "d05ab24b4afd4ccd23e7954fa1111ce9c74893f8572c62617a3e568e3a743994", + "SURFACE_TASK_COMMON": "1cca2d06215207aa85ae36b368298a0b13ad4c2ef475079f1df0b2c4d9a5edd1", + "SURFACE_TASK_DESCRIPTIVE": "06b9b9c6a65ceae6818f30387cfd6e18bb46bbc1ccdf79adbba73f8849f6b354", + "SURFACE_TASK_CONFUSING": "7df3748ab186bbb6d9f421de5a7200754105154036d5cf6fb21ee32ac1f61848", + "SURFACE_TASK_MISLEADING": "fcde08ded42df91b44cb403f9a439550077245d71586adb6da9fb4983a4ea76e", + "SURFACE_TASK_GARBLED": "576301ca8e9eff2e4a2d9ed6a74f884a5a881ba22fe3a38e22e5eb46e341fac0", + "SURFACE_RETURN_SPEC": "b9f33abc9c8ab5e30b0154a6eebbd139d91afea6a744b454d4e150a6f7224a26", + "SURFACE_USER_TEMPLATE": "5d9043d7d1c1033db1aea0696812a2c4cbb6f3e28ca9d7b6436c036b40831fe7", +} + + +def test_prompt_values_are_byte_locked() -> None: + actual = { + name: hashlib.sha256(getattr(prompts, name).encode("utf-8")).hexdigest() + for name in EXPECTED + } + assert actual == EXPECTED + + +def test_o3_adapter_does_not_send_temperature() -> None: + source = inspect.getsource(OpenAIJsonClient.generate_json) + assert '"temperature"' not in source diff --git a/tests/test_release.py b/tests/test_release.py index 3653ff1..b825478 100644 --- a/tests/test_release.py +++ b/tests/test_release.py @@ -70,7 +70,7 @@ def test_offline_release_export_verifies_and_assembles( (output_root / "records" / f"{item.item_id}.json").read_text() ) assert set(output["variants"]) == {*SURFACE_FAMILIES, "kernel_variant"} - assert output["variants"]["kernel_variant"]["question"] == candidate.problem + assert output["variants"]["kernel_variant"]["question"] == candidate.question with pytest.raises(FileExistsError): export_release( source_dataset=source_dir, diff --git a/tests/test_surface.py b/tests/test_surface.py index 78c4d5e..4d92de3 100644 --- a/tests/test_surface.py +++ b/tests/test_surface.py @@ -1,43 +1,58 @@ from __future__ import annotations +import asyncio + import pytest -from gap_pipeline.surface import ( - apply_rename_map, - create_surface_variant, - validate_rename_map, -) +from gap_pipeline.clients import ScriptedClient +from gap_pipeline.store import RunStore +from gap_pipeline.surface import SurfacePipeline, validate_surface_variant -def test_math_only_rename_preserves_english_article(item) -> None: - text = r"Let \(a\) be a randomly chosen value with \(a>0\)." - renamed = apply_rename_map(text, {"a": "coefalpha"}) - assert renamed == ( - r"Let \(coefalpha\) be a randomly chosen value with \(coefalpha>0\)." +def test_surface_pipeline_uses_full_original_contract(tmp_path, item) -> None: + response = { + "map": {"a": "positivequantity"}, + "question": r"Let \(positivequantity>0\). Prove the renamed inequality.", + "solution": r"Apply the same proof to \(positivequantity\).", + } + client = ScriptedClient( + {f"{item.item_id}.surface.descriptive_long": response} ) - - -def test_surface_variant_changes_problem_and_solution(item) -> None: - variant = create_surface_variant( - item, - "descriptive_long", - {"a": "positiveScalar"}, + variant = asyncio.run( + SurfacePipeline( + client, + RunStore(tmp_path, item.item_id), + ).run_family(item, "descriptive_long") ) - assert "positiveScalar" in variant.problem - assert "positiveScalar" in variant.solution - assert "Let positiveScalar" not in variant.problem - - -def test_collision_and_nonbijection_are_rejected(item) -> None: - with pytest.raises(ValueError, match="collides"): - validate_rename_map( - {"a": "a"}, - existing_identifiers=["a"], - scientific_constants=[], + assert variant.rename_map == {"a": "positivequantity"} + assert variant.question == response["question"] + assert variant.solution == response["solution"] + + +def test_missing_symbol_and_nonbijection_are_rejected(item) -> None: + with pytest.raises(ValueError, match="every var and param"): + validate_surface_variant( + item, + rename_map={}, + question="question", + solution="solution", ) + + item.variables.append("b") with pytest.raises(ValueError, match="one-to-one"): - validate_rename_map( - {"a": "sharedName", "b": "sharedName"}, - existing_identifiers=["a", "b"], - scientific_constants=[], + validate_surface_variant( + item, + rename_map={"a": "sharedname", "b": "sharedname"}, + question="question", + solution="solution", + ) + + +def test_original_identifier_contract_is_enforced(item) -> None: + with pytest.raises(ValueError, match="original prompt contract"): + validate_surface_variant( + item, + rename_map={"a": "TooShort"}, + question="question", + solution="solution", ) -- cgit v1.2.3