From 15efc30e9e7179accd30375d3edb2e34a3b4dc5f Mon Sep 17 00:00:00 2001 From: Oscar Wan Date: Fri, 24 Jul 2026 20:45:42 -0700 Subject: updated generation process --- tests/test_e2e.py | 4 ++ tests/test_kernel_models.py | 138 ++++++++++++++++++++++++++++++++++++++++++++ tests/test_models.py | 20 +++++++ tests/test_offline.py | 68 +++++++++++++++++++++- tests/test_prompts.py | 73 +++++++++++++++++++++++ tests/test_release.py | 47 +++++++++++++-- tests/test_surface.py | 37 +++++++++++- 7 files changed, 379 insertions(+), 8 deletions(-) create mode 100644 tests/test_kernel_models.py (limited to 'tests') diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 9da1190..9672527 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -28,3 +28,7 @@ def test_offline_end_to_end_smoke(tmp_path: Path) -> None: "garbled_string", "kernel_variant", } + kernel_meta = record["variants"]["kernel_variant"]["_meta"] + assert kernel_meta["proof_dag"]["terminal_node_id"] == "n3" + assert kernel_meta["replacement_plan"]["changes"][0]["slot_id"] == "s1" + assert kernel_meta["diffused_proof"]["terminal_answer"] == "4" diff --git a/tests/test_kernel_models.py b/tests/test_kernel_models.py new file mode 100644 index 0000000..864d6d5 --- /dev/null +++ b/tests/test_kernel_models.py @@ -0,0 +1,138 @@ +from __future__ import annotations + +import pytest + +from gap_pipeline.kernel_models import ( + DiffusedProof, + JudgeVerdict, + MethodPlan, + ProofDAG, + RenderedVariant, + ReplacementPlan, +) + + +def branched_dag() -> ProofDAG: + return ProofDAG.model_validate( + { + "nodes": [ + {"node_id": "n1", "claim": "first fact", "dependencies": []}, + {"node_id": "n2", "claim": "left branch", "dependencies": ["n1"]}, + {"node_id": "n3", "claim": "right branch", "dependencies": ["n1"]}, + { + "node_id": "n4", + "claim": "combine branches", + "dependencies": ["n2", "n3"], + }, + ], + "terminal_node_id": "n4", + } + ) + + +def test_proof_dag_supports_real_branching() -> None: + dag = branched_dag() + assert dag.nodes[-1].dependencies == ["n2", "n3"] + + +def test_replacement_must_reference_a_real_dag_node() -> None: + replacements = ReplacementPlan.model_validate( + { + "changes": [ + { + "slot_id": "s1", + "source_node_id": "n9", + "description": "constant", + "original_value": "1", + "replacement_value": "2", + "guard_condition": "positive", + "guard_justification": "2 is positive", + } + ], + "closure_statement": "No undeclared changes.", + } + ) + with pytest.raises(ValueError, match="unknown nodes"): + replacements.validate_against(branched_dag()) + + +def test_diffusion_preserves_dependencies_and_methods() -> None: + dag = branched_dag() + methods = MethodPlan.model_validate( + { + "nodes": [ + {"node_id": node.node_id, "method_label": f"method {node.node_id}"} + for node in dag.nodes + ] + } + ) + diffused = DiffusedProof.model_validate( + { + "nodes": [ + { + "node_id": node.node_id, + "dependencies": node.dependencies, + "method_label": f"method {node.node_id}", + "instantiated_claim": f"new {node.claim}", + "justification": "valid re-instantiation", + } + for node in dag.nodes + ], + "terminal_node_id": "n4", + "terminal_answer": "answer", + } + ) + assert diffused.validate_against(dag, methods) is diffused + + broken = diffused.model_copy(deep=True) + broken.nodes[-1].dependencies = ["n3"] + with pytest.raises(ValueError, match="dependencies changed"): + broken.validate_against(dag, methods) + + +def test_rendered_solution_must_expose_every_node() -> None: + dag = branched_dag() + methods = MethodPlan.model_validate( + { + "nodes": [ + {"node_id": node.node_id, "method_label": f"method {node.node_id}"} + for node in dag.nodes + ] + } + ) + diffused = DiffusedProof.model_validate( + { + "nodes": [ + { + "node_id": node.node_id, + "dependencies": node.dependencies, + "method_label": f"method {node.node_id}", + "instantiated_claim": f"new {node.claim}", + "justification": "valid", + } + for node in dag.nodes + ], + "terminal_node_id": "n4", + "terminal_answer": "answer", + } + ).validate_against(dag, methods) + variant = RenderedVariant( + question="New problem", + solution="[n1] first [n2] left [n3] right", + node_order=dag.node_ids(), + terminal_answer="answer", + ) + with pytest.raises(ValueError, match="missing node markers.*n4"): + variant.validate_against(dag, diffused) + + +def test_accept_verdict_normalizes_explicit_none_sentinel() -> None: + verdict = JudgeVerdict( + verdict="accept", + step_by_step_check="n1 valid", + replacement_check="s1 valid", + blocking_issues="None detected.", + patch_suggestion="N/A", + ) + assert verdict.blocking_issues == "" + assert verdict.patch_suggestion == "" diff --git a/tests/test_models.py b/tests/test_models.py index d19b536..b329ecd 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -111,6 +111,26 @@ def test_judge_verdict_requires_every_dag_node(plan_dict: dict) -> None: ).validate_coverage(dag) +def test_judge_verdict_normalizes_structured_text_fields(plan_dict: dict) -> None: + dag = ProofPlanDAG.from_plan(KernelPlan.model_validate(plan_dict)) + verdict = JudgeVerdict.model_validate( + { + "verdict": "accept", + "step_by_step_check": { + "n1": "correctly instantiated", + "n2": "correctly instantiated", + }, + "blocking_issues": None, + "patch_suggestion": None, + } + ) + + assert '"n1"' in verdict.step_by_step_check + assert verdict.blocking_issues == "" + assert verdict.patch_suggestion == "" + assert verdict.validate_coverage(dag) is verdict + + def test_dag_labels_must_match_prompt_a_plan(plan_dict: dict) -> None: plan = KernelPlan.model_validate(plan_dict) dag = ProofPlanDAG.from_plan(plan) diff --git a/tests/test_offline.py b/tests/test_offline.py index 04b7f46..9851826 100644 --- a/tests/test_offline.py +++ b/tests/test_offline.py @@ -1,6 +1,72 @@ -from gap_pipeline.offline import normalize_latex_symbol +import json + +from gap_pipeline.offline import ( + align_run_to_release, + normalize_latex_symbol, + summarize_run, +) def test_latex_symbol_aliases_normalize_together() -> None: assert normalize_latex_symbol(r"x_{n}") == normalize_latex_symbol("x_n") assert normalize_latex_symbol(r"\\phi") == normalize_latex_symbol(r"\phi") + + +def test_run_summary_counts_acceptance_after_repair(tmp_path) -> None: + item_dir = tmp_path / "items" / "demo" + item_dir.mkdir(parents=True) + (item_dir / "final.json").write_text( + json.dumps( + { + "item_id": "demo", + "status": "accepted", + "iterations": [ + {"unanimous": False}, + {"unanimous": True}, + {"unanimous": True}, + ], + } + ), + encoding="utf-8", + ) + + summary = summarize_run(tmp_path) + assert summary["accepted_after_repair"] == 1 + + +def test_align_release_uses_current_candidate_schema(tmp_path) -> None: + run_dir = tmp_path / "run" + item_dir = run_dir / "items" / "demo" + item_dir.mkdir(parents=True) + (item_dir / "final.json").write_text( + json.dumps( + { + "item_id": "demo", + "status": "accepted", + "accepted_candidate": { + "question": "New question", + "solution": "New solution", + }, + } + ), + encoding="utf-8", + ) + dataset_dir = tmp_path / "dataset" + dataset_dir.mkdir() + (dataset_dir / "demo.json").write_text( + json.dumps( + { + "index": "demo", + "variants": { + "kernel_variant": { + "question": "New question", + "solution": "New solution", + } + }, + } + ), + encoding="utf-8", + ) + + alignment = align_run_to_release(run_dir, dataset_dir) + assert alignment["status_counts"] == {"exact_match": 1} diff --git a/tests/test_prompts.py b/tests/test_prompts.py index a2bf892..4f6568b 100644 --- a/tests/test_prompts.py +++ b/tests/test_prompts.py @@ -5,6 +5,20 @@ import inspect from gap_pipeline import prompts from gap_pipeline.clients import OpenAIJsonClient +from gap_pipeline.kernel_models import ( + DiffusedProof, + MethodPlan, + ProofDAG, + ReplacementPlan, +) +from gap_pipeline.kernel_prompts import ( + dag_user, + diffusion_user, + judge_user, + method_user, + render_user, + replacement_user, +) EXPECTED = { @@ -38,3 +52,62 @@ def test_prompt_values_are_byte_locked() -> None: def test_o3_adapter_does_not_send_temperature() -> None: source = inspect.getsource(OpenAIJsonClient.generate_json) assert '"temperature"' not in source + + +def test_literal_five_stage_prompts_render(item) -> None: + dag = ProofDAG.model_validate( + { + "nodes": [{"node_id": "n1", "claim": "claim", "dependencies": []}], + "terminal_node_id": "n1", + } + ) + methods = MethodPlan.model_validate( + {"nodes": [{"node_id": "n1", "method_label": "method"}]} + ) + replacements = ReplacementPlan.model_validate( + { + "changes": [ + { + "slot_id": "s1", + "source_node_id": "n1", + "description": "constant", + "original_value": "1", + "replacement_value": "2", + "guard_condition": "positive", + "guard_justification": "2 is positive", + } + ], + "closure_statement": "No undeclared changes.", + } + ) + diffused = DiffusedProof.model_validate( + { + "nodes": [ + { + "node_id": "n1", + "dependencies": [], + "method_label": "method", + "instantiated_claim": "new claim", + "justification": "valid", + } + ], + "terminal_node_id": "n1", + "terminal_answer": "answer", + } + ) + variant = { + "question": "question", + "solution": "[n1] solution", + "node_order": ["n1"], + "terminal_answer": "answer", + } + + rendered = [ + dag_user(item), + method_user(dag), + replacement_user(item, dag, methods), + diffusion_user(item, dag, methods, replacements), + render_user(replacements, diffused), + judge_user(item, dag, methods, replacements, diffused, variant), + ] + assert all("{" in value and "}" in value for value in rendered) diff --git a/tests/test_release.py b/tests/test_release.py index b825478..8927b14 100644 --- a/tests/test_release.py +++ b/tests/test_release.py @@ -4,7 +4,6 @@ import json import pytest -from gap_pipeline.models import KernelCandidate from gap_pipeline.release import export_release from gap_pipeline.store import RunStore, sha256_payload from gap_pipeline.surface import SURFACE_FAMILIES @@ -47,13 +46,52 @@ def test_offline_release_export_verifies_and_assembles( kernel_root = tmp_path / "kernel-runs" kernel_store = RunStore(kernel_root, item.item_id) - candidate = KernelCandidate.model_validate(candidate_dict) + candidate = { + **candidate_dict, + "node_order": ["n1"], + "terminal_answer": "4", + } kernel_store.write_final( { "item_id": item.item_id, "status": "accepted", - "accepted_candidate": candidate.model_dump(mode="json"), + "proof_dag": { + "nodes": [{"node_id": "n1", "claim": "claim", "dependencies": []}], + "terminal_node_id": "n1", + }, + "method_plan": { + "nodes": [{"node_id": "n1", "method_label": "method"}] + }, + "accepted_replacement_plan": { + "changes": [ + { + "slot_id": "s1", + "source_node_id": "n1", + "description": "value", + "original_value": "1", + "replacement_value": "2", + "guard_condition": "positive", + "guard_justification": "2 is positive", + } + ], + "closure_statement": "closed", + }, + "accepted_diffused_proof": { + "nodes": [ + { + "node_id": "n1", + "dependencies": [], + "method_label": "method", + "instantiated_claim": "claim", + "justification": "reason", + } + ], + "terminal_node_id": "n1", + "terminal_answer": "4", + }, + "accepted_candidate": candidate, "accepted_candidate_sha256": sha256_payload(candidate), + "accepted_bundle_sha256": "bundle-sha", } ) @@ -70,7 +108,8 @@ def test_offline_release_export_verifies_and_assembles( (output_root / "records" / f"{item.item_id}.json").read_text() ) assert set(output["variants"]) == {*SURFACE_FAMILIES, "kernel_variant"} - assert output["variants"]["kernel_variant"]["question"] == candidate.question + assert output["variants"]["kernel_variant"]["question"] == candidate["question"] + assert output["variants"]["kernel_variant"]["_meta"]["replacement_plan"] with pytest.raises(FileExistsError): export_release( source_dataset=source_dir, diff --git a/tests/test_surface.py b/tests/test_surface.py index 4d92de3..5f38d57 100644 --- a/tests/test_surface.py +++ b/tests/test_surface.py @@ -6,7 +6,11 @@ import pytest from gap_pipeline.clients import ScriptedClient from gap_pipeline.store import RunStore -from gap_pipeline.surface import SurfacePipeline, validate_surface_variant +from gap_pipeline.surface import ( + SurfacePipeline, + apply_rename_map, + validate_surface_variant, +) def test_surface_pipeline_uses_full_original_contract(tmp_path, item) -> None: @@ -25,8 +29,15 @@ def test_surface_pipeline_uses_full_original_contract(tmp_path, item) -> None: ).run_family(item, "descriptive_long") ) assert variant.rename_map == {"a": "positivequantity"} - assert variant.question == response["question"] - assert variant.solution == response["solution"] + assert variant.question == apply_rename_map( + item.problem, {"a": "positivequantity"} + ) + assert variant.solution == apply_rename_map( + item.solution, {"a": "positivequantity"} + ) + assert "that" in variant.question + assert "expand" in variant.solution + assert "obtain" in variant.solution def test_missing_symbol_and_nonbijection_are_rejected(item) -> None: @@ -56,3 +67,23 @@ def test_original_identifier_contract_is_enforced(item) -> None: question="question", solution="solution", ) + + +def test_surface_text_must_be_only_a_token_safe_rename(item) -> None: + rename_map = {"a": "positivequantity"} + question = apply_rename_map(item.problem, rename_map) + solution = apply_rename_map(item.solution, rename_map) + validate_surface_variant( + item, + rename_map=rename_map, + question=question, + solution=solution, + ) + + with pytest.raises(ValueError, match="exact deterministic rename"): + validate_surface_variant( + item, + rename_map=rename_map, + question=question + " Extra text.", + solution=solution, + ) -- cgit v1.2.3