summaryrefslogtreecommitdiff
path: root/tests/test_pipeline.py
diff options
context:
space:
mode:
Diffstat (limited to 'tests/test_pipeline.py')
-rw-r--r--tests/test_pipeline.py176
1 files changed, 70 insertions, 106 deletions
diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py
index ea1077a..864e967 100644
--- a/tests/test_pipeline.py
+++ b/tests/test_pipeline.py
@@ -10,62 +10,56 @@ from gap_pipeline.store import RunStore
from conftest import changed_candidate
-def accept_verdict() -> dict:
+def accept_review() -> dict[str, str]:
return {
"verdict": "accept",
- "step_by_step_check": "n1 passes; n2 passes",
+ "step_by_step_check": "every method label is instantiated",
"blocking_issues": "",
"patch_suggestion": "",
}
-def reject_verdict(iteration: int) -> dict:
+def reject_review(iteration: int) -> dict[str, str]:
return {
"verdict": "reject",
- "step_by_step_check": f"n1 passes; n2 fails at iteration {iteration}",
- "blocking_issues": "the terminal algebra needs correction",
- "patch_suggestion": "correct the terminal algebra without changing the plan",
+ "step_by_step_check": f"terminal step fails at iteration {iteration}",
+ "blocking_issues": "terminal algebra is incorrect",
+ "patch_suggestion": "correct the terminal algebra",
}
def proposer_responses(
item_id: str,
- dag_dict: dict,
plan_dict: dict,
- replacement_dict: dict,
- diffused_dict: dict,
candidate_dict: dict,
) -> dict:
return {
- f"{item_id}.stage1": dag_dict,
- f"{item_id}.stage2": plan_dict,
- f"{item_id}.stage3": replacement_dict,
- f"{item_id}.stage4": diffused_dict,
- f"{item_id}.stage5": candidate_dict,
+ f"{item_id}.plan": plan_dict,
+ f"{item_id}.candidate": candidate_dict,
+ }
+
+
+def repair_response(candidate: dict) -> dict[str, str]:
+ return {
+ "corrected_question": candidate["question"],
+ "corrected_solution": candidate["solution"],
+ "changes_made": "corrected the terminal algebra",
}
def test_two_consecutive_unanimous_passes_accept(
tmp_path,
item,
- dag_dict,
plan_dict,
- replacement_dict,
- diffused_dict,
candidate_dict,
) -> None:
proposer = ScriptedClient(
- proposer_responses(
- item.item_id,
- dag_dict,
- plan_dict,
- replacement_dict,
- diffused_dict,
- candidate_dict,
- )
+ proposer_responses(item.item_id, plan_dict, candidate_dict)
)
judges = [
- ScriptedClient({f"{item.item_id}.verify": [accept_verdict(), accept_verdict()]})
+ ScriptedClient(
+ {f"{item.item_id}.verify": [accept_review(), accept_review()]}
+ )
for _ in range(5)
]
pipeline = KernelPipeline(
@@ -78,138 +72,105 @@ def test_two_consecutive_unanimous_passes_accept(
assert result.status == "accepted"
assert len(result.iterations) == 2
assert [row.pass_streak_after for row in result.iterations] == [1, 2]
- assert not any(row.repaired for row in result.iterations)
final_path = tmp_path / "items" / item.item_id / "final.json"
final = json.loads(final_path.read_text())
- assert final["human_audit_selected"] in {True, False}
- assert len(list((final_path.parent / "calls").glob("*.json"))) == 15
+ assert final["accepted_candidate_sha256"]
+ assert len(list((final_path.parent / "calls").glob("*.json"))) == 12
def test_rejection_resets_streak_and_repairs(
tmp_path,
item,
- dag_dict,
plan_dict,
- replacement_dict,
- diffused_dict,
candidate_dict,
) -> None:
repaired = changed_candidate(candidate_dict, "fixed")
- responses = proposer_responses(
- item.item_id,
- dag_dict,
- plan_dict,
- replacement_dict,
- diffused_dict,
- candidate_dict,
- )
- responses[f"{item.item_id}.repair"] = repaired
+ responses = proposer_responses(item.item_id, plan_dict, candidate_dict)
+ responses[f"{item.item_id}.repair"] = repair_response(repaired)
proposer = ScriptedClient(responses)
judges = []
for judge_id in range(1, 6):
- first = reject_verdict(1) if judge_id == 1 else accept_verdict()
+ first = reject_review(1) if judge_id == 1 else accept_review()
judges.append(
ScriptedClient(
{
f"{item.item_id}.verify": [
first,
- accept_verdict(),
- accept_verdict(),
+ accept_review(),
+ accept_review(),
]
}
)
)
- pipeline = KernelPipeline(
- proposer=proposer,
- judges=judges,
- store=RunStore(tmp_path, item.item_id),
- config=PipelineConfig(proposer_model="scripted", judge_model="scripted"),
+ result = asyncio.run(
+ KernelPipeline(
+ proposer=proposer,
+ judges=judges,
+ store=RunStore(tmp_path, item.item_id),
+ config=PipelineConfig(
+ proposer_model="scripted",
+ judge_model="scripted",
+ ),
+ ).run(item)
)
- result = asyncio.run(pipeline.run(item))
assert result.status == "accepted"
- assert len(result.iterations) == 3
- assert result.iterations[0].repaired
assert [row.pass_streak_after for row in result.iterations] == [0, 1, 2]
- assert (
- result.iterations[0].repaired_candidate_sha256
- == result.iterations[1].candidate_sha256
- == result.iterations[2].candidate_sha256
- )
+ assert result.iterations[0].candidate_sha256 != result.iterations[1].candidate_sha256
+ assert result.iterations[1].candidate_sha256 == result.iterations[2].candidate_sha256
-def test_rejection_after_one_pass_resets_streak_on_new_candidate(
+def test_rejection_after_one_pass_resets_streak(
tmp_path,
item,
- dag_dict,
plan_dict,
- replacement_dict,
- diffused_dict,
candidate_dict,
) -> None:
repaired = changed_candidate(candidate_dict, "after-broken-streak")
- responses = proposer_responses(
- item.item_id,
- dag_dict,
- plan_dict,
- replacement_dict,
- diffused_dict,
- candidate_dict,
- )
- responses[f"{item.item_id}.repair"] = repaired
+ responses = proposer_responses(item.item_id, plan_dict, candidate_dict)
+ responses[f"{item.item_id}.repair"] = repair_response(repaired)
proposer = ScriptedClient(responses)
judges = []
for judge_id in range(1, 6):
- second = reject_verdict(2) if judge_id == 3 else accept_verdict()
+ second = reject_review(2) if judge_id == 3 else accept_review()
judges.append(
ScriptedClient(
{
f"{item.item_id}.verify": [
- accept_verdict(),
+ accept_review(),
second,
- accept_verdict(),
- accept_verdict(),
+ accept_review(),
+ accept_review(),
]
}
)
)
- pipeline = KernelPipeline(
- proposer=proposer,
- judges=judges,
- store=RunStore(tmp_path, item.item_id),
- config=PipelineConfig(proposer_model="scripted", judge_model="scripted"),
+ result = asyncio.run(
+ KernelPipeline(
+ proposer=proposer,
+ judges=judges,
+ store=RunStore(tmp_path, item.item_id),
+ config=PipelineConfig(
+ proposer_model="scripted",
+ judge_model="scripted",
+ ),
+ ).run(item)
)
- result = asyncio.run(pipeline.run(item))
assert result.status == "accepted"
assert [row.pass_streak_after for row in result.iterations] == [1, 0, 1, 2]
assert result.iterations[0].candidate_sha256 == result.iterations[1].candidate_sha256
- assert result.iterations[1].repaired_candidate_sha256 != result.iterations[1].candidate_sha256
- assert (
- result.iterations[1].repaired_candidate_sha256
- == result.iterations[2].candidate_sha256
- == result.iterations[3].candidate_sha256
- )
+ assert result.iterations[1].candidate_sha256 != result.iterations[2].candidate_sha256
def test_fifteen_failed_rounds_reject(
tmp_path,
item,
- dag_dict,
plan_dict,
- replacement_dict,
- diffused_dict,
candidate_dict,
) -> None:
- responses = proposer_responses(
- item.item_id,
- dag_dict,
- plan_dict,
- replacement_dict,
- diffused_dict,
- candidate_dict,
- )
+ responses = proposer_responses(item.item_id, plan_dict, candidate_dict)
responses[f"{item.item_id}.repair"] = [
- changed_candidate(candidate_dict, f"repair-{iteration}")
+ repair_response(changed_candidate(candidate_dict, f"repair-{iteration}"))
for iteration in range(1, 15)
]
proposer = ScriptedClient(responses)
@@ -217,20 +178,23 @@ def test_fifteen_failed_rounds_reject(
ScriptedClient(
{
f"{item.item_id}.verify": [
- reject_verdict(iteration) for iteration in range(1, 16)
+ reject_review(iteration) for iteration in range(1, 16)
]
}
)
for _ in range(5)
]
- pipeline = KernelPipeline(
- proposer=proposer,
- judges=judges,
- store=RunStore(tmp_path, item.item_id),
- config=PipelineConfig(proposer_model="scripted", judge_model="scripted"),
+ result = asyncio.run(
+ KernelPipeline(
+ proposer=proposer,
+ judges=judges,
+ store=RunStore(tmp_path, item.item_id),
+ config=PipelineConfig(
+ proposer_model="scripted",
+ judge_model="scripted",
+ ),
+ ).run(item)
)
- result = asyncio.run(pipeline.run(item))
assert result.status == "rejected"
assert len(result.iterations) == 15
- assert sum(row.repaired for row in result.iterations) == 14
assert all(row.pass_streak_after == 0 for row in result.iterations)
generated by cgit v1.2.3 (git 2.25.1) at 2026-07-24 22:18:22 +0000