1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
|
from __future__ import annotations
import hashlib
import inspect
from gap_pipeline import kernel_prompts, prompts
from gap_pipeline.clients import OpenAIJsonClient
from gap_pipeline.kernel_models import (
DiffusedProof,
MethodPlan,
ProofDAG,
ReplacementPlan,
)
from gap_pipeline.kernel_prompts import (
JUDGE_SYSTEM,
dag_user,
diffusion_user,
judge_user,
method_user,
render_user,
replacement_user,
)
EXPECTED = {
"KERNEL_PLAN_SYSTEM": "0c817ede22fd407411ee588a02cf7a14ddcbb3d607aa4db6b5f81619609bcfef",
"KERNEL_PLAN_PROMPT": "4aafd1c25d67c5a433c8ad48dcf347cfe53436e81cfde1883e044ca3c17d855d",
"KERNEL_GENERATE_SYSTEM": "db0c902e15c9830a3c3860509bdb8a17b412d6b646b0f7e3278f32669046212b",
"KERNEL_GENERATE_PROMPT": "89c8a05dcda1b866b7835167c41184de9176ea058922e2cb250325825f79db92",
"JUDGE_SYSTEM_PROMPT": "50d5ee3ccbdf5196afea1eea60e61e74a324c89f14086b0ec6a8e14cb4ad0f30",
"JUDGE_USER_TEMPLATE": "59338f0a2522bda73006e7f62243578596cb8113da90fd8bfe5767d5a2bf4c2b",
"FIX_SYSTEM_PROMPT": "1c028afeac4bfbf92dd56cfce7b004b0823ab721f6f71e776bb90a5c652c1ed8",
"FIX_USER_TEMPLATE": "f12126acdc1518e872cd61f6baf2eedd49f1ad85eca69438e78128293959a8e9",
"SURFACE_SYSTEM_BASE": "d05ab24b4afd4ccd23e7954fa1111ce9c74893f8572c62617a3e568e3a743994",
"SURFACE_TASK_COMMON": "1cca2d06215207aa85ae36b368298a0b13ad4c2ef475079f1df0b2c4d9a5edd1",
"SURFACE_TASK_DESCRIPTIVE": "06b9b9c6a65ceae6818f30387cfd6e18bb46bbc1ccdf79adbba73f8849f6b354",
"SURFACE_TASK_CONFUSING": "7df3748ab186bbb6d9f421de5a7200754105154036d5cf6fb21ee32ac1f61848",
"SURFACE_TASK_MISLEADING": "fcde08ded42df91b44cb403f9a439550077245d71586adb6da9fb4983a4ea76e",
"SURFACE_TASK_GARBLED": "576301ca8e9eff2e4a2d9ed6a74f884a5a881ba22fe3a38e22e5eb46e341fac0",
"SURFACE_RETURN_SPEC": "b9f33abc9c8ab5e30b0154a6eebbd139d91afea6a744b454d4e150a6f7224a26",
"SURFACE_USER_TEMPLATE": "5d9043d7d1c1033db1aea0696812a2c4cbb6f3e28ca9d7b6436c036b40831fe7",
}
EXPECTED_FIVE_STAGE = {
"DAG_SYSTEM": "86e596a0c3aea07100ba642bb07d22e25385c2a51007b8f7578b662502ae4650",
"DAG_USER": "28757954499cee438cb3676fbb8aff39b2bbb4644a1f9dac2604b71d24b03547",
"METHOD_SYSTEM": "1091d4db002673abf67f4407418d0d7e34f648878f907d21e1ee910318d447ce",
"METHOD_USER": "79901153a7e2ef16ca4ade9763e4fd05a6d9d248a76085998723389f9934dca6",
"REPLACEMENT_SYSTEM": "2bf314a8cdcf986b344d45af081f7a1f675da6be5687ae4cd594db2ed7c0b25b",
"REPLACEMENT_USER": "6e88657245d951e47893eb9bcd6dbc5cd7a826fc67b8b2ce849da5b559f19fd9",
"DIFFUSION_SYSTEM": "f6ce12280b592830f5bacca3c8bc8131da26c9c0f94ffb9bfb7e9f53347f63ac",
"DIFFUSION_USER": "e92b39f15d1af1acc12daf34fabc0dfaf63bcebb4c947821fadb121aec44c9c1",
"RENDER_SYSTEM": "ead495a3f7df7d9655dc24ac2a437d4358ec080cd5e4477b3e97641828c3233e",
"RENDER_USER": "9e3eabd0fecf6946552db3217efa2eea7938099bcb093c9ca02fabf5b7d8f70c",
}
def test_prompt_values_are_byte_locked() -> None:
actual = {
name: hashlib.sha256(getattr(prompts, name).encode("utf-8")).hexdigest()
for name in EXPECTED
}
assert actual == EXPECTED
def test_five_stage_prompt_values_are_byte_locked() -> None:
actual = {
name: hashlib.sha256(
getattr(kernel_prompts, name).encode("utf-8")
).hexdigest()
for name in EXPECTED_FIVE_STAGE
}
assert actual == EXPECTED_FIVE_STAGE
def test_o3_adapter_does_not_send_temperature() -> None:
source = inspect.getsource(OpenAIJsonClient.generate_json)
assert '"temperature"' not in source
def test_literal_five_stage_prompts_render(item) -> None:
dag = ProofDAG.model_validate(
{
"nodes": [{"node_id": "n1", "claim": "claim", "dependencies": []}],
"terminal_node_id": "n1",
}
)
methods = MethodPlan.model_validate(
{"nodes": [{"node_id": "n1", "method_label": "method"}]}
)
replacements = ReplacementPlan.model_validate(
{
"changes": [
{
"slot_id": "slot1",
"source_node_id": "n1",
"description": "constant",
"original_value": "1",
"replacement_value": "2",
"guard_condition": "positive",
"guard_justification": "2 is positive",
}
],
"closure_statement": "No undeclared changes.",
}
)
diffused = DiffusedProof.model_validate(
{
"nodes": [
{
"node_id": "n1",
"dependencies": [],
"method_label": "method",
"instantiated_claim": "new claim",
"justification": "valid",
}
],
"terminal_node_id": "n1",
"terminal_answer": "answer",
}
)
variant = {
"question": "question",
"solution": "[n1] solution",
"node_order": ["n1"],
"terminal_answer": "answer",
}
rendered = [
dag_user(item),
method_user(dag),
replacement_user(item, dag, methods),
diffusion_user(item, dag, methods, replacements),
render_user(replacements, diffused),
judge_user(item, methods, replacements, variant),
]
assert all("{" in value and "}" in value for value in rendered)
assert JUDGE_SYSTEM == prompts.JUDGE_SYSTEM_PROMPT
assert rendered[-1] == prompts.JUDGE_USER_TEMPLATE.format(
original_problem=item.problem,
original_solution=item.solution,
method_labels='["n1: method"]',
slot_replacement=replacements.model_dump_json(indent=2),
candidate_problem="question",
candidate_proof="[n1] solution",
)
|
