1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
|
{
"cells": [
{
"cell_type": "markdown",
"id": "gap-title",
"metadata": {},
"source": [
"# GAP end-to-end reproduction\n",
"\n",
"Run all cells to generate four surface variants and one verified kernel variant for a Putnam problem, then export a machine-readable GAP record.\n",
"\n",
"The live path uses `o3` for generation and five independent verification calls per round. At least two unanimous rounds are required, so a live run makes multiple API calls and may take several minutes. The API key is read with `getpass` and is never written into the notebook or run artifacts."
]
},
{
"cell_type": "code",
"id": "install-package",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from pathlib import Path\n",
"import json\n",
"import os\n",
"import subprocess\n",
"import sys\n",
"from datetime import datetime, timezone\n",
"\n",
"candidates = [Path.cwd(), Path.cwd().parent]\n",
"PACKAGE_ROOT = next((p.resolve() for p in candidates if (p / \"pyproject.toml\").exists()), None)\n",
"if PACKAGE_ROOT is None:\n",
" raise RuntimeError(\"Open this notebook from the GAP package root or its parent directory.\")\n",
"\n",
"subprocess.run(\n",
" [sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"-e\", f\"{PACKAGE_ROOT}[api,test]\"],\n",
" check=True,\n",
")\n",
"src_path = str(PACKAGE_ROOT / \"src\")\n",
"if src_path not in sys.path:\n",
" sys.path.insert(0, src_path)\n",
"print(f\"GAP package root: {PACKAGE_ROOT}\")"
]
},
{
"cell_type": "code",
"id": "configure-run",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Configuration. Set GAP_OFFLINE_SMOKE=1 to exercise the full pipeline without an API call.\n",
"OFFLINE_SMOKE = os.environ.get(\"GAP_OFFLINE_SMOKE\", \"0\") == \"1\"\n",
"MODEL = os.environ.get(\"GAP_MODEL\", \"o3\")\n",
"ITEM_ID = os.environ.get(\"GAP_ITEM_ID\", \"1998-B-1\")\n",
"\n",
"dataset_candidates = [\n",
" Path(os.environ[\"PUTNAMGAP_DATASET\"]).expanduser() if os.environ.get(\"PUTNAMGAP_DATASET\") else None,\n",
" PACKAGE_ROOT / \"examples\" / \"sample_data\",\n",
" PACKAGE_ROOT.parent / \"putnamsup\" / \"PutnamGAP\",\n",
"]\n",
"DATASET_DIR = next((p.resolve() for p in dataset_candidates if p is not None and p.exists()), None)\n",
"if not OFFLINE_SMOKE and DATASET_DIR is None:\n",
" raise RuntimeError(\"Set PUTNAMGAP_DATASET to the directory containing the PutnamGAP JSON files.\")\n",
"\n",
"stamp = datetime.now(timezone.utc).strftime(\"%Y%m%dT%H%M%SZ\")\n",
"OUTPUT_BASE = Path(os.environ.get(\"GAP_NOTEBOOK_OUTPUT_BASE\", PACKAGE_ROOT / \"notebook_runs\")).expanduser()\n",
"WORK_ROOT = OUTPUT_BASE / f\"{ITEM_ID}-{stamp}\"\n",
"print({\"offline_smoke\": OFFLINE_SMOKE, \"model\": MODEL, \"item_id\": ITEM_ID, \"dataset\": str(DATASET_DIR), \"work_root\": str(WORK_ROOT)})"
]
},
{
"cell_type": "code",
"id": "load-api-key",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"api_key = None\n",
"if not OFFLINE_SMOKE:\n",
" api_key = os.environ.get(\"OPENAI_API_KEY\")\n",
" if not api_key:\n",
" from getpass import getpass\n",
" api_key = getpass(\"OPENAI_API_KEY (input hidden): \").strip()\n",
" if not api_key:\n",
" raise RuntimeError(\"An OpenAI API key is required for the live run.\")\n",
" print(\"API key loaded in memory; it will not be displayed or saved.\")\n",
"else:\n",
" print(\"Offline scripted smoke mode: no API key required.\")"
]
},
{
"cell_type": "code",
"id": "run-pipeline",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from gap_pipeline.e2e import run_live_item, run_offline_smoke\n",
"\n",
"if OFFLINE_SMOKE:\n",
" result = await run_offline_smoke(WORK_ROOT)\n",
"else:\n",
" result = await run_live_item(\n",
" dataset_dir=DATASET_DIR,\n",
" item_id=ITEM_ID,\n",
" work_root=WORK_ROOT,\n",
" model=MODEL,\n",
" api_key=api_key,\n",
" )\n",
"\n",
"print(json.dumps(result, indent=2, ensure_ascii=False))"
]
},
{
"cell_type": "code",
"id": "verify-output",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# End-to-end assertions and a compact preview of the exported record.\n",
"record = json.loads(Path(result[\"release_record\"]).read_text(encoding=\"utf-8\"))\n",
"expected_families = {\n",
" \"descriptive_long\",\n",
" \"descriptive_long_confusing\",\n",
" \"descriptive_long_misleading\",\n",
" \"garbled_string\",\n",
" \"kernel_variant\",\n",
"}\n",
"assert set(record[\"variants\"]) == expected_families\n",
"assert result[\"kernel_status\"] == \"accepted\"\n",
"assert result[\"verification_rounds\"] >= 2\n",
"assert result[\"export_status\"] == \"complete\"\n",
"\n",
"kernel = record[\"variants\"][\"kernel_variant\"]\n",
"print(\"END-TO-END PASS\")\n",
"print(f\"Item: {record['index']}\")\n",
"print(f\"Variant families: {sorted(record['variants'])}\")\n",
"print(f\"Verification rounds: {result['verification_rounds']}\")\n",
"print(f\"Kernel question preview: {kernel['question'][:500]}\")\n",
"print(f\"Exported record: {result['release_record']}\")\n",
"print(f\"Manifest: {result['manifest']}\")"
]
},
{
"cell_type": "markdown",
"id": "full-dataset-note",
"metadata": {},
"source": [
"## Run another released item\n",
"\n",
"The notebook intentionally runs one item so the complete path can be checked at modest cost. Set `PUTNAMGAP_DATASET` and `GAP_ITEM_ID` before running to choose another source item; repeat the same runner over IDs for a larger generation job."
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"name": "python",
"version": "3.11"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
|
