diff options
Diffstat (limited to 'GAP_End_to_End.ipynb')
| -rw-r--r-- | GAP_End_to_End.ipynb | 361 |
1 files changed, 190 insertions, 171 deletions
diff --git a/GAP_End_to_End.ipynb b/GAP_End_to_End.ipynb index bdb519f..2b8c10e 100644 --- a/GAP_End_to_End.ipynb +++ b/GAP_End_to_End.ipynb @@ -1,174 +1,193 @@ { - "cells": [ - { - "cell_type": "markdown", - "id": "gap-title", - "metadata": {}, - "source": [ - "# GAP end-to-end reproduction\n", - "\n", - "Run all cells to generate four surface variants and one verified kernel variant for a Putnam problem, then export a machine-readable GAP record.\n", - "\n", - "The live path uses `o3` for generation and five independent verification calls per round. At least two unanimous rounds are required, so a live run makes multiple API calls and may take several minutes. The API key is read with `getpass` and is never written into the notebook or run artifacts." - ] + "cells": [ + { + "cell_type": "markdown", + "id": "gap-title", + "metadata": {}, + "source": [ + "# GAP end-to-end reproduction\n", + "\n", + "Run all cells to generate four surface variants and one verified kernel variant for a Putnam problem, then export a machine-readable GAP record.\n", + "\n", + "The live kernel path executes the paper's five stages explicitly: concrete proof-DAG construction, content-free method abstraction, guarded replacement with recorded old/new values, node-by-node DAG diffusion, and answer-to-question rendering. It then uses five independent `o3` verification calls per round. At least two unanimous rounds on the unchanged full-provenance bundle are required, so a live run makes multiple API calls and may take several minutes.\n", + "\n", + "The API key is loaded from the process environment, a gitignored `.env` file, or a hidden `getpass` prompt. It is never written into the notebook or run artifacts." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "install-package", + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "import json\n", + "import os\n", + "import subprocess\n", + "import sys\n", + "from datetime import datetime, timezone\n", + "\n", + "candidates = [Path.cwd(), Path.cwd().parent]\n", + "PACKAGE_ROOT = next((p.resolve() for p in candidates if (p / \"pyproject.toml\").exists()), None)\n", + "if PACKAGE_ROOT is None:\n", + " raise RuntimeError(\"Open this notebook from the GAP package root or its parent directory.\")\n", + "\n", + "subprocess.run(\n", + " [sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"-e\", f\"{PACKAGE_ROOT}[api,test]\"],\n", + " check=True,\n", + ")\n", + "src_path = str(PACKAGE_ROOT / \"src\")\n", + "if src_path not in sys.path:\n", + " sys.path.insert(0, src_path)\n", + "subprocess.run(\n", + " [sys.executable, \"-m\", \"pytest\", \"-q\", str(PACKAGE_ROOT / \"tests\" / \"test_prompts.py\")],\n", + " cwd=PACKAGE_ROOT,\n", + " check=True,\n", + ")\n", + "print(f\"GAP package root: {PACKAGE_ROOT}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "configure-run", + "metadata": {}, + "outputs": [], + "source": [ + "# Configuration. Set GAP_OFFLINE_SMOKE=1 to exercise the full pipeline without an API call.\n", + "OFFLINE_SMOKE = os.environ.get(\"GAP_OFFLINE_SMOKE\", \"0\") == \"1\"\n", + "MODEL = os.environ.get(\"GAP_MODEL\", \"o3\")\n", + "ITEM_ID = os.environ.get(\"GAP_ITEM_ID\", \"1998-B-1\")\n", + "\n", + "dataset_candidates = [\n", + " Path(os.environ[\"PUTNAMGAP_DATASET\"]).expanduser() if os.environ.get(\"PUTNAMGAP_DATASET\") else None,\n", + " PACKAGE_ROOT / \"examples\" / \"sample_data\",\n", + " PACKAGE_ROOT.parent / \"putnamsup\" / \"PutnamGAP\",\n", + "]\n", + "DATASET_DIR = next((p.resolve() for p in dataset_candidates if p is not None and p.exists()), None)\n", + "if not OFFLINE_SMOKE and DATASET_DIR is None:\n", + " raise RuntimeError(\"Set PUTNAMGAP_DATASET to the directory containing the PutnamGAP JSON files.\")\n", + "\n", + "stamp = datetime.now(timezone.utc).strftime(\"%Y%m%dT%H%M%SZ\")\n", + "OUTPUT_BASE = Path(os.environ.get(\"GAP_NOTEBOOK_OUTPUT_BASE\", PACKAGE_ROOT / \"notebook_runs\")).expanduser()\n", + "WORK_ROOT = OUTPUT_BASE / f\"{ITEM_ID}-{stamp}\"\n", + "print({\"offline_smoke\": OFFLINE_SMOKE, \"model\": MODEL, \"item_id\": ITEM_ID, \"dataset\": str(DATASET_DIR), \"work_root\": str(WORK_ROOT)})" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "load-api-key", + "metadata": {}, + "outputs": [], + "source": [ + "api_key = None\n", + "if not OFFLINE_SMOKE:\n", + " api_key = os.environ.get(\"OPENAI_API_KEY\")\n", + " if not api_key:\n", + " env_path = PACKAGE_ROOT / \".env\"\n", + " if env_path.exists():\n", + " for raw_line in env_path.read_text(encoding=\"utf-8\").splitlines():\n", + " line = raw_line.strip()\n", + " if line and not line.startswith(\"#\") and line.startswith(\"OPENAI_API_KEY=\"):\n", + " api_key = line.split(\"=\", 1)[1].strip().strip('\"').strip(\"'\")\n", + " break\n", + " if not api_key:\n", + " from getpass import getpass\n", + " api_key = getpass(\"OPENAI_API_KEY (input hidden): \").strip()\n", + " if not api_key:\n", + " raise RuntimeError(\"An OpenAI API key is required for the live run.\")\n", + " print(\"API key loaded in memory; it will not be displayed or saved.\")\n", + "else:\n", + " print(\"Offline scripted smoke mode: no API key required.\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "run-pipeline", + "metadata": {}, + "outputs": [], + "source": [ + "from gap_pipeline.e2e import run_live_item, run_offline_smoke\n", + "\n", + "if OFFLINE_SMOKE:\n", + " result = await run_offline_smoke(WORK_ROOT)\n", + "else:\n", + " result = await run_live_item(\n", + " dataset_dir=DATASET_DIR,\n", + " item_id=ITEM_ID,\n", + " work_root=WORK_ROOT,\n", + " model=MODEL,\n", + " api_key=api_key,\n", + " )\n", + "\n", + "print(json.dumps(result, indent=2, ensure_ascii=False))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "verify-output", + "metadata": {}, + "outputs": [], + "source": [ + "# End-to-end assertions and a compact preview of the exported record.\n", + "record = json.loads(Path(result[\"release_record\"]).read_text(encoding=\"utf-8\"))\n", + "expected_families = {\n", + " \"descriptive_long\",\n", + " \"descriptive_long_confusing\",\n", + " \"descriptive_long_misleading\",\n", + " \"garbled_string\",\n", + " \"kernel_variant\",\n", + "}\n", + "assert set(record[\"variants\"]) == expected_families\n", + "assert result[\"kernel_status\"] == \"accepted\"\n", + "assert result[\"verification_rounds\"] >= 2\n", + "assert result[\"export_status\"] == \"complete\"\n", + "\n", + "kernel = record[\"variants\"][\"kernel_variant\"]\n", + "meta = kernel[\"_meta\"]\n", + "assert meta[\"proof_dag\"][\"nodes\"]\n", + "assert meta[\"method_plan\"][\"nodes\"]\n", + "assert meta[\"replacement_plan\"][\"changes\"]\n", + "assert meta[\"diffused_proof\"][\"nodes\"]\n", + "assert meta[\"accepted_bundle_sha256\"]\n", + "\n", + "print(\"END-TO-END PASS\")\n", + "print(f\"Item: {record['index']}\")\n", + "print(f\"Variant families: {sorted(record['variants'])}\")\n", + "print(f\"Verification rounds: {result['verification_rounds']}\")\n", + "print(f\"Declared replacements: {len(meta['replacement_plan']['changes'])}\")\n", + "print(f\"Proof-DAG nodes: {len(meta['proof_dag']['nodes'])}\")\n", + "print(f\"Kernel question preview: {kernel['question'][:500]}\")\n", + "print(f\"Exported record: {result['release_record']}\")\n", + "print(f\"Manifest: {result['manifest']}\")" + ] + }, + { + "cell_type": "markdown", + "id": "full-dataset-note", + "metadata": {}, + "source": [ + "## Run another released item\n", + "\n", + "The notebook intentionally runs one item so the complete path can be checked at modest cost. Set `PUTNAMGAP_DATASET` and `GAP_ITEM_ID` before running to choose another source item; repeat the same runner over IDs for a larger generation job." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.11" + } }, - { - "cell_type": "code", - "id": "install-package", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from pathlib import Path\n", - "import json\n", - "import os\n", - "import subprocess\n", - "import sys\n", - "from datetime import datetime, timezone\n", - "\n", - "candidates = [Path.cwd(), Path.cwd().parent]\n", - "PACKAGE_ROOT = next((p.resolve() for p in candidates if (p / \"pyproject.toml\").exists()), None)\n", - "if PACKAGE_ROOT is None:\n", - " raise RuntimeError(\"Open this notebook from the GAP package root or its parent directory.\")\n", - "\n", - "subprocess.run(\n", - " [sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"-e\", f\"{PACKAGE_ROOT}[api,test]\"],\n", - " check=True,\n", - ")\n", - "src_path = str(PACKAGE_ROOT / \"src\")\n", - "if src_path not in sys.path:\n", - " sys.path.insert(0, src_path)\n", - "subprocess.run(\n", - " [sys.executable, \"-m\", \"pytest\", \"-q\", str(PACKAGE_ROOT / \"tests\" / \"test_prompts.py\")],\n", - " cwd=PACKAGE_ROOT,\n", - " check=True,\n", - ")\n", - "print(f\"GAP package root: {PACKAGE_ROOT}\")" - ] - }, - { - "cell_type": "code", - "id": "configure-run", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Configuration. Set GAP_OFFLINE_SMOKE=1 to exercise the full pipeline without an API call.\n", - "OFFLINE_SMOKE = os.environ.get(\"GAP_OFFLINE_SMOKE\", \"0\") == \"1\"\n", - "MODEL = os.environ.get(\"GAP_MODEL\", \"o3\")\n", - "ITEM_ID = os.environ.get(\"GAP_ITEM_ID\", \"1998-B-1\")\n", - "\n", - "dataset_candidates = [\n", - " Path(os.environ[\"PUTNAMGAP_DATASET\"]).expanduser() if os.environ.get(\"PUTNAMGAP_DATASET\") else None,\n", - " PACKAGE_ROOT / \"examples\" / \"sample_data\",\n", - " PACKAGE_ROOT.parent / \"putnamsup\" / \"PutnamGAP\",\n", - "]\n", - "DATASET_DIR = next((p.resolve() for p in dataset_candidates if p is not None and p.exists()), None)\n", - "if not OFFLINE_SMOKE and DATASET_DIR is None:\n", - " raise RuntimeError(\"Set PUTNAMGAP_DATASET to the directory containing the PutnamGAP JSON files.\")\n", - "\n", - "stamp = datetime.now(timezone.utc).strftime(\"%Y%m%dT%H%M%SZ\")\n", - "OUTPUT_BASE = Path(os.environ.get(\"GAP_NOTEBOOK_OUTPUT_BASE\", PACKAGE_ROOT / \"notebook_runs\")).expanduser()\n", - "WORK_ROOT = OUTPUT_BASE / f\"{ITEM_ID}-{stamp}\"\n", - "print({\"offline_smoke\": OFFLINE_SMOKE, \"model\": MODEL, \"item_id\": ITEM_ID, \"dataset\": str(DATASET_DIR), \"work_root\": str(WORK_ROOT)})" - ] - }, - { - "cell_type": "code", - "id": "load-api-key", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "api_key = None\n", - "if not OFFLINE_SMOKE:\n", - " api_key = os.environ.get(\"OPENAI_API_KEY\")\n", - " if not api_key:\n", - " from getpass import getpass\n", - " api_key = getpass(\"OPENAI_API_KEY (input hidden): \").strip()\n", - " if not api_key:\n", - " raise RuntimeError(\"An OpenAI API key is required for the live run.\")\n", - " print(\"API key loaded in memory; it will not be displayed or saved.\")\n", - "else:\n", - " print(\"Offline scripted smoke mode: no API key required.\")" - ] - }, - { - "cell_type": "code", - "id": "run-pipeline", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from gap_pipeline.e2e import run_live_item, run_offline_smoke\n", - "\n", - "if OFFLINE_SMOKE:\n", - " result = await run_offline_smoke(WORK_ROOT)\n", - "else:\n", - " result = await run_live_item(\n", - " dataset_dir=DATASET_DIR,\n", - " item_id=ITEM_ID,\n", - " work_root=WORK_ROOT,\n", - " model=MODEL,\n", - " api_key=api_key,\n", - " )\n", - "\n", - "print(json.dumps(result, indent=2, ensure_ascii=False))" - ] - }, - { - "cell_type": "code", - "id": "verify-output", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# End-to-end assertions and a compact preview of the exported record.\n", - "record = json.loads(Path(result[\"release_record\"]).read_text(encoding=\"utf-8\"))\n", - "expected_families = {\n", - " \"descriptive_long\",\n", - " \"descriptive_long_confusing\",\n", - " \"descriptive_long_misleading\",\n", - " \"garbled_string\",\n", - " \"kernel_variant\",\n", - "}\n", - "assert set(record[\"variants\"]) == expected_families\n", - "assert result[\"kernel_status\"] == \"accepted\"\n", - "assert result[\"verification_rounds\"] >= 2\n", - "assert result[\"export_status\"] == \"complete\"\n", - "\n", - "kernel = record[\"variants\"][\"kernel_variant\"]\n", - "print(\"END-TO-END PASS\")\n", - "print(f\"Item: {record['index']}\")\n", - "print(f\"Variant families: {sorted(record['variants'])}\")\n", - "print(f\"Verification rounds: {result['verification_rounds']}\")\n", - "print(f\"Kernel question preview: {kernel['question'][:500]}\")\n", - "print(f\"Exported record: {result['release_record']}\")\n", - "print(f\"Manifest: {result['manifest']}\")" - ] - }, - { - "cell_type": "markdown", - "id": "full-dataset-note", - "metadata": {}, - "source": [ - "## Run another released item\n", - "\n", - "The notebook intentionally runs one item so the complete path can be checked at modest cost. Set `PUTNAMGAP_DATASET` and `GAP_ITEM_ID` before running to choose another source item; repeat the same runner over IDs for a larger generation job." - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python", - "version": "3.11" - } - }, - "nbformat": 4, - "nbformat_minor": 5 + "nbformat": 4, + "nbformat_minor": 5 } |