"""Surface-renaming generation using the verbatim original prompts.""" from __future__ import annotations import re from dataclasses import dataclass from typing import Literal from .clients import JsonLLM from .models import CanonicalItem, ModelCallRecord from .prompts import surface_system, surface_user from .store import RunStore SurfaceFamily = Literal[ "descriptive_long", "descriptive_long_confusing", "descriptive_long_misleading", "garbled_string", ] SURFACE_FAMILIES: tuple[SurfaceFamily, ...] = ( "descriptive_long", "descriptive_long_confusing", "descriptive_long_misleading", "garbled_string", ) IDENTIFIER_RE = re.compile(r"^[a-z]{8,}$") def apply_rename_map(text: str, rename_map: dict[str, str]) -> str: """Apply symbol renames without replacing letters inside other tokens.""" rendered = text for source in sorted(rename_map, key=len, reverse=True): pattern = re.escape(source) if source[0].isalnum(): pattern = rf"(? dict[str, object]: return { "map": dict(self.rename_map), "question": self.question, "solution": self.solution, } def validate_surface_variant( item: CanonicalItem, *, rename_map: dict[str, str], question: str, solution: str, ) -> None: expected = set(item.variables + item.parameters) if set(rename_map) != expected: raise ValueError( "rename map must contain every var and param exactly once; " f"expected {sorted(expected)}, received {sorted(rename_map)}" ) if len(rename_map.values()) != len(set(rename_map.values())): raise ValueError("surface replacements must be one-to-one") forbidden = expected | set(item.scientific_constants) for replacement in rename_map.values(): if replacement in forbidden: raise ValueError(f"replacement {replacement!r} collides with input symbols") if not IDENTIFIER_RE.fullmatch(replacement): raise ValueError( f"replacement {replacement!r} violates the original prompt contract" ) if not question.strip() or not solution.strip(): raise ValueError("surface question and solution must be non-empty") expected_question = apply_rename_map(item.problem, rename_map) expected_solution = apply_rename_map(item.solution, rename_map) if question != expected_question or solution != expected_solution: raise ValueError( "surface text must be the exact deterministic rename of the " "canonical question and solution" ) class SurfacePipeline: def __init__(self, proposer: JsonLLM, store: RunStore) -> None: self.proposer = proposer self.store = store async def run_family( self, item: CanonicalItem, family: SurfaceFamily, ) -> SurfaceVariant: request_id = f"{item.item_id}.surface.{family}" system_prompt = surface_system(family) user_prompt = surface_user(item) response = await self.proposer.generate_json( system_prompt=system_prompt, user_prompt=user_prompt, request_id=request_id, ) self.store.write_call( ModelCallRecord( request_id=request_id, model=response.model, system_prompt=system_prompt, user_prompt=user_prompt, response_data=response.data, raw_text=response.raw_text, provider_response_id=response.response_id, usage=response.usage, ) ) rename_map = { str(old): str(new) for old, new in dict(response.data["map"]).items() } question = apply_rename_map(item.problem, rename_map) solution = apply_rename_map(item.solution, rename_map) validate_surface_variant( item, rename_map=rename_map, question=question, solution=solution, ) variant = SurfaceVariant( family=family, rename_map=rename_map, question=question, solution=solution, ) self.store.write_stage( f"surface_{family}_variant", variant.as_release_payload(), request_id=request_id, ) return variant async def run_all( self, item: CanonicalItem, ) -> dict[SurfaceFamily, SurfaceVariant]: output: dict[SurfaceFamily, SurfaceVariant] = {} for family in SURFACE_FAMILIES: output[family] = await self.run_family(item, family) return output