import re from typing import Dict, Tuple, Optional def extract_solution(solution_str: str) -> Tuple[Optional[str], str]: """Extracts the final answer from the model's response string. Args: solution_str: Raw response string from the language model Returns: Tuple containing (extracted_answer, processed_string) """ # Split response to isolate assistant output # if "Assistant:" in solution_str: # processed_str = solution_str.split("Assistant:", 1)[1] # elif "<|im_start|>assistant" in solution_str: # processed_str = solution_str.split("<|im_start|>assistant", 1)[1] # else: # print("[Error] Failed to locate model response header") # return None, solution_str processed_str = solution_str # Extract final answer using XML-style tags answer_pattern = r'(.*?)' matches = list(re.finditer(answer_pattern, processed_str, re.DOTALL)) if not matches: print("[Error] No valid answer tags found") return None, processed_str final_answer = matches[-1].group(1).strip() return final_answer, processed_str def parse_solution_text_format(solution_text: str) -> Dict[str, str]: """Parses ground truth solution text into status dictionary. Args: solution_text: Formatted solution text from dataset Returns: Dictionary mapping character names to their roles (knight/knave) """ status_dict = {} print("\n[Ground Truth Parsing]") for line in solution_text.split('\n'): line = line.strip() if not line: continue match = re.search(r'\b([A-Za-z]+)\b.*?\b(knight|knave)\b', line, re.IGNORECASE) if match: name, role = match.groups() status_dict[name] = role.lower() print(f" Found: {name} → {role}") else: print(f" [Warning] Unparseable line: '{line}'") return status_dict def parse_model_answer(answer_text: str, expected_names: list) -> Optional[Dict[str, str]]: """Parses model's answer text into status dictionary. Args: answer_text: Text extracted from model's tags expected_names: List of character names requiring identification Returns: Dictionary mapping character names to predicted roles, or None if incomplete """ status_dict = {} print("\n[Model Answer Parsing]") print(f" Expected characters: {expected_names}") for name in expected_names: pattern = re.compile( rf'\b{re.escape(name)}\b.*?\b(knight|knave)\b', re.IGNORECASE ) match = pattern.search(answer_text) if match: role = match.group(1).lower() status_dict[name] = role print(f" Found: {name} → {role}") else: print(f" [Error] Missing identification for {name}") return None return status_dict def validate_response_structure(processed_str: str) -> bool: """Performs comprehensive validation of response structure. Args: processed_str: Processed response string from the model Returns: Boolean indicating whether all formatting requirements are met """ print("\n[Structure Validation]") validation_passed = True return validation_passed # Check required tags tags = { 'think_end': ('', 1), 'answer_start': ('', 1), 'answer_end': ('', 1) } positions = {} for tag_name, (tag_str, expected_count) in tags.items(): count = processed_str.count(tag_str) positions[tag_name] = pos = processed_str.find(tag_str) print(f" {tag_str}: count={count}, position={pos}") if count != expected_count: print(f" [Error] {tag_str} appears {count} times (expected {expected_count})") validation_passed = False # Verify tag order if ( positions['think_end'] > positions['answer_start'] or positions['answer_start'] > positions['answer_end']): print(" [Error] Incorrect tag order: Expected ......") validation_passed = False else: print(" Tag sequence validation passed") return validation_passed