From 73c194f304f827b55081b15524479f82a1b7d94c Mon Sep 17 00:00:00 2001 From: maszhongming Date: Tue, 16 Sep 2025 15:15:29 -0500 Subject: Initial commit --- notebooks/true_false_cypher_rag_vs_kg_rag.ipynb | 533 ++++++++++++++++++++++++ 1 file changed, 533 insertions(+) create mode 100644 notebooks/true_false_cypher_rag_vs_kg_rag.ipynb (limited to 'notebooks/true_false_cypher_rag_vs_kg_rag.ipynb') diff --git a/notebooks/true_false_cypher_rag_vs_kg_rag.ipynb b/notebooks/true_false_cypher_rag_vs_kg_rag.ipynb new file mode 100644 index 0000000..9e98fb2 --- /dev/null +++ b/notebooks/true_false_cypher_rag_vs_kg_rag.ipynb @@ -0,0 +1,533 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "id": "038208fa-0a00-47cf-90b3-1f131c6b13b3", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import os\n", + "from tqdm import tqdm\n", + "import re\n", + "from scipy import stats\n", + "import seaborn as sns\n", + "import matplotlib.pyplot as plt\n", + "\n", + "from IPython.display import clear_output\n" + ] + }, + { + "cell_type": "markdown", + "id": "33583b95-7796-4100-ad7a-89803377abbb", + "metadata": {}, + "source": [ + "## Custom functions" + ] + }, + { + "cell_type": "code", + "execution_count": 61, + "id": "491eabed-bbb6-4cc4-be9c-3c95559ed36d", + "metadata": {}, + "outputs": [], + "source": [ + "def extract_answer(text):\n", + " pattern = r\"(True|False|Don't know)\"\n", + " matches = re.findall(pattern, text)\n", + " return matches\n", + "\n", + "def process_df(rag_response_df):\n", + " rag_response_df.loc[:, \"answer_count\"] = rag_response_df.extracted_answer.apply(lambda x:len(x))\n", + " rag_response_df_multiple_answers = rag_response_df[rag_response_df.answer_count > 1]\n", + " rag_response_df_single_answer = rag_response_df.drop(rag_response_df_multiple_answers.index)\n", + " rag_response_df_single_answer.drop(\"answer_count\", axis=1, inplace=True)\n", + " rag_response_df_multiple_answers_ = []\n", + " for index, row in rag_response_df_multiple_answers.iterrows():\n", + " if row[\"extracted_answer\"][0] == row[\"extracted_answer\"][1]:\n", + " rag_response_df_multiple_answers_.append((row[\"question\"], row[\"label\"], row[\"llm_answer\"], row[\"extracted_answer\"][0]))\n", + " else:\n", + " rag_response_df_multiple_answers_.append((row[\"question\"], row[\"label\"], row[\"llm_answer\"], None))\n", + " rag_response_df_multiple_answers_ = pd.DataFrame(rag_response_df_multiple_answers_, columns=[\"question\", \"label\", \"llm_answer\", \"extracted_answer\"])\n", + " rag_response_df_final = pd.concat([rag_response_df_single_answer, rag_response_df_multiple_answers_], ignore_index=True)\n", + " rag_response_df_final = rag_response_df_final.explode(\"extracted_answer\")\n", + " \n", + " rag_incorrect_answers_because_of_na = rag_response_df_final[rag_response_df_final.extracted_answer.isna()]\n", + "\n", + " row_index_to_drop = list(rag_incorrect_answers_because_of_na.index.values)\n", + "\n", + " rag_response_df_final.drop(row_index_to_drop, inplace=True)\n", + "\n", + " rag_response_df_final = rag_response_df_final.reset_index()\n", + " response_transform = {\n", + " \"True\" : True,\n", + " \"False\" : False\n", + " }\n", + "\n", + " rag_response_df_final.extracted_answer = rag_response_df_final.extracted_answer.apply(lambda x:response_transform[x])\n", + "\n", + " return rag_response_df_final\n", + "\n", + "\n", + "def evaluate(df):\n", + " correct = df[df.label == df.extracted_answer]\n", + " incorrect = df[df.label != df.extracted_answer]\n", + " correct_frac = correct.shape[0]/df.shape[0]\n", + " incorrect_frac = incorrect.shape[0]/df.shape[0]\n", + " return correct_frac, incorrect_frac\n", + "\n", + "def evaluate_2(df):\n", + " correct = df[df.cypher_rag_final_answer == df.label]\n", + " incorrect = df[df.cypher_rag_final_answer != df.label]\n", + " correct_frac = correct.shape[0]/df.shape[0]\n", + " incorrect_frac = incorrect.shape[0]/df.shape[0]\n", + " return correct_frac, incorrect_frac\n", + "\n", + "\n", + "def bootstrap(cypher_rag, kg_rag, niter = 1000, nsample = 150):\n", + " cypher_rag_correct_frac_list = []\n", + " kg_rag_correct_frac_list = []\n", + " for i in tqdm(range(niter)):\n", + " cypher_rag_ = cypher_rag.sample(n=nsample, random_state=i)\n", + " cypher_rag_correct_frac, cypher_rag_incorrect_frac = evaluate_2(cypher_rag_)\n", + " kg_rag_ = kg_rag.iloc[cypher_rag_.index]\n", + " kg_rag_correct_frac, kg_rag_incorrect_frac = evaluate(kg_rag_)\n", + " cypher_rag_correct_frac_list.append(cypher_rag_correct_frac)\n", + " kg_rag_correct_frac_list.append(kg_rag_correct_frac)\n", + " return cypher_rag_correct_frac_list, kg_rag_correct_frac_list\n", + "\n", + "def plot_figure(cypher_rag_correct_frac_list, kg_rag_correct_frac_list):\n", + " fig = plt.figure(figsize=(5, 3))\n", + " ax = plt.gca()\n", + "\n", + " sns.kdeplot(cypher_rag_correct_frac_list, color=\"blue\", shade=True, label=\"Cypher-RAG\", ax=ax, lw=2, linestyle=\"-\", alpha=0.6)\n", + " sns.kdeplot(kg_rag_correct_frac_list, color=\"lightcoral\", shade=True, label=\"KG-RAG\", ax=ax, lw=2, linestyle=\"-\", alpha=0.6)\n", + "\n", + " for artist in ax.lines:\n", + " artist.set_edgecolor(\"black\")\n", + "\n", + " plt.xlabel(\"Accuracy\")\n", + " plt.ylabel(\"Density\")\n", + " plt.legend(loc=\"upper left\")\n", + " plt.xlim(0,1)\n", + "\n", + " ax.axvline(np.mean(cypher_rag_correct_frac_list), color='black', linestyle='--', lw=2)\n", + " ax.axvline(np.mean(kg_rag_correct_frac_list), color='black', linestyle='--', lw=2)\n", + "\n", + " sns.despine(top=True, right=True)\n", + " plt.legend(bbox_to_anchor=(0.3, 0.9))\n", + "\n", + " plt.show()\n", + " return fig\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "id": "ba398cb5-2fcf-48d1-bfb5-449821cb13d5", + "metadata": {}, + "source": [ + "## Load data and process it" + ] + }, + { + "cell_type": "code", + "execution_count": 62, + "id": "c71243a2-b043-40fa-ab6c-700e0ad58452", + "metadata": {}, + "outputs": [], + "source": [ + "cypher_rag = pd.read_csv('../data/results/cypher_rag_true_false_output.csv')\n", + "kg_rag = pd.read_csv('../data/results/gpt_4_PubMedBert_and_entity_recognition_based_node_retrieval_rag_based_true_false_binary_response.csv')\n", + "curated_data = pd.read_csv('../data/benchmark_data/true_false_questions.csv').drop('Unnamed: 0', axis=1)\n", + "\n", + "kg_rag = pd.merge(curated_data, kg_rag, left_on='text', right_on='question').drop(['text', 'label_y'], axis=1).rename(columns={'label_x':'label'})\n", + "kg_rag.loc[:, 'extracted_answer'] = kg_rag['llm_answer'].apply(extract_answer)\n", + "kg_rag = process_df(kg_rag)\n" + ] + }, + { + "cell_type": "markdown", + "id": "1cca48bf-85b9-442b-8913-c6584aa09f9d", + "metadata": {}, + "source": [ + "## Bootstrap" + ] + }, + { + "cell_type": "code", + "execution_count": 63, + "id": "395594eb-4048-49b6-8f2d-dbe2ea38e595", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████████████████████████████████| 1000/1000 [00:01<00:00, 894.39it/s]\n" + ] + } + ], + "source": [ + "cypher_rag_correct_frac_list, kg_rag_correct_frac_list = bootstrap(cypher_rag, kg_rag)\n" + ] + }, + { + "cell_type": "markdown", + "id": "016fd3c3-6e9c-4de7-b4cd-706c4ee69e0d", + "metadata": {}, + "source": [ + "## Plot the figure" + ] + }, + { + "cell_type": "code", + "execution_count": 68, + "id": "ec2376b0-e4ff-4f38-be91-9c8dc8dca074", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/var/folders/p1/h56gxdhs5vgb0ztp7h4z606h0000gn/T/ipykernel_39048/2066733470.py:69: FutureWarning: \n", + "\n", + "`shade` is now deprecated in favor of `fill`; setting `fill=True`.\n", + "This will become an error in seaborn v0.14.0; please update your code.\n", + "\n", + " sns.kdeplot(cypher_rag_correct_frac_list, color=\"blue\", shade=True, label=\"Cypher-RAG\", ax=ax, lw=2, linestyle=\"-\", alpha=0.6)\n", + "/var/folders/p1/h56gxdhs5vgb0ztp7h4z606h0000gn/T/ipykernel_39048/2066733470.py:70: FutureWarning: \n", + "\n", + "`shade` is now deprecated in favor of `fill`; setting `fill=True`.\n", + "This will become an error in seaborn v0.14.0; please update your code.\n", + "\n", + " sns.kdeplot(kg_rag_correct_frac_list, color=\"lightcoral\", shade=True, label=\"KG-RAG\", ax=ax, lw=2, linestyle=\"-\", alpha=0.6)\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAdEAAAEmCAYAAAA9VKo6AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/TGe4hAAAACXBIWXMAAA9hAAAPYQGoP6dpAABGTElEQVR4nO3dd3hUVfoH8O9MQiaNTAglBQIEIkUIsNIERBGyNFdprvwQpYkNRCUqiLKgoMaFpVgACyGRXZq6gLuCFCPFgiAlgotEQpEaAsH0ZFLm/P4Y5uZO6sxkZu6dyffzPPN4586de1/uZvPmPefcczRCCAEiIiKymVbpAIiIiNwVkygREZGdmESJiIjsxCRKRERkJyZRIiIiOzGJEhER2YlJlIiIyE5MokRERHby+CQqhEBOTg44pwQRETmat9IBOFtubi70ej2ys7MRFBSkdDg1Wrp0KXJychAUFIS4uDilwyEiUoQ7/S7UKDnt36pVq7Bq1SqcP38eANCpUyfMmzcPw4YNAwAUFRXhhRdewMaNG2EwGDBkyBCsXLkSoaGhVl8jJyfHbZJoixYtcPnyZTRv3hyXLl1SOhwiIkW40+9CRZtzW7RogbfffhtHjhzB4cOHMXDgQIwYMQL/+9//AAAzZ87Ef//7X3z22WfYt28frly5gtGjRysZMhERkUTRSrQqISEhWLx4MR588EE0bdoU69evx4MPPggAOHXqFDp27IgDBw7gzjvvtOp8rESJiNyLO/0uVM3AorKyMmzcuBH5+fno06cPjhw5gpKSEsTGxkrHdOjQAS1btsSBAweqPY/BYEBOTo7Fi4iIyBkUT6InTpxAYGAgdDodnnrqKWzZsgW333470tPT4ePjg+DgYIvjQ0NDkZ6eXu354uPjodfrpVdkZKST/wVERFRfKZ5E27dvj5SUFBw8eBBPP/00Jk6ciJMnT9p9vjlz5iA7O1t6Xbx40YHREhERlVP8ERcfHx9ER0cDALp3746ffvoJ77zzDsaOHYvi4mJkZWVZVKPXrl1DWFhYtefT6XTQ6XTODpuIiEj5SrQio9EIg8GA7t27o0GDBkhOTpY+S01NxYULF9CnTx8FIyQiIjJRtBKdM2cOhg0bhpYtWyI3Nxfr16/H3r17sXPnTuj1ejz22GOIi4tDSEgIgoKCMGPGDPTp08fqkblEROReRFkZoK6HRmqkaBLNyMjAhAkTcPXqVej1enTp0gU7d+7En//8ZwDAsmXLoNVqMWbMGIvJFjzVfffdh5s3byIkJETpUIiIXM5YUIC8jz/Gn8PDkRMTgyatWikdUq1U95yoo7nTc6JERPWZ4ccfUbRzp/Teu0MH+D/4IDReXgpGVTPV9YkSEVH9ZLx50+J96alTKDl1SqForMMkSkREqmDMy6u0r+zKFQUisR6TKBERqYIoLKy8T+Wzzin+nCiV69GjB9LT0xEWFobDhw8rHQ4RkUuZk+iADz9ERl4emgUG4tv4eIWjqhmTqIqkp6fj8uXLSodBRKQIUVQEAMjIz8eV3FzTvvx8JUOqFZtziYhIHYqLK+0SBoMCgViPSZSIiFRBVJVEb1WnasUkSkREihNGI1BWZnqj0ZR/UFxs+kylmESJiEh5VVShEhU36TKJEhGR4kRJSfWf1ZRgFcYkSkREiqsxibISJSIiqkFpafm2vE8UrESJiIhqVFMlWmN/qcI42YKKLFq0CAUFBfD391c6FCIi15JVogtGjUJhaSl8b020oOZKlElURR5++GGlQyAiUoS8En2od29ovL1RdumS6TMVJ1E25xIRkfJklahGqwW0svRUU1OvwphEiYhIcaLiwCJZElVzJcrmXBVJTU1FaWkpvL290b59e6XDcWtlZWUoUfFfr+R8DRo0gJeXl9JhkLXMsxUB+C0jA2VaLTQ3buC2Jk2YRMk6gwYNwuXLl9G8eXNcutUXQLYRQiA9PR1ZWVlKh0IqEBwcjLCwMGgqPDJBKiSrRO//xz9w5Y8/ENGwIU6+8IKqm3OZRMmjmBNos2bN4O/vz1+e9ZQQAgUFBcjIyAAAhIeHKxwR1caiObfiZ6xEiZyvrKxMSqCNGzdWOhxSmJ+fHwAgIyMDzZo1Y9Ou2tWURFVciXJgEXkMcx8on7MlM/PPAvvH1c9dK1EmUfI4bMIlM/4suJEakqiaZyxiEiUiIsUJ2ehcd5o7l32i5PF69ADS0117zbAw4PBh117TGq1bt8bzzz+P559/XulQiCxVVYneSqZcxYVIQenpwOXLrn3Zk7TT09MxY8YMtGnTBjqdDpGRkbj//vuRnJzs+JviAgMGDIBGo4FGo4Gvry/atWuH+Ph4CCEqHXvgwAF4eXnhvvvuq/JcxcXFWLx4Me644w4EBARAr9eja9eumDt3Lq5cueLsfwq5QE19ompuzmUlSvWGRgM4e8xRQQFQRY6o1fnz59GvXz8EBwdj8eLFiImJQUlJCXbu3Inp06fj1KlTjg/WQYqLi+Hj41PlZ48//jgWLFgAg8GAb775Bk888QSCg4Px9NNPWxyXkJCAGTNmICEhAVeuXEFERIT0mcFgwODBg3H8+HG8/vrr6NevH5o2bYpz585hw4YNeO+99xAfH+/UfyO5gLw5twI251YjPj4emzdvxqlTp+Dn54e+ffvi73//u8VsPQMGDMC+ffssvvfkk0/igw8+cHW45Ob8/YHx4517jXXrgFsLT9hk2rRp0Gg0OHToEAICAqT9nTp1wpQpUzBlyhRkZGTgyy+/lD4rKSlB8+bNER8fj8ceewwDBgxA586dAQD//Oc/0aBBAzz99NNYsGCBxQCbgoICTJkyBZ999hkaNWqEuXPn4oknnpA+v3jxIl544QXs2rULWq0W/fv3xzvvvIPWrVsDACZNmoSsrCz07NkTK1asgE6nw7lz56r8d/n7+yMsLAwAMHnyZLz//vvYvXu3RRLNy8vDpk2bcPjwYaSnpyMpKQmvvPKK9PmyZcvw3Xff4fDhw/jTn/4k7W/ZsiXuueeeKitbckM1JVE251Zt3759mD59On788Ufs3r0bJSUlGDx4MPIr/BZ6/PHHcfXqVem1aNEihSJ2rp9++gkXL17ETz/9pHQo5EI3b97Ejh07MH36dIsEahYcHIypU6dix44duHr1qrT/yy+/REFBAcaOHSvt++STT+Dt7Y1Dhw7hnXfewdKlS7F69WqL8y1ZsgQ9evTAsWPHMG3aNDz99NNITU0FYErMQ4YMQcOGDfHtt9/i+++/R2BgIIYOHYpiWTWQnJyM1NRU7N692yKxV0cIgW+//RanTp2qVLV++umn6NChA9q3b49HHnkEa9assUiMGzZswJ///GeLBCrHEbieQd6cu3/RIvy2Zg32PPecaUdpqeXAIxVRNInu2LEDkyZNQqdOndC1a1ckJSXhwoULOHLkiMVx5r9mza+goCCFInau8PBwtGjRgrOr1DNpaWkQQqBDhw7VHtO3b1+0b98e//znP6V9iYmJ+Otf/4rAwEBpX2RkJJYtW4b27dtj/PjxmDFjBpYtW2ZxruHDh2PatGmIjo7G7Nmz0aRJE+zZswcAsGnTJhiNRqxevRoxMTHo2LEjEhMTceHCBezdu1c6R0BAAFavXo1OnTqhU6dO1ca9cuVKBAYGQqfT4e6774bRaMSzzz5rcUxCQgIeeeQRAMDQoUORnZ1t0fr022+/VZpLetSoUQgMDERgYCD69u1b7fXJjciSZHhICFo0aYKwRo2kfWpt0lXVwKLs7GwAQEhIiMX+devWoUmTJujcuTPmzJmDgoKCas9hMBiQk5Nj8SJSM2ubI6dOnYrExEQAwLVr1/DVV19hypQpFsfceeedFpVZnz59cPr0aZTJfkF16dJF2tZoNAgLC5Omx/v555+RlpaGhg0bSkkqJCQERUVFOHPmjPS9mJgYqaJct26ddGxgYCC+/fZb6bjx48cjJSUF33//PYYNG4ZXX33VIumlpqbi0KFDGDduHADA29sbY8eORUJCQo33YuXKlUhJScGUKVNq/H1A7qPSKi64tSSaWVGRiyOyjmoGFhmNRjz//PPo16+f1K8DmBaqbtWqFSIiInD8+HHMnj0bqamp2Lx5c5XniY+Px+uvv+6qsInq7LbbboNGo6l18NCECRPw8ssv48CBA/jhhx8QFRWF/v3723y9Bg0aWLzXaDQwGo0ATP2T3bt3x7p16yp9r2nTptK2vNn5gQceQO/evaX3zZs3l7b1ej2io6MBmJpto6OjceeddyI2NhaAqQotLS21GEgkhIBOp8P7778PvV6P2267TWpuNjO31lT8g5vcmLy51pw8vctTlFr7RVWTRKdPn45ffvkF3333ncV++YCHmJgYhIeHY9CgQThz5gzatm1b6Txz5sxBXFyc9D4nJweRkZHOC9yBPvroI+Tl5SEwMNDi302eLSQkBEOGDMGKFSvw7LPPVuoXzcrKQnBwMBo3boyRI0ciMTERBw4cwOTJkyud6+DBgxbvf/zxR9x2221Wzxt7xx13YNOmTWjWrJnV3SYNGzZEw4YNaz0uMDAQzz33HF588UUcO3YMZWVlWLt2LZYsWYLBgwdbHDty5Ehs2LABTz31FMaNG4e5c+fi2LFj1faLkgeQVaJrdu5EflER/IqKMLFdOwCAKCxUKrIaqSKJPvPMM/jyyy+xf/9+tGjRosZjzX/xpqWlVZlEdToddDqdU+J0tgULFkhLoTGJOl5BgWn0rLOvYY8VK1agX79+6NWrFxYsWIAuXbqgtLQUu3fvxqpVq/Drr78CMDXp/uUvf0FZWRkmTpxY6TwXLlxAXFwcnnzySRw9ehTvvfcelixZYnUc48ePx+LFizFixAgsWLAALVq0wO+//47Nmzdj1qxZtf7/szZPPvkkFi5ciH//+9/w9vbGH3/8gcceewx6vd7iuDFjxiAhIQFPPfUUZs6ciW3btmHQoEGYP38++vfvj0aNGuG3337DV199xYnlPYQ0cEirRfymTbiSmYmI4ODyJMrm3MqEEJgxYwa2bNmCvXv3IioqqtbvpKSkAODSRmQ7Iex7/MQV2rRpg6NHj+LNN9/ECy+8gKtXr6Jp06bo3r07Vq1aJR0XGxuL8PBwdOrUyaIJ1GzChAkoLCxEr1694OXlheeee86mP8j8/f2xf/9+zJ49G6NHj0Zubi6aN2+OQYMGOWRAX0hICCZMmIDXXnsNUVFRiI2NrZRAAVMSXbRoEY4fP44uXbogOTkZy5cvR2JiIubMmQOj0YioqCgMGzYMM2fOrHNcpALmSlQ+2lq2rdZKVCMUfMhq2rRpWL9+Pb744guL0Xd6vR5+fn44c+YM1q9fj+HDh6Nx48Y4fvw4Zs6ciRYtWlR6drQ6OTk50Ov1yM7OVv2o3hYtWnBR7jooKirCuXPnEBUVBV9fX2m/J037l5eXh+bNmyMxMRGjR4+2+GzAgAHo1q0bli9f7vgLu6nqfiZIfXKWLYPIyQG8vXH7smWmSrRRI5y89ZiLb2wsdP36KRxlZYpWoua/sAcMGGCxPzExEZMmTYKPjw++/vprLF++HPn5+YiMjMSYMWMwd+5cBaIld6XGOWxtZTQacePGDSxZsgTBwcF44IEHlA6JyLFkzblVMap0FLbizbk1iYyMtLriJPJkFy5cQFRUFFq0aIGkpCR4e6tiOAORw4jamnOZRInIXq1bt671j075ZAhEbudWJaqpphJVaxJV1WQLRERU/wghah9YxCRKRERUhVuTfQCo3Cd6671aR+cyiRIRkbKqmPJPcqv/X61JlH2iKtKuXTvo9XqEhoYqHQoRkctUnDf3togI6P390Sw4GBpvb4jiYojCQgghVLdqD5OoinzzzTdKh0BE5HoV5s396s03pbclp0+bNoQADAZAZc/7sjmXiIgUJa9EK1aaGtm0jmqc+o+VKHm8vI8+gjEvz6XX1AYGIpDzHxNZR96cW3FgkXwlFxUmUVai5PGMeXkQubkufdmatCdNmoSRI0da7Pv888/h6+srTSCfnp6O5557DtHR0fD19UVoaCj69euHVatW1bqmpkajkV5BQUHo2bMnvvjiiyqPjY+Ph5eXFxYvXlzl53WJg6hKVS2DVsV7NSZRVqIqMn78eNy4cQNNmjSpcj1HcoAKa2k6XEmJQ06zevVqTJ8+HR988AEmT56Ms2fPol+/fggODsZbb72FmJgY6HQ6nDhxAh999BGaN29e61SAiYmJGDp0KHJycrBy5Uo8+OCDOHr0KGJiYiyOW7NmDWbNmoU1a9bgpZdesvjMEXEQVSTk/7/RaDB5yRJk5uSgcVAQVj/8cPlxKlxTlElURfbt2ydNQE9O0KABfCokDEcrPnGizol00aJFmD9/PjZu3IhRo0YBMC3W4O3tjcOHD1usN9qmTRuMGDGi1tmMACA4OBhhYWEICwvDwoUL8c4772DPnj0WSXTfvn0oLCzEggULsHbtWvzwww/o27ev9Lkj4iCqpEJz7re//GKagL5xY0DeJ1pcrEBwNWNzLpGKzJ49GwsXLsSXX34pJdDMzEzs2rUL06dPr7Rgt5ktw/5LS0uRkJAAAPDx8bH4LCEhAePGjUODBg0wbtw46ThnxEFkVtPAIoskqsJKlEmUSCW++uorLFq0CF988QUGDRok7U9LS4MQwmK5QABo0qQJAgMDERgYiNmzZ9d6/nHjxiEwMBA6nQ4zZ85E69at8dBDD0mf5+Tk4PPPP8cjjzwCAHjkkUfw6aefIu9W/66j4iCqpIaBRRZz6TKJElF1unTpgtatW2P+/PlS4qrJoUOHkJKSgk6dOsFw65fLU089JSW0wMBAi+OXLVuGlJQUfPXVV7j99tuxevVqhISESJ9v2LABbdu2RdeuXQEA3bp1Q6tWrbBp0yab4yCyRcXJFiywOZeIrNG8eXPs3bsXly9fxtChQ5GbmwsAiI6OhkajQWpqqsXxbdq0QXR0NPz8/KR9CxYsQEpKivSSCwsLQ3R0NAYPHozExESMHTsWGRkZ0ucJCQn43//+B29vb+l18uRJrFmzxuY4iGxS0yMu8tG5TKJEVJNWrVph3759SE9PlxJp48aN8ec//xnvv/8+8vPza/x+s2bNEB0dLb2q06tXL3Tv3h1v3poZ5sSJEzh8+DD27t1rkYT37t2LAwcO4NSpUzbFQWSLiqNzLciTqgqTKEfnUv1RUmIaPevka9RVZGQk9u7di3vvvRdDhgzBjh07sHLlSvTr1w89evTAa6+9hi5dukCr1eKnn37CqVOn0L17d5uv8/zzz2PUqFGYNWsWEhIS0KtXL9x9992VjuvZsycSEhKwePFip8RBJK9EK64najFjkYMeIXMkJlGqX1T4f8KqtGjRwiKR7ty5E8eOHcNbb72FOXPm4NKlS9DpdLj99tvx4osvYtq0aTZfY+jQoYiKisKbb76JTz/9tNpBQWPGjMGSJUvw1ltvoW3btg6Pg8giObpZc65GePiDXTk5OdDr9cjOzkZQUJDS4dSoRYsW0nOily5dUjoct1NUVIRz584hKioKvrJJqjntX/1V3c8EqUvR11/D8P33AADv225Du2eflZ4TPb16NUp+/hkA4BUVhcAJE5QMtRJWoiry+OOPIzs7G3q9XulQPAqTGZG6VaxEJw8ejJyCAgT5+1tWpipsSWISVZH58+crHQIRkctVfMTl1XHjLA/QaAAhVNmcy9G5RESkrBoGFgEor0ZVWIkyiRIRkaJqHFgk28dKlMgFPHysHNmAPwtuwtokykqUatKiRQtoNBq0aNFC6VDcUoNby5xxTUsyM/8sNHD2EnhUJxUr0ejJk+H/wAOInjwZgOxZ0ZIS1f1hxIFFKpGZCWRlmbZVuO6sW/Dy8kJwcLA0lZ2/vz9XFamnhBAoKChARkYGgoOD4SV7YJ9UqKYZi4Dy6lQIwGi0mE9XaYom0fj4eGzevBmnTp2Cn58f+vbti7///e8Wq0QUFRXhhRdewMaNG2EwGDBkyBCsXLkSoaGhCkbueBMnAuaZ1DIzgf/+F7j/fmVjckdhYWEAYDEnLNVf5jVUSd2kSlSrrfoP34pT/6lonmZFk+i+ffswffp09OzZE6WlpXjllVcwePBgnDx5UlqvcObMmdi2bRs+++wz6PV6PPPMMxg9ejS+v/VgridISwO2bbPcN3UqcPo0oPL5IVRHo9EgPDwczZo1Q4kK+0/IdRo0aMAK1E3Ik2iV5LMWlZRAwyRqsmPHDov3SUlJaNasGY4cOYK7774b2dnZSEhIwPr16zFw4EAAQGJiIjp27Igff/wRd955pxJhO9z27ZX3ZWQAH34IvPSS6+PxBF5eXvwFSuQuakmiGq0W5p5QtQ0uUtXAouzsbACQ1jg8cuQISkpKEBsbKx3ToUMHtGzZEgcOHKjyHAaDATk5ORYvtavmn4I1a0xdAEREnsz86EqVz4gCql7JRTVJ1Gg04vnnn0e/fv3QuXNnAEB6ejp8fHwQHBxscWxoaCjS09OrPE98fDz0er30ioyMdHbodVZh2Ufp5+XUKVNTLxGRpxJC1FqJVmzOVRPVJNHp06fjl19+wcaNG+t0njlz5iA7O1t6Xbx40UEROkdJSXmiNP+cyFsh9+51eUhERK5T04LcVexnEq3CM888gy+//BJ79uyxeEYyLCwMxcXFyDI/+3HLtWvXqh1xp9PpEBQUZPFSs7Nny3+GzIPS5En04EHXx0RE5Cq1zlaECs28bM4tJ4TAM888gy1btuCbb75BVFSUxefdu3dHgwYNkJycLO1LTU3FhQsX0KdPH1eH6xRnz5Zvm39O5D8vt1YAIiLyTFYkUTWvKaro6Nzp06dj/fr1+OKLL9CwYUOpn1Ov18PPzw96vR6PPfYY4uLiEBISgqCgIMyYMQN9+vTxmJG58iTaseO/0Ly5AV5eOuzfD+TkmPpFhaj6+WMiIncnT4rminNNXBwMJSXQmWeakjXPqa05V9EkumrVKgDAgAEDLPYnJiZi0qRJAIBly5ZBq9VizJgxFpMteIoLF8q3IyMHoHlz07Zeb0qieXnAlSuQ9hMReZQqKtG7Y2IsDlFzc66iSdSaORB9fX2xYsUKrFixwgURuZ48iQYGlm/r9YB5TNS5c0yiROSZrOkTVXNzrioGFtVn8sHDtyZpAgA0bFi+LW/yJSLyJDYnUTbnktzly6b/6nRARsZelJWZ+kQDAwdIx6j8KR0iIvvJkqK52Xb/iRNSn+jdMTGWjywYDK6OsEZMogoSojyJBgQAe/Y8gvz8ywgIaI7Bgy9JxzGJEpGnqqoSnbJ0Ka5kZiKicWOkJSZa9ImyOZckmZnlf4TJm3Irvr90CUREnsma5lz56FyVVaJMogq6cqV829/f8jM/v/LHWq5edV1MRESuZFWfqDyJekIlepYjXRxCnhwrJlGNpnyfucmXiMjTiCr6RCuRPSgvioqcHZJN7Eqi0dHRuPfee/Gvf/0LRSr7B7mTmpKofN/160BZmWtiIiJyKWum/dNoyqtRT2jOPXr0KLp06YK4uDiEhYXhySefxKFDhxwdm8eTL0RT1Rqz5n1GI3DjhmtiIiJyJauacwEpiXpEJdqtWze88847uHLlCtasWYOrV6/irrvuQufOnbF06VJcv37d0XF6JHkSraoSlSfWalZ+IyJyb1YmUY05iRoMVk3U4yp1Gljk7e2N0aNH47PPPsPf//53pKWl4cUXX0RkZCQmTJiAqxwRUyNbkmhGhvPjISJyNSFbCq3aPlEA8L71RKbRaJl4FVanJHr48GFMmzYN4eHhWLp0KV588UWcOXMGu3fvxpUrVzBixAhHxemRrG3OBZhEicgz2dqcCwCisNCJEdnGrskWli5disTERKSmpmL48OFYu3Ythg8fDu2tGxAVFYWkpCS0bt3akbF6HHMSbdDA9KpInkSvXXNNTERELmVDc665EVcUFpomGFcBu5LoqlWrMGXKFEyaNAnh4eFVHtOsWTMkJCTUKThPZ06M5mQ5frzlrAryJMpuZiLyRBaV6K1HWdISEysf6F2erty+Et29ezdatmwpVZ5mQghcvHgRLVu2hI+PDyZOnOiQID2RwQBkZZm2q2rKBQBf3/JtJlEi8kiyPtEaK1GVJlG7+kTbtm2LG1U8c3Hz5k1ERUXVOaj6QN7HWV0SZZ8oEXk6qRLVaEzPg1ZH1idqLChwclTWsyuJVje8OC8vD77y8omqJe/jtKYSZRIlIo9kTqI1DSqCeitRm5pz4+LiAJhmj5g3bx78Zc9llJWV4eDBg+jWrZtDA/RUVSXRI0deR3FxNnx89OjefT60WtMSaQYDm3OJyDNJj7jIkuibGzYgp6AAQf7+eHXcONNOeRJVUSVqUxI9duwYAFMleuLECfj4+Eif+fj4oGvXrnjxxRcdG6GHkleW5r9FTp36WFoKrXv3+QBM1SiTKBF5rCoq0cRdu6Sl0MxJVOMJj7js2bMHADB58mS88847CAoKckpQ9YE1zbmAKYlmZwO5uaZkqtM5PzYiIlcxV6I19ocCqh2da1efaGJiIhNoHVkzsKjiZ6xGiciTCCGs7hN1+8kWRo8ejaSkJAQFBWH06NE1Hrt58+Y6B+bp5JVoTWOxKj7m0qKF82IiInIp+fJUtQ0s0mpNxxiN7plE9Xq9VG7rVTJThDuTJ9Gq5s01YyVKRB7LymdEJV5e7ptEE2UzSCRWNZsE2cScRL28qp7yz4wTLhCRp6pqtqKaaLy9IUpKVLUcml19ooWFhSiQDTH+/fffsXz5cuzatcthgXk6c5+or2/NPzusRInIY1m7gouZuV+0rMwyASvIriQ6YsQIrF27FgCQlZWFXr16YcmSJRgxYgRWrVrl0AA9UVlZ+SLbNTXlApxwgYg8l9UruJjJBxeppBq1K4kePXoU/fv3BwB8/vnnCAsLw++//461a9fi3XffdWiAnigz07QkHlDzyNyKnzOJEpEnETb2iWpUmETtmoC+oKAADRs2BADs2rULo0ePhlarxZ133onff//doQF6oupG5oaH34Oiohvw9W0i7WNzLhF5rGr6RPt37ozMnBw0rvgopTyJGgzOjs4qdlWi0dHR2Lp1Ky5evIidO3di8ODBAICMjAybnh/dv38/7r//fkRERECj0WDr1q0Wn0+aNAmaW5MSm19Dhw61J2RVqWq2IgAYOHAdhg/fiYED10n75EmWa4oSkSeprhJNfOEF/Of115H4wguWX1BhJWpXEp03bx5efPFFtG7dGr1790afPn0AmKrSP/3pT1afJz8/H127dsWKFSuqPWbo0KG4evWq9NqwYYM9IauKtbMVAZDmz634PSIityerRK0ZWKRRYSVqV3Pugw8+iLvuugtXr15F165dpf2DBg3CqFGjrD7PsGHDMGzYsBqP0el0CAsLsydM1bJ2tiL5MQYD+0SJyLPUZWARVFKJ2pVEASAsLKxScuvVq1edA6po7969aNasGRo1aoSBAwfijTfeQOPGjas93mAwwCD7CyUnJ8fhMdWVLZWo+ZisLKCgAMjPBwICnBYaEZHryJtzrXhOVI19onYl0fz8fLz99ttITk5GRkYGjOahprecPXvWIcENHToUo0ePRlRUFM6cOYNXXnkFw4YNw4EDB+Al/4tEJj4+Hq+//rpDru8s1SXRL78ciMLCa/DzC8Vf/vJNlcdcuwa0aeOCIImInKy6SnTYq68iIysLzYKD8dWbb0r7PaY5d+rUqdi3bx8effRRhIeH1z77vp3+7//+T9qOiYlBly5d0LZtW+zduxeDBg2q8jtz5syR1j0FTJVoZGSkU+KzV3XNudnZvyE//zKKi7MtjmcSJSKPVE2f6OkrV3AlMxPZFdcNVeHAIruS6FdffYVt27ahX79+jo6nRm3atEGTJk2QlpZWbRLV6XTQqXy9MHMlqtFYt7SZfARverpzYiIicjVb+0TliVYtlahdo3MbNWqEkJAQR8dSq0uXLiEzMxPh4eEuv7YjmZOor691fenySpRJlIg8hj0T0N/i1kl04cKFmDdvnsX8ufbIy8tDSkoKUlJSAADnzp1DSkoKLly4gLy8PLz00kv48ccfcf78eSQnJ2PEiBGIjo7GkCFD6nRdJQlRnkStGVQEWFaifMyFiDyFJ0z7Z1dz7pIlS3DmzBmEhoaidevWaFBhGZKjR49adZ7Dhw/j3nvvld6b+zInTpyIVatW4fjx4/jkk0+QlZWFiIgIDB48GAsXLlR9c21NsrOB4mLTtj1JlJUoEXkKm5OoCptz7UqiI0eOdMjFBwwYYFrZvBo7d+50yHXUxNbHWyoed/WqY+MhIlKMrZMtaDSmarSszL2fE50/f76j46g3mESJiExsrkQBKYmqpRK1q08UMC2Btnr1asyZMwc3b94EYGrGvXz5ssOC80T2JFGttvxYNucSkcewI4manxUVRUU1tmS6il2V6PHjxxEbGwu9Xo/z58/j8ccfR0hICDZv3owLFy5Ia41SZfIkWttaonL+/kBhoakSNRqt/6ONiEit7KpEvW+lLaPRlIR9fBwfmA3sSqJxcXGYNGkSFi1aJC2JBgDDhw/Hww8/7LDgPFFNlegdd8xDSUkeGjQIrPQ9f3/TOqSlpab/Nm3q5ECJiJxMmEdZ3lqly2zO2LHILypCgHwZKzP5CN3CQmjcMYn+9NNP+PDDDyvtb968OdLZ3lijmiaf79jxiWq/J69ar15lEiUiD2CuRCtUoY/VsOSlxssL5kZcUVgI6PVOCs46djUK6nS6Kid2/+2339CUv91rZE+fKFA5iRIRuTupErWlf6pCJao0u5LoAw88gAULFqDk1l8RGo0GFy5cwOzZszFmzBiHBuhpmESJiEykJFrNgiJV0XiXN6C6bRJdsmQJ8vLy0LRpUxQWFuKee+5BdHQ0GjZsiDdlM+5TZeYkqtNV/uOroOAq8vIuoaCgcpZkEiUiTyKEkGaeqfiM6NWbN3Hpxg1cvfXkhwVZEjXWcdY8R7CrT1Sv12P37t34/vvv8fPPPyMvLw933HEHYmNjHR2fx6lpyr8tW3oiP/8yAgKaY/z4SxafMYkSkUepYd7c/i+8gCuZmYho3BhpiYkWn1lUou6YRI1GI5KSkrB582acP38eGo0GUVFRCAsLgxDCacuieYL8fNPC2oBtTbmAZRK9csVxMRERKUFqygVs6xOVJ9H8fAdGZB+bmnOFEHjggQcwdepUXL58GTExMejUqRN+//13TJo0CaNGjXJWnB7B3v5QgJUoEXkW+YxDGnv7RN2tEk1KSsL+/fuRnJxsMXE8AHzzzTcYOXIk1q5diwkTJjg0SE9RlyTq5WXqRzUYOGsREXkAeSVqQxKFbMETo7tVohs2bMArr7xSKYECwMCBA/Hyyy9j3bp1DgvO09QliQLl1ejVq6Yl1YiI3JXF3Le2NOdqtcCtbkM1VKI2JdHjx49jaA0PwQ4bNgw///xznYPyVDVNtGAN83cKC4HcXMfERESkBHmfqE3NuRqN1C/qdn2iN2/eRGhoaLWfh4aG4o8//qhzUJ7KUZVoxXMREbkbuytRlPeLioICxSehtynysrIyeHtX343q5eWFUvmwZbIgr0RtmXzeTJ542S9KRG5NnkRt6RMFLCehV3hdUZsGFgkhMGnSJOh0uio/N6hkfTe1klePVc2rXBsmUSLyFPaOzgVMlai5/jQWFsLLnqY9B7EpiU6cOLHWYzgyt3p1rUTZnEtEnkLIK0h7K1EoP/WfTUk0scLMEWQbcxJt0MDiZ0By333JMBpLodVW/T+L/I8tJlEicmeihubc7QsXotRohHd1faXySegVHqFr17R/ZB9z4quuKTc4uH2N32cSJSJPYdGcWyFZtmvRosbvqmkSersmoCfblZQA5rmU7W2+lyff69frHhMRkVJqqkRrJa9EFR6LwyTqIjdulG/bm0Tl35P3rxIRuZu69InKByIJdxqdS/az5hnRtLT1KC0tgLe3P6KjH670uZcX4ONjmi2LSZSI3FoNz4lu2rcPBQYD/HU6jL3nnsrfZRKtf6yZrejgwVnSUmhVJVHA1KRbXMzmXCJyb1IzrJdXpdW/Xk1KkpZCqy2Jgs259UNdZyuq+N3sbMv5m4mI3IlUQdraHwrLgUhC4V+ETKIuIq9E7ZlooarvyvtZiYjcibkStXWiBQAcWFQf1XWiBTN5FcsmXSJyR6K0FCgrM71hErXf/v37cf/99yMiIgIajQZbt261+FwIgXnz5iE8PBx+fn6IjY3F6dOnlQm2jliJEhGZ1GXyeQDSUmhAPW/Ozc/PR9euXbFixYoqP1+0aBHeffddfPDBBzh48CACAgIwZMgQFCk8GssedV0GzYzPihKRu6vLvLnAreXQzN9TOIkqOjp32LBhGDZsWJWfCSGwfPlyzJ07FyNGjAAArF27FqGhodi6dSv+7//+r8rvGQwGi4nwc3JyHB+4HcxJVKMBqpm/3ypMokTk9uoy0YKZVguUldXvSrQm586dQ3p6OmJjY6V9er0evXv3xoEDB6r9Xnx8PPR6vfSKjIx0Rbi1MidRX1/7Wi/M5Ek0M7NuMRERKaHOzbmy7zGJViP91lpfFRcBDw0NlT6rypw5c5CdnS29Ll686NQ4rSFEeRKt64o97BMlIncnT3x2jc6Vf6+4WNGFuT1usgWdTlfteqdKyc0tXze2pkFFfn5hFv+tCpMoEbm72ubNDW3UyOK/VTJXsEKYRvpWtTSWC6g2iYaFmRLJtWvXEB4eLu2/du0aunXrplBU9pH3XdZUiY4efbjWc7FPlIjcXW3Nud8vXVr7SSpMuKBRKImqtjk3KioKYWFhSE5Olvbl5OTg4MGD6NOnj4KR2c5RI3MB01qk5p8d9okSkVtyRHOuPPmWlNQ1IrspWonm5eUhLS1Nen/u3DmkpKQgJCQELVu2xPPPP4833ngDt912G6KiovC3v/0NERERGDlypHJB20E+5V9dnhEFTKN7fX2BggImUSJyT3VaBs1MJVP/KZpEDx8+jHvvvVd6HxcXBwCYOHEikpKSMGvWLOTn5+OJJ55AVlYW7rrrLuzYsQO+dc1ELuao2YrMzEn0+nVTd0CFuZuJiFTNIaNz5bMW1ddKdMCAATWOqtJoNFiwYAEWLFjgwqgcz9rZivbvfxIGw03odCG4++4Pqz3OfA6DwZRMAwIcFCgRkQtYVI5VVKLPrFiBP3Jz0ahhQ7w/fXrVJ5En3/paidYX1vaJXry4TVoKrSYVR+gyiRKRW5H3iVZRie44fFhaCq06Fiu5KFiJqnZgkSdx5MAiwHLGIz7mQkTuxtF9okpWokyiLuCoyeerOgcfcyEid2PRnFvHGYsAVqIez5xEvbxMj6jUlbya5QhdInI3UiWq0VTZnGsVJtH6w1wt+vk5ZiQtK1EicmvmSrQOE4lr2JxbP5SVlfdbOqI/FGASJSL3JlWi9vaHAqxE64ubNwGj0bTtqMdbOX8uEbkzc5+o3U25AJNofeHokbkAkygRuS9hNAKlpaY3DqpElZz2j0nUyaydfN4WbM4lIrdVy0QL1lLLc6KcbMHJbKlE27YdB4PhD+h0NSz/A9OKP97epj/mmESJyJ3InxGtrjn3obvvxh95eWgUGFj9ieQJmDMWeS5bnhG9887FVp/Xz8+0TimTKBG5E2smWnhr8uTaT6SSSpTNuU7mjD5RoDwh37xpGgFMROQOHDJbEaCaVVyYRJ3M2UlUCE64QETuwyEruACWD92zEvVcjp7yz0yekNmkS0TuQhQVSdv2LsgNmFb5MifherueaH1gSyW6aVMHFBRcgb9/BMaOPVXjsfJzZWQAnTrVIUgiIlexojm329NP4+rNmwgPCUHKqlXVn8vLCzAa2ZzrycxJ1Men9ub/0tI8lJTkorQ0r9bzypPotWt1CJCIyIWsqUTzioqQW1iIPNmxVTI3BzOJei5zgnNkU27F87E5l4jchcMGFqH8ERlWoh6qqAjIyTFtO3JQEQD4+5dvsxIlInchr0TrmkSl7xuNEAo9psAk6kTOGpkLWFaiTKJE5C4cNbAIgCoec2ESdSJnJlF5JSq/DhGRmllUot51G9tqMeORvJnYhZhEnUheITqzEk1Pd+y5iYicRRQWlr9xVHMuWIl6JHkSlVeOjqDVlidSJlEichdSEtVqTc961oW8OZeVqOdxZiUqP+e1a6aZi4iI1E5KonVsygVgWYkyiXoeZydRc3VrMADZ2Y4/PxGRIwkhpCSqcUAS1ahgJRfOWOREtjbn3nXXBygrK4SXl3UZV37Oq1eB4GDb4iMiciVRVFTebFZDEn132jQUGQzw1elqPqG8Eq1tYgYnYRJ1InlfpTWVaKtWf7Hp/BWTaMeONn2diMilREGBtF3T4y3De/a07oTsE63Za6+9Bo1GY/Hq0KGD0mFZzVyJensDDRo4/vwVkygRkZqJ/PzyNw5uzmUlWo1OnTrh66+/lt57O6Iz2kXMlaijR+aayc975YpzrkFE5CgWlaiHDCxSfUby9vZGWFiY0mHYzGAA/vjDtG3toKLr14/AaCyGVuuDpk2713p8QED5NpMoEamdRSVaQ/Pc0bQ0lJSWooG3N+6Ijq72OFaiVjh9+jQiIiLg6+uLPn36ID4+Hi1btqz2eIPBAIPsL5Ic8+S1LibvD7W2Et21awTy8y8jIKA5xo+/VOvx8vNevmxjgERELma0shJ96M03cSUzExGNGyMtMbH6E8rOoVQSVXWfaO/evZGUlIQdO3Zg1apVOHfuHPr374/c3NxqvxMfHw+9Xi+9IiMjXRhxOXkfpSuac5lEiUjtHN0nqobRuapOosOGDcNf//pXdOnSBUOGDMH27duRlZWFTz/9tNrvzJkzB9nZ2dLr4sWLLoy4nD2VqK28vctnLWISJSK1c3SfqEarBW7NemQxnaALqb45Vy44OBjt2rVDWlpatcfodDroanu2yAXkfZTOSqKAqV+0qMiURI1GixHfRESqYm2fqE28vYGSEsWSqFv9ys3Ly8OZM2cQHh6udCi1ckVzLlA+uKi0lEuiEZG6Gc1JVKNx2F/85sFFTKJVePHFF7Fv3z6cP38eP/zwA0aNGgUvLy+MGzdO6dBqJW9edWYSDQws31ao5ZqIyCpSJertXffJ583MzcKlpRAlJY45py2Xd/kVbXDp0iWMGzcOmZmZaNq0Ke666y78+OOPaNq0qdKh1UrenCt/FMXR5En0wgWgVy/nXYuIyF5CCKlP1CHPiJrJBxcVFkLjjJltaqDqJLpx40alQ7CbuRLVagFndtFWTKJERKpUVGQauAE4ZmTuLRpvb5gXsRIFBUBQkMPObQ1VN+e6M3MSDQiQBo85hTyJ/v67865DRFQX1j4jajP5s6Kya7gKk6gT5OeXz1bkzKZcgEmUiNyDU0bmwjIhGxVIoqpuznVXl2STDdmSRP/6118BCADWl67+/qYmY6MROHfO+msREbmSPInWVokeW7HC+t+ECleiTKJOIB8lK68Ua+Pj09Dma2m1pmvk5JiSqBDObT4mIrKHRYKrJYk2tOGRBg2bcz2PPIk6uzkXABreyr25uUBmpvOvR0RkK6MNlahNmEQ9j7xvsqHtxaXN5Nc4e9b51yMispXD5829RelKlM25TiBPorY05x4/vhTFxTnw8QlCly5xVn9PPqL7zBk+K0pE6mMxb24tA4ve3boVOQUFCPL3x7MjR9Z8YvnAInmidhEmUSc4f75825ZK9MSJpdJSaLYkUb2+fLuGaYWJiBRjzMsrf1NLJfruF19IS6HVmkTNk9ALYVntugibc53A3KSq0wE+Ps6/njyJnj7t/OsREdlKSnBarUNXytBoNNIjM0yiHqCoqHxgkasmzpBf57ffXHNNIiJbCHMl6sh5c28x94uKggII86xILsIk6mBnz5oeMwEsK0Rn8vYu73tNTS2/PhGRGojSUmmVFafMbavg4CImUQc7dap825VTOAYHm/6blQVkZLjuukREtRHy/lAnJFF5YnZ1ky6TqIP9+mv5dqNGrruuOYkCwMmTrrsuEVFtjLm50rZTKlHZOeXXcgUmUQf75ZfybVcmUfm1mESJSE2MOTnStjOSqEUlKq96XYBJ1MGOHTP9V6u1rA6dTZ5ET5xw3XWJiGojsrLK3zjhkQUNK1HPkJNTPjo2JMSho7hrFRJSvp2S4rrrEhHVxihLohpnPPcnr0RlVa8rcLIFBzp4sHxkbLNmtn+/ceM7EBAQCV/fpjZ/18fHNJApJ8dUiZaVWSz4TkSkGKN5bUgAGp2u1uO7tW2LFk2aoImVjzhYVKJMou5rz57y7bAw278/dOh/6nT9Jk1MSbSgwDTAqXPnOp2OiMghyswrY2i1Vs2b+/ncubZdoEEDadYiY3a2HRHaj825DrRjR/l2RITrr9+kSfn2oUOuvz4RUUWipETqE9X4+jp8ogXg1qxFt5qJjX/8AeHCh+WZRB3k3LnyQUVNmpgWy3a10NDy7e+/d/31iYgqKpM9uK7x83PadaS+1uJil064wCTqIOvXl29HRSkTQ9Om5YOZvv1WmRiIiOSM6enStlOTqK9v+TWvX3fadSpin6gDlJUBCQnl76Oj7TvPjh0PoKjoOnx9m9rVP+rtbRrQlJ5umoj+4kUgMtK+WIiIHKH08mVpW2NlE92Db7yBG9nZaKLXW90/Kk+iZRkZ8G7d2qY47cVK1AG2bzc15wKmpGXvQtyZmUeRkfEjMjOP2h1L8+bl2199ZfdpiIgcokyeRK2sRFPOnMGh1FSknDlj9XXk5y67etX6AOuISdQBli0r3779duXiACwrz61bFQuDiAjGwkIYb/WJavz9oXHic3fyKrfs0iWnXaciJtE6OnSo/NEWvR5o2VLZeJo2BQICTNu7dwMu7BogIrJQZm6iA6AxLzXlJBqtVkqkxhs3XDZzEZNoHQgB/O1v5e+7dDE9qqQkjaa8T7a0FFi9Wtl4iKj+Kjl9WtrW2tvPZQONbOms0rQ0p18PYBKtk08/BXbtMm0HBgLt2ikbj1nHjuXbS5cCLn72mIgIoqQEJea1IbVaaFyQRLWyGY5KXLQSh1sk0RUrVqB169bw9fVF7969cUjhmQRKSoBVq4BJk8r39e6tnmn2goKAtm1N2zduAM89x4W6ici1io8eBYqKAADa4GBoXDCZuMbfX5p0ofTMGYvpBp1F9Ul006ZNiIuLw/z583H06FF07doVQ4YMQYYCK08bjcC6daaKc9o06ecDt90GtGnj8nBq1KtX+exan3xiivfWwvJERA4lhIAoLpZmCirLzETRN99In2vtmUzcDhqNBl6NG5uDQuHOnRBGo3OvKVw5P5IdevfujZ49e+L9998HABiNRkRGRmLGjBl4+eWXa/1+Tk4O9Ho9srOzESRrL7fW+fOmATq//AJ89FF54jQLDQX69XPMii3btrVAYeFl+Pk1x3331X102Zkz5bMomY0ZA9x5p+l50n79yitWIiJblF2/jtLz51F69ixKzc22ALwiIlB25Yr0XuPvD69WrWw6d7unn8aVmzcRERKC31atsjGwMpSal9MCoAkIgE/v3vCKiIB3mzYOn3ZQ1Um0uLgY/v7++PzzzzFy5Ehp/8SJE5GVlYUvvvii0ncMBgMMBoP0Pjs7Gy1btsTFixdtTqJbtgCTJ7uyKbQDgKsAwgGcquVYx1i5Ehg/3iWXIiIPYTh8GEW7dzvt/L3efx/peXkICwzEoWeecdh5vdu0QdjUqY5NpELFLl++LACIH374wWL/Sy+9JHr16lXld+bPny8A8MUXX3zxxVelV0ZGhkPzlMdN+zdnzhzExcVJ77OystCqVStcuHABeivXpqvvcnJyEBkZaVf1Xp/xvtmO98w+vG+2M98zHwcvCq7qJNqkSRN4eXnh2rVrFvuvXbuGsGoW7NTpdNBVseirXq/nD5uNgoKCeM/swPtmO94z+/C+2c7RfaKqHp3r4+OD7t27Izk5WdpnNBqRnJyMPn36KBgZERGRyitRAIiLi8PEiRPRo0cP9OrVC8uXL0d+fj4mT56sdGhERFTPqT6Jjh07FtevX8e8efOQnp6Obt26YceOHQiVr0BdA51Oh/nz51fZxEtV4z2zD++b7XjP7MP7Zjtn3TNVP+JCRESkZqruEyUiIlIzJlEiIiI7MYkSERHZiUmUiIjITh6RRG1dKu2zzz5Dhw4d4Ovri5iYGGzfvt1FkaqHLffs448/Rv/+/dGoUSM0atQIsbGxii9HpxR7l+XbuHEjNBqNxRzQ9YWt9ywrKwvTp09HeHg4dDod2rVrx/+PWnHfli9fjvbt28PPzw+RkZGYOXMmiiqumOHB9u/fj/vvvx8RERHQaDTYunVrrd/Zu3cv7rjjDuh0OkRHRyMpKcn2Czt0EkEFbNy4Ufj4+Ig1a9aI//3vf+Lxxx8XwcHB4tq1a1Ue//333wsvLy+xaNEicfLkSTF37lzRoEEDceLECRdHrhxb79nDDz8sVqxYIY4dOyZ+/fVXMWnSJKHX68WlS5dcHLmybL1vZufOnRPNmzcX/fv3FyNGjHBNsCph6z0zGAyiR48eYvjw4eK7774T586dE3v37hUpKSkujlxZtt63devWCZ1OJ9atWyfOnTsndu7cKcLDw8XMmTNdHLlytm/fLl599VWxefNmAUBs2bKlxuPPnj0r/P39RVxcnDh58qR47733hJeXl9ixY4dN13X7JNqrVy8xffp06X1ZWZmIiIgQ8fHxVR7/0EMPifvuu89iX+/evcWTTz7p1DjVxNZ7VlFpaalo2LCh+OSTT5wVoirZc99KS0tF3759xerVq8XEiRPrXRK19Z6tWrVKtGnTRhQXF7sqRFWy9b5Nnz5dDBw40GJfXFyc6Nevn1PjVCtrkuisWbNEp06dLPaNHTtWDBkyxKZruXVzbnFxMY4cOYLY2Fhpn1arRWxsLA4cOFDldw4cOGBxPAAMGTKk2uM9jT33rKKCggKUlJQgJCTEWWGqjr33bcGCBWjWrBkee+wxV4SpKvbcs//85z/o06cPpk+fjtDQUHTu3BlvvfUWysrKXBW24uy5b3379sWRI0ekJt+zZ89i+/btGD58uEtidkeOygWqn7GoJjdu3EBZWVml2YtCQ0Nx6lTV63Gmp6dXeXx6errT4lQTe+5ZRbNnz0ZERESlH0BPZs99++6775CQkICUlBQXRKg+9tyzs2fP4ptvvsH48eOxfft2pKWlYdq0aSgpKcH8+fNdEbbi7LlvDz/8MG7cuIG77roLQgiUlpbiqaeewiuvvOKKkN1SdbkgJycHhYWF8PPzs+o8bl2Jkuu9/fbb2LhxI7Zs2QJfX1+lw1Gt3NxcPProo/j444/RpEkTpcNxG0ajEc2aNcNHH32E7t27Y+zYsXj11VfxwQcfKB2aqu3duxdvvfUWVq5ciaNHj2Lz5s3Ytm0bFi5cqHRoHs+tK1F7lkoLCwuz6XhPY889M/vHP/6Bt99+G19//TW6dOnizDBVx9b7dubMGZw/fx7333+/tM9oNAIAvL29kZqairZt2zo3aIXZ87MWHh6OBg0awMvLS9rXsWNHpKeno7i42OFrQaqRPfftb3/7Gx599FFMnToVABATE4P8/Hw88cQTePXVV6HVsl6qqLpcEBQUZHUVCrh5JWrPUml9+vSxOB4Adu/eXW+WVrN3eblFixZh4cKF2LFjB3r06OGKUFXF1vvWoUMHnDhxAikpKdLrgQcewL333ouUlBRERka6MnxF2POz1q9fP6SlpUl/cADAb7/9hvDw8HqRQAH77ltBQUGlRGn+Q0RwevQqOSwX2DbmSX02btwodDqdSEpKEidPnhRPPPGECA4OFunp6UIIIR599FHx8ssvS8d///33wtvbW/zjH/8Qv/76q5g/f369fMTFlnv29ttvCx8fH/H555+Lq1evSq/c3Fyl/gmKsPW+VVQfR+faes8uXLggGjZsKJ555hmRmpoqvvzyS9GsWTPxxhtvKPVPUISt923+/PmiYcOGYsOGDeLs2bNi165dom3btuKhhx5S6p/gcrm5ueLYsWPi2LFjAoBYunSpOHbsmPj999+FEEK8/PLL4tFHH5WONz/i8tJLL4lff/1VrFixon4+4iKEEO+9955o2bKl8PHxEb169RI//vij9Nk999wjJk6caHH8p59+Ktq1ayd8fHxEp06dxLZt21wcsfJsuWetWrUSACq95s+f7/rAFWbrz5pcfUyiQth+z3744QfRu3dvodPpRJs2bcSbb74pSktLXRy18my5byUlJeK1114Tbdu2Fb6+viIyMlJMmzZN/PHHH64PXCF79uyp8veU+T5NnDhR3HPPPZW+061bN+Hj4yPatGkjEhMTbb4ul0IjIiKyk1v3iRIRESmJSZSIiMhOTKJERER2YhIlIiKyE5MoERGRnZhEiYiI7MQkSkREZCcmUSIiIjsxiRIREdmJSZRIAQcOHICXlxfuu+8+pUMhojrgtH9ECpg6dSoCAwORkJCA1NRUREREKBJHfVlejMhZWIkSuVheXh42bdqEp59+Gvfddx+SkpIsPv/vf/+Lnj17wtfXF02aNMGoUaOkzwwGA2bPno3IyEjodDpER0cjISEBAJCUlITg4GCLc23duhUajUZ6/9prr6Fbt25YvXo1oqKipIXVd+zYgbvuugvBwcFo3Lgx/vKXv+DMmTMW57p06RLGjRuHkJAQBAQEoEePHjh48CDOnz8PrVaLw4cPWxy/fPlytGrVymJZMyJPwyRK5GKffvopOnTogPbt2+ORRx7BmjVrpDUft23bhlGjRmH48OE4duwYkpOT0atXL+m7EyZMwIYNG/Duu+/i119/xYcffojAwECbrp+WloZ///vf2Lx5M1JSUgAA+fn5iIuLw+HDh5GcnAytVotRo0ZJCTAvLw/33HMPLl++jP/85z/4+eefMWvWLBiNRrRu3RqxsbFITEy0uE5iYiImTZrEBaHJs9Vp7Rkislnfvn3F8uXLhRCmJayaNGki9uzZI4QQok+fPmL8+PFVfi81NVUAELt3767y88TERKHX6y32bdmyRcj/b25ePzcjI6PGGK9fvy4ASOvsfvjhh6Jhw4YiMzOzyuM3bdokGjVqJIqKioQQQhw5ckRoNBpx7ty5Gq9D5O74JyKRC6WmpuLQoUMYN24cAMDb2xtjx46VmmRTUlIwaNCgKr+bkpICLy8v3HPPPXWKoVWrVmjatKnFvtOnT2PcuHFo06YNgoKC0Lp1awDAhQsXpGv/6U9/QkhISJXnHDlyJLy8vLBlyxYApqble++9VzoPkafyVjoAovokISEBpaWlFgOJhBDQ6XR4//334efnV+13a/oMALRardQsbFZSUlLpuICAgEr77r//frRq1Qoff/wxIiIiYDQa0blzZxQXF1t1bR8fH0yYMAGJiYkYPXo01q9fj3feeafG7xB5AlaiRC5SWlqKtWvXYsmSJUhJSZFeP//8MyIiIrBhwwZ06dIFycnJVX4/JiYGRqMR+/btq/Lzpk2bIjc3F/n5+dI+c59nTTIzM5Gamoq5c+di0KBB6NixI/744w+LY7p06YKUlBTcvHmz2vNMnToVX3/9NVauXInS0lKMHj261msTuT2l25OJ6ostW7YIHx8fkZWVVemzWbNmiR49eog9e/YIrVYr5s2bJ06ePCmOHz8u3n77bem4SZMmicjISLFlyxZx9uxZsWfPHrFp0yYhhBCZmZkiICBAPPvssyItLU2sW7dOREREVOoT7dq1q8W1y8rKROPGjcUjjzwiTp8+LZKTk0XPnj0FALFlyxYhhBAGg0G0a9dO9O/fX3z33XfizJkz4vPPPxc//PCDxbn69u0rfHx8xFNPPeWgu0akbqxEiVwkISEBsbGx0Ov1lT4bM2YMDh8+jJCQEHz22Wf4z3/+g27dumHgwIE4dOiQdNyqVavw4IMPYtq0aejQoQMef/xxqfIMCQnBv/71L2zfvh0xMTHYsGEDXnvttVrj0mq12LhxI44cOYLOnTtj5syZWLx4scUxPj4+2LVrF5o1a4bhw4cjJiYGb7/9Nry8vCyOe+yxx1BcXIwpU6bYcYeI3A8nWyAih1m4cCE+++wzHD9+XOlQiFyClSgR1VleXh5++eUXvP/++5gxY4bS4RC5DJMoEdXZM888g+7du2PAgAFsyqV6hc25REREdmIlSkREZCcmUSIiIjsxiRIREdmJSZSIiMhOTKJERER2YhIlIiKyE5MoERGRnZhEiYiI7PT/AlWBT73nAugAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "---Cypher-RAG based mean and std---\n", + "0.11178666666666666\n", + "0.018260066934281605\n", + "\n", + "---KG-RAG based mean and std---\n", + "0.9459866666666666\n", + "0.013465339868632277\n" + ] + } + ], + "source": [ + "cypher_rag_vs_kg_rag_fig = plot_figure(cypher_rag_correct_frac_list, kg_rag_correct_frac_list)\n", + "\n", + "fig_path = '../data/results/figures'\n", + "os.makedirs(fig_path, exist_ok=True)\n", + "cypher_rag_vs_kg_rag_fig.savefig(os.path.join(fig_path, 'cypher_rag_vs_kg_rag_true_false.svg'), format='svg', bbox_inches='tight') \n", + "\n", + "print('---Cypher-RAG based mean and std---')\n", + "print(np.mean(cypher_rag_correct_frac_list))\n", + "print(np.std(cypher_rag_correct_frac_list))\n", + "print('')\n", + "print('---KG-RAG based mean and std---')\n", + "print(np.mean(kg_rag_correct_frac_list))\n", + "print(np.std(kg_rag_correct_frac_list))\n" + ] + }, + { + "cell_type": "markdown", + "id": "e2611e5f-ffcc-4e36-8d24-7097bd017db3", + "metadata": {}, + "source": [ + "## Missed questions by Cypher-rag" + ] + }, + { + "cell_type": "code", + "execution_count": 75, + "id": "edc899fb-d41a-4ec0-b89b-fce27a0cc2df", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Cypher-RAG missed 72.02572347266882% of questions\n" + ] + } + ], + "source": [ + "missed_question_percentage = 100*cypher_rag[cypher_rag.cypher_rag_final_answer.isna()].shape[0]/cypher_rag.shape[0]\n", + "print(f\"Cypher-RAG missed {missed_question_percentage}% of questions\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 79, + "id": "19b610c1-4e9a-464a-8646-c548dc1e71c6", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
textlabelcypher_rag_answercypher_rag_final_answertotal_tokens
0enhanced S-cone syndrome is not a vitreoretina...FalseNaNNaN3817
1metronidazole treats crohn's diseaseTrueI'm sorry, but I don't have the information to...NaN4049
2KLEEFSTRA SYNDROME 1 is not associated with Ge...FalseNaNNaN3861
3Juvenile polyposis syndrome associates Gene SMAD4TrueI'm sorry, but I don't have the information to...NaN4054
4Disease ontology identifier for congenital gen...FalseNaNNaN3856
..................
304Noonan Syndrome associates Gene SOS1TrueI'm sorry, but I don't have the information to...NaN4044
306Congenital amegakaryocytic thrombocytopenia is...FalseI'm sorry, but I don't have the information to...NaN4090
307Leigh Disease associates Gene NDUFS4TrueNaNNaN3908
308Sandhoff Disease is not associated with Gene HEXBFalseI'm sorry, but I don't have the information to...NaN4064
310Juvenile polyposis syndrome associates Gene BM...TrueI'm sorry, but I don't have the information to...NaN4065
\n", + "

224 rows × 5 columns

\n", + "
" + ], + "text/plain": [ + " text label \\\n", + "0 enhanced S-cone syndrome is not a vitreoretina... False \n", + "1 metronidazole treats crohn's disease True \n", + "2 KLEEFSTRA SYNDROME 1 is not associated with Ge... False \n", + "3 Juvenile polyposis syndrome associates Gene SMAD4 True \n", + "4 Disease ontology identifier for congenital gen... False \n", + ".. ... ... \n", + "304 Noonan Syndrome associates Gene SOS1 True \n", + "306 Congenital amegakaryocytic thrombocytopenia is... False \n", + "307 Leigh Disease associates Gene NDUFS4 True \n", + "308 Sandhoff Disease is not associated with Gene HEXB False \n", + "310 Juvenile polyposis syndrome associates Gene BM... True \n", + "\n", + " cypher_rag_answer \\\n", + "0 NaN \n", + "1 I'm sorry, but I don't have the information to... \n", + "2 NaN \n", + "3 I'm sorry, but I don't have the information to... \n", + "4 NaN \n", + ".. ... \n", + "304 I'm sorry, but I don't have the information to... \n", + "306 I'm sorry, but I don't have the information to... \n", + "307 NaN \n", + "308 I'm sorry, but I don't have the information to... \n", + "310 I'm sorry, but I don't have the information to... \n", + "\n", + " cypher_rag_final_answer total_tokens \n", + "0 NaN 3817 \n", + "1 NaN 4049 \n", + "2 NaN 3861 \n", + "3 NaN 4054 \n", + "4 NaN 3856 \n", + ".. ... ... \n", + "304 NaN 4044 \n", + "306 NaN 4090 \n", + "307 NaN 3908 \n", + "308 NaN 4064 \n", + "310 NaN 4065 \n", + "\n", + "[224 rows x 5 columns]" + ] + }, + "execution_count": 79, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cypher_rag[cypher_rag.cypher_rag_final_answer.isna()]" + ] + }, + { + "cell_type": "code", + "execution_count": 85, + "id": "6e69e08b-daff-4cda-948e-f623ad1c5960", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "\"I'm sorry, but I don't have the information to answer that question.\"" + ] + }, + "execution_count": 85, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cypher_rag[cypher_rag.cypher_rag_final_answer.isna()].iloc[219].cypher_rag_answer" + ] + }, + { + "cell_type": "code", + "execution_count": 80, + "id": "62bb42fd-cc62-4562-ba1c-406717fa31ed", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "index 304\n", + "label True\n", + "question Noonan Syndrome associates Gene SOS1\n", + "llm_answer {\\n \"answer\": \"True\"\\n}\n", + "extracted_answer True\n", + "Name: 304, dtype: object" + ] + }, + "execution_count": 80, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "kg_rag.iloc[304]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "faa4837a-9052-4350-9038-67ddcb1de8c5", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python (cypher_rag)", + "language": "python", + "name": "cypher_rag" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} -- cgit v1.2.3