diff options
Diffstat (limited to 'notebooks/mcq_cypher_rag_vs_kg_rag.ipynb')
| -rw-r--r-- | notebooks/mcq_cypher_rag_vs_kg_rag.ipynb | 233 |
1 files changed, 233 insertions, 0 deletions
diff --git a/notebooks/mcq_cypher_rag_vs_kg_rag.ipynb b/notebooks/mcq_cypher_rag_vs_kg_rag.ipynb new file mode 100644 index 0000000..fac059f --- /dev/null +++ b/notebooks/mcq_cypher_rag_vs_kg_rag.ipynb @@ -0,0 +1,233 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "2712dac1-b37c-41b9-867b-6f53e5f1da33", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import os\n", + "from tqdm import tqdm\n", + "import re\n", + "from scipy import stats\n", + "import seaborn as sns\n", + "import matplotlib.pyplot as plt\n", + "\n", + "from IPython.display import clear_output\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "30cdc471-b72e-4f73-9086-909913988e6b", + "metadata": {}, + "outputs": [], + "source": [ + "def extract_answer(text):\n", + " try:\n", + " text_processed = text.split('\"answer\":')[-1].split('\\n')[0].strip().split('\"')[1].strip()\n", + " except:\n", + " text_processed = text.split('\"answer\":')[-1].split('\\n')[0].strip()\n", + " return text_processed\n", + "\n", + "def correct_paranthesis_split(text):\n", + " try:\n", + " text_processed = text.split('\"answer\":')[-1].split(\"\\n\")[1].split(\":\")[-1].split('\"')[1].strip()\n", + " return text_processed\n", + " except:\n", + " return text\n", + " \n", + "def process_df(rag_response_df):\n", + " rag_response_df.loc[:, 'extracted_answer'] = rag_response_df['llm_answer'].apply(extract_answer)\n", + "\n", + "\n", + " rag_response_df_paranthesis_split = rag_response_df[rag_response_df.extracted_answer==\"{\"]\n", + " if rag_response_df_paranthesis_split.shape[0] > 0:\n", + " rag_response_df_paranthesis_split.loc[:, \"extracted_answer\"] = rag_response_df_paranthesis_split.llm_answer.apply(correct_paranthesis_split)\n", + " rag_response_df_wo_paranthesis_split = rag_response_df[rag_response_df.extracted_answer != \"{\"]\n", + " rag_response_df = pd.concat([rag_response_df_wo_paranthesis_split, rag_response_df_paranthesis_split])\n", + " return rag_response_df\n", + "\n", + "def evaluate(df):\n", + " correct = df[df.correct_answer == df.extracted_answer]\n", + " incorrect = df[df.correct_answer != df.extracted_answer]\n", + " correct_frac = correct.shape[0]/df.shape[0]\n", + " incorrect_frac = incorrect.shape[0]/df.shape[0]\n", + " return correct_frac, incorrect_frac\n", + "\n", + "def evaluate_2(df):\n", + " correct = df[df.cypher_rag_eval == True]\n", + " incorrect = df[df.cypher_rag_eval == False]\n", + " correct_frac = correct.shape[0]/df.shape[0]\n", + " incorrect_frac = incorrect.shape[0]/df.shape[0]\n", + " return correct_frac, incorrect_frac\n", + "\n", + "\n", + "def bootstrap(cypher_rag, kg_rag, niter=1000, nsample=150):\n", + " cypher_rag_correct_frac_list = []\n", + " kg_rag_correct_frac_list = []\n", + " for i in tqdm(range(niter)):\n", + " cypher_rag_response_df_sample = cypher_rag.sample(n=nsample, random_state=i)\n", + " cypher_rag_correct_frac, cypher_rag_incorrect_frac = evaluate_2(cypher_rag_response_df_sample)\n", + " kg_rag_response_df_sample = kg_rag.iloc[cypher_rag_response_df_sample.index]\n", + " kg_rag_correct_frac, kg_rag_incorrect_frac = evaluate(kg_rag_response_df_sample)\n", + " cypher_rag_correct_frac_list.append(cypher_rag_correct_frac)\n", + " kg_rag_correct_frac_list.append(kg_rag_correct_frac)\n", + " return cypher_rag_correct_frac_list, kg_rag_correct_frac_list\n", + "\n", + "def plot_figure(cypher_rag_correct_frac_list, kg_rag_correct_frac_list):\n", + " fig = plt.figure(figsize=(5, 3))\n", + " ax = plt.gca()\n", + " sns.kdeplot(cypher_rag_correct_frac_list, color=\"blue\", shade=True, label=\"Cypher-RAG\", ax=ax, lw=2, linestyle=\"-\", alpha=0.6)\n", + " sns.kdeplot(kg_rag_correct_frac_list, color=\"lightcoral\", shade=True, label=\"KG-RAG\", ax=ax, lw=2, linestyle=\"-\", alpha=0.6)\n", + "\n", + " for artist in ax.lines:\n", + " artist.set_edgecolor(\"black\")\n", + " plt.xlabel(\"Accuracy\")\n", + " plt.ylabel(\"Density\")\n", + " plt.legend(bbox_to_anchor=(0.45, 0.9))\n", + " plt.xlim(0.1,0.9)\n", + " ax.axvline(np.mean(cypher_rag_correct_frac_list), color='black', linestyle='--', lw=2)\n", + " ax.axvline(np.mean(kg_rag_correct_frac_list), color='black', linestyle='--', lw=2)\n", + " sns.despine(top=True, right=True)\n", + " plt.show()\n", + " return fig\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "2c03e505-4232-4aa0-96db-9238aa4eb285", + "metadata": {}, + "outputs": [], + "source": [ + "cypher_rag = pd.read_csv('../data/results/cypher_rag_mcq_output_with_eval.csv')\n", + "kg_rag = pd.read_csv('../data/results/gpt_4_PubMedBert_entity_recognition_based_node_retrieval_rag_based_mcq_from_monarch_and_robokop_response.csv')\n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "1864df4c-9d34-4dd8-84b0-86c145bd86f1", + "metadata": {}, + "outputs": [], + "source": [ + "kg_rag = process_df(kg_rag)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "82a61695-9db0-438b-895f-72dd8dbe6698", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████| 1000/1000 [00:00<00:00, 1203.58it/s]\n" + ] + } + ], + "source": [ + "cypher_rag_correct_frac_list, kg_rag_correct_frac_list = bootstrap(cypher_rag, kg_rag)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "e4b076c6-d63b-46ab-a44e-42b0a141cdad", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/var/folders/p1/h56gxdhs5vgb0ztp7h4z606h0000gn/T/ipykernel_63840/889265924.py:56: FutureWarning: \n", + "\n", + "`shade` is now deprecated in favor of `fill`; setting `fill=True`.\n", + "This will become an error in seaborn v0.14.0; please update your code.\n", + "\n", + " sns.kdeplot(cypher_rag_correct_frac_list, color=\"blue\", shade=True, label=\"Cypher-RAG\", ax=ax, lw=2, linestyle=\"-\", alpha=0.6)\n", + "/var/folders/p1/h56gxdhs5vgb0ztp7h4z606h0000gn/T/ipykernel_63840/889265924.py:57: FutureWarning: \n", + "\n", + "`shade` is now deprecated in favor of `fill`; setting `fill=True`.\n", + "This will become an error in seaborn v0.14.0; please update your code.\n", + "\n", + " sns.kdeplot(kg_rag_correct_frac_list, color=\"lightcoral\", shade=True, label=\"KG-RAG\", ax=ax, lw=2, linestyle=\"-\", alpha=0.6)\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAdEAAAEmCAYAAAA9VKo6AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/TGe4hAAAACXBIWXMAAA9hAAAPYQGoP6dpAABSjUlEQVR4nO3dd3xT9foH8M9J2qY7HbS0hQKFInjZUwEVkF4RUBneKyIbNzigKsh14IWLRbkgKMNBaUURcIDXnwqClSEqo5SyKZRNoZRSuts04/v74zQnJ91JTs5J2uf9euXFyTxPGton3/V8OcYYAyGEEEJsplI6AEIIIcRdURIlhBBC7ERJlBBCCLETJVFCCCHETpRECSGEEDtREiWEEELsREmUEEIIsRMlUUIIIcROjT6JMsZQWFgIqilBCCFEah5KB+BsRUVF0Gq1KCgoQGBgoNLhEOKSli5disLCQgQGBiI+Pl7pcEgD0GfmGrjGXvavsLCQkigh9WjZsiWysrLQokULXL16VelwSAPQZ+YaGn13LiGEEOIslEQJIYQQO1ESJYQQQuxESZQQQgixEyVRQgghxE6URAkhhBA7URIlhBBC7ERJlBBCCLFTo69YRAip34gRI5CXl4eQkBClQyEN9GD37sjVahHs5wf96dPw7NhR6ZCaJKpYRAghbkafmYnS9estN2g0CHj5Zah8fJQLqomi7lxCCHEzut27q9ygg+HUKWWCaeIoiRJCiBsxZmfDWEOtXP3p0wpEQyiJEtKE6fXAn38CxcVKR0IaquLIEeFYHR0NePBTWwxXrtCWjwqgiUWENFG3bwNDhgCHDwOenr0REpKNli0jkJqaqnRopBaMMehPnAAADPr0U+SUlyPc1xe7nnoKKC+HKTcX6rAwhaNsWiiJEtJEJSTwCRQA9Pps3LiRZW7UEBdlvHoVrKgIAJBTUoJrBQWAyWS5PzubkqjMqDuXkCaovBz45JPqtxsM8sdCGs5q3FNV+eeb44SbTDk5MkdEFE2ie/bswcMPP4yoqChwHIfvv/++1sc+99xz4DgOy5Ytky0+QhqrP/4ACgur315eLn8spOEMGRmWK+bkKUqixps3ZY6IKJpES0pK0K1bN6xcubLOx23ZsgX79u1DVFSUTJER0rht32457tnTckxJ1HUZc3NhunULAMD5+1sn0cpjU26uUuE1WYqOgAwbNgzDhg2r8zFZWVl48cUX8csvv2DEiBEyRUZI4/bHH5bjTp34sVHGAJ2O79KlsVHXYzhzRjhWabVW93EaDVh5OUz5+WAmEzgVjdTJxaV/0iaTCRMnTsRrr72GTp06Neg5Op0OhYWFVhdCiIXRaJlQFBAA+PhYhtcAoHLyJ3Ex+rNnheOqSRQaDf+v0QhGf/Nk5dJJ9L333oOHhwdeeumlBj8nISEBWq1WuERHRzsxQkLcT0YGUFrKHzdrxv8rTqK0wsX1MJ0OxsuX+SsajSVpVuJE1023b8sZWpPnskn00KFDWL58OZKTk8GJBs7rM3fuXBQUFAiXK1euODFKQtxPerrlODSU/1etttx26JCs4ZAGMFy4ICxlUQUGVvubSElUOS6bRH///Xfk5OSgVatW8PDwgIeHBy5duoRXXnkFbdq0qfV5Go0GgYGBVhdCiIW4xKp50xbqznVthnPnhGNVDX/TOC8v4diUny9HSKSSy04fmDhxIuLi4qxuGzp0KCZOnIipU6cqFBUh7k+81DA4mP/37rvfx969paio8MWJE/wkIxs6gIiTGS5cEI45f38AwMIpU1Cq08FXowHESbSgQPb4mjJFk2hxcTEyMzOF6xcuXEB6ejpCQkLQqlUrhJr7mip5enoiIiICHTp0kDtUQhoNc0tUpeInFgFAbOwTyMgAsrKAW7eAmzeB8HDlYiQWpsJCy9IWPz9wlX3vYwcOFB7DjEbL4ymJykrR7tzU1FT06NEDPXr0AADEx8ejR48eePvtt5UMi5BGy2gEzJM8tVrrblxzqxQATp6UNy5SO8OlS8IxZ/7WUwWnVgsD29SdKy9FW6KDBg2yadeBixcvOi8YQpqAq1eBigr+uOoqCfH1s2eBQYNkC4vUwShKoqrKrtyacF5eYGVlYEVFtFZURi47JtqYGY1G6PV6pcMgCvPy8oJK5j90ovkpEM9Pyc/PgEplAP8noQNESxKJwgyivUM5Pz/h+MzVqzCYTPBQqXBHy5b8uGhZGWAygRUXg6NJlbKgJCojxhiys7ORT90tBIBKpUJMTAy8RJNCnK22JPrTT0NQUpIFoAWAqxAVxyEKYjodTDduAAA4Hx9hPBQAhr/1Fq7duoWo0FBkJiWB8/SEuV/PVFhY4yxeIj1KojIyJ9Dw8HD4+vratP6VNC4mkwnXrl3D9evX0apVK9n+L5w/bzmuZXgNAKgl6iKM164Jx+JWaE3Ey1xYQQHQsqXT4iIWlERlYjQahQRaddYxaZrCwsJw7do1GAwGeHp6ynJOcRKtqaHCcfzylvPnaZmLKzBmZQnH9SVRq2UuVPpPNjTyLBPzGKivr6/CkRBXYe7GNYqWJzibaI4KapqjYh6iLS8HsrPliYnUznj9unDM1fO3gxN9EaMkKh9KojKjLlxipsT/BfMEd19f61J/ZuKQxK1Wogyj+ZsMx4Hz9q7zsVbduZREZUNJlJAmorwcqJyjUut4qHiyMCVRZTGdDqa8PACVk4rq+9JFLVFF0Jiownr3VqbbLCLCNXfraNOmDWbOnImZM2cqHUqjY94EBKi5KxeglqgrMebkCMf1deUC4NeFengABgNVLZIRtUQVlp3Nl1qT+2JP4s7OzsaLL76Itm3bQqPRIDo6Gg8//DBSUlKk/8HIYNCgQeA4DhzHwdvbG3fccQcSEhJqLADy119/Qa1W17oxfEVFBRYvXoyePXvCz88PWq0W3bp1w5tvvolrohmWSqpvPBSwbomKH0/kZzR3GwD1duUKj6vs0mXFxWCVu74Q56KWqIvgOH6cytlKS/lZl7a6ePEiBgwYgKCgICxevBhdunSBXq/HL7/8ghkzZuC0uKq5i6moqKh1LebTTz+N+fPnQ6fT4bfffsMzzzyDoKAgPP/881aPS0xMxIsvvojExERcu3YNUVFRwn06nQ4PPPAAjh49in//+98YMGAAwsLCcOHCBWzYsAEfffQREhISnPoeG8LWligVCFOWSdwS9fFp2JPMXbqMgRUVgatalopIjpKoi/D1BcaPd/551q8HSkpsf9706dPBcRwOHDgAP9FU+06dOmHatGmYNm0acnJy8OOPPwr36fV6tGjRAgkJCXjyyScxaNAgdO7cGQDwxRdfwNPTE88//zzmz59vNd5TWlqKadOm4ZtvvkFwcDDefPNNPPPMM8L9V65cwSuvvILt27dDpVLh3nvvxfLly4Ut8qZMmYL8/Hz06dMHK1euhEajwQXRLhhivr6+iIiIAABMnToVK1aswI4dO6ySaHFxMTZt2oTU1FRkZ2cjOTkZ//rXv4T7P/jgA+zdu1eoBW3WqlUrDBw40KbSls4k3lq3ttUSHMevlKiooJao0ow3bwrHDU2inJeXdcEFSqJOR925pF55eXnYtm0bZsyYYZVAzYKCgvDUU09h27ZtuC6akv/jjz+itLQUY8eOFW77/PPP4eHhgQMHDmD58uVYunQp1qxZY/V6S5YsQe/evXH48GFMnz4dzz//PDIyMgDwiXno0KEICAjA77//jj/++AP+/v548MEHUWEuCgsgJSUFGRkZ2LFjh1Virw1jDL///jtOnz5drdX69ddfo2PHjujQoQMmTJiAtWvXWiXGDRs24O9//7tVAhVzlRnZ4iRatSU6evRBPPHEFYwefVC478oVvmA9UYYpN5c/8PAA51G9vfP7kiU4s3Ytfl+yRLjNapkLjYvKgpIoqVdmZiYYY+jYsWOtj+nfvz86dOiAL774QrgtKSkJ//znP+Ev+osdHR2NDz74AB06dMD48ePx4osv4oMPPrB6reHDh2P69OmIjY3FnDlz0KxZM+zcuRMAsGnTJphMJqxZswZdunTBnXfeiaSkJFy+fBm7du0SXsPPzw9r1qxBp06d0KlTp1rjXrVqFfz9/aHRaHDffffBZDLhpZdesnpMYmIiJkyYAAB48MEHUVBQgN27dwv3nzlzptr2fKNHj4a/vz/8/f3Rv3//Ws8vJ1EJ1mpJ1Nc3Ev7+LeHrGynM3NXrAdF3IiIjU1kZWHExgNrHQyNDQtCyWTNEmndWB6wKLtAyF3lQEiX1amh35FNPPYWkpCQAwI0bN7B161ZMmzbN6jF33323VcusX79+OHv2rFXBga5duwrHHMchIiICOZXjQ0eOHEFmZiYCAgKEJBUSEoLy8nKcExWG7dKli9CiXL9+vfBYf39//P7778Ljxo8fj/T0dPzxxx8YNmwY3njjDaukl5GRgQMHDmDcuHEAAA8PD4wdOxaJiYl1/ixWrVqF9PR0TJs2DaWlpQ36+TmbuSWqVgMaTe2PEydY8TgqkY/QCkXDJxUB1mtFqSUqDxoTJfVq3749OI6rd/LQpEmT8Prrr+Ovv/7Cn3/+iZiYGNx77702n69qCTyO42CqnGlYXFyMXr16Yf369dWeFxYWJhyLu50feeQR3HXXXcL1Fi1aCMdarRaxsbEA+G7b2NhY3H333YiLiwPAt0INBoPVRCLGGDQaDVasWAGtVov27dsL3c1mkZGRAIAQcStBYeYk6u9fdzm/qknURRrSTYp5E26AkqiroyRK6hUSEoKhQ4di5cqVeOmll6qNi+bn5yMoKAihoaEYNWoUkpKS8Ndff2Hq1KnVXmv//v1W1/ft24f27dtDXVP5nBr07NkTmzZtQnh4OAIbuEtFQEAAAuqqtl7J398fL7/8Ml599VUcPnwYRqMR69atw5IlS/DAAw9YPXbUqFHYsGEDnnvuOYwbNw5vvvkmDh8+XOu4qNIKC4HK3sEaJxWdOvUp9PpieHr6w9/fMomLWqLKsEqitXQbJG7bhpLycvh5e+PJBx/kbxSPidJuUbKgJOoiSkv5mbNynMceK1euxIABA9C3b1/Mnz8fXbt2hcFgwI4dO7B69WqcOnUKAN+l+9BDD8FoNGLy5MnVXufy5cuIj4/Hs88+i7S0NHz00UdYIpoYUZ/x48dj8eLFGDlyJObPn4+WLVvi0qVL2Lx5M2bPno2WDu5c8eyzz2LBggX47rvv4OHhgdu3b+PJJ5+Etsosx0cffRSJiYl47rnnMGvWLPz0008YMmQI5s2bh3vvvRfBwcE4c+YMtm7d2uAvCM4kHg+tKYmmpc1HSUkW/PxaIC6OkqjSjJWVioDak2jCpk3CVmjmJMqJpldTS1QelERdBGP2LT2RS9u2bZGWloaFCxfilVdewfXr1xEWFoZevXph9erVwuPi4uIQGRmJTp06WXWBmk2aNAllZWXo27cv1Go1Xn75ZavlK/Xx9fXFnj17MGfOHIwZMwZFRUVo0aIFhgwZ0uCWaV1CQkIwadIkvPPOO4iJiUFcXFy1BArwSfT999/H0aNH0bVrV6SkpGDZsmVISkrC3LlzYTKZEBMTg2HDhmHWrFkOx+Wo+pKoGI2JKs8kSqJ1DmDXgPPyAquoAMrLwcrLbeoOJrajJKqwyiWKbnHeyMhIrFixAitWrKj1MSUlJULrrSaenp5YtmyZVeIVu1jDCv/09HSr6xEREfj8889rjSE5ObnW+8TEs3nFPv7443qf27dvX6sJVxqNBnPmzMGcOXMadG65iXbUqjeJ+vrylYtMJkqiSmCMWZKoRmPzEilOoxFm9ppu34a6cnyeOIeiSXTPnj1YvHgxDh06hOvXr2PLli0YNWoUAH494Jtvvomff/4Z58+fh1arRVxcHBYtWlRjC8dduWL9WnuYTCbk5uZiyZIlCAoKwiOPPKJ0SETElpYox/GPKSqyXltK5MFKS/lqF7CeKNRQ4u5fSqLOp+gSl5KSEnTr1g0rV66sdl9paSnS0tLw1ltvIS0tDZs3b0ZGRgb9cXZRly9fRvPmzfHVV19h7dq18KhhcThRji0tUfFj8vLsH0cn9jE1YDy0TuIZurdvSxESqYOif+mGDRuGYcOG1XifVqvFjh07rG5bsWIF+vbti8uXL6NVq1ZyhEgaqE2bNvWuJ62t+5Q4n61JVDwueuUKUKWWBHEiceKzJ4mKnyOeoEScw62aCwUFBeA4DkFBQbU+RqfTQafTCdcLqWoHIUIS5TigIWVYxYmWkqi8xEtT7OrOFU0kEi+VIc7hNhWLysvLMWfOHIwbN67OWZgJCQnQarXCJTo6WsYoCXFN5iTq61t3oQWzqi1RIh+r9Z32tETVan5fUVhXPiLO4RZJVK/X47HHHgNjrNZZnWZz585FQUGBcLlCfwFIE1dRAZh31WpIV27Vx9GvkLwcbYkCltYoKymBqaxMirBILVy+O9ecQC9duoTffvut3rWAGo0GGnsG4wlppMRF5GtLolrtHfDy0sLHpzkAaokqiZmTqEpV4+4tZu2joqD19UV4DcNbnLe3ZZlLTg5UrVs7IVICuHgSNSfQs2fPYufOnQgNDVU6JELcjnhSUW0bvz/00G9W18XJVrw8hjgXY0yoNFRfK3TrwoW13ifef9R44wY8KIk6jaJJtLi4GJmZmcL1CxcuID09HSEhIYiMjMQ//vEPpKWl4ccff4TRaER2djYAvqpM1T0fCSE1s3VmLgB4e/O7vRiN1BKVEysu5qtcAFZLVWxllUQr/24S51A0iaampmLw4MHC9fj4eADA5MmT8c477+CHH34AAHTv3t3qeTt37sSgQYPkCtOpij/9FCZzZXAZqfz94W9DuT3ivuxJouaCC4WFlETlJMV4KFAliV675khIpB6KTiwaNGgQGGPVLsnJycK6w5oujSWBAoCpuBisqEj2iy2Je8qUKUIlKbNvv/0W3t7eQvH47OxsvPzyy4iNjYW3tzeaN2+OAQMGYPXq1fXup8lxnHAJDAxEnz598L///a/GxyYkJECtVmPx4sU13u9IHI2V+G9oQ5Oo+LGFhfyFOJ+4aLxDSVStFhKpKSeHr6VLnMKlx0SbnCr7aDqFXu/wS6xZswYzZszAxx9/jKlTp+L8+fMYMGAAgoKC8O6776JLly7QaDQ4duwYPv30U7Ro0aLeSlNJSUl48MEHUVhYiFWrVgld+V26dLF63Nq1azF79mysXbsWr732mtV9UsTRGDVkTPS338ajvDwX3t7NcP/9/HZCVcdF//Y3JwZJAABM/G2lniQ6dckS3CosRGhgIJJeeaXa/ZyvL1hZGcAYjFlZ8IiJkTpcAkqirsPTE15VEoYzVBw75lAiff/99zFv3jxs3LgRo0ePBgBMnz4dHh4eSE1NtdprtG3bthg5cmS9lYwAICgoCBEREYiIiMCCBQuwfPly7Ny50yqJ7t69G2VlZZg/fz7WrVuHP//8E/1FO0ZLEUdj1JDu3OvXdwtboZmJZ+hSEpWHLS3R348fF7ZCqwnn7w9UFlswXL5MSdRJ3GKdKHENc+bMwYIFC/Djjz8KCfTWrVvYvn07ZsyYUW2zbjNbdqEwGAxITEwEgGqTxxITEzFu3Dh4enpi3LhxwuOcEUdjYu7O9fS0rbODlrnIz1RUJBxzDvZMqUQfoOHSJYdei9SOkihpkK1bt+L999/H//73PwwZMkS4PTMzE4wxdKhSF65Zs2bw9/eHv79/g7YHGzduHPz9/aHRaDBr1iy0adMGjz32mHB/YWEhvv32W0yYMAEAMGHCBHz99dcorhzblSqOxoYxS0vUlvHQqo+nJCoPJt5I29EVCF5ewrcm45UrYEajY69HakRJlDRI165d0aZNG8ybN09IXHU5cOAA0tPT0alTJ6GW8XPPPSckNH9xMwfABx98gPT0dGzduhV/+9vfsGbNGoSEhAj3b9iwAe3atUO3bt0A8DO2W7dujU2bNtkcR1NSUGDZhcWRJEprReVhMo+Jeno63HPCcRxUAQH8FYOBZuk6CSVR0iAtWrTArl27kJWVhQcffBBFld1OsbGx4DgOGRkZVo9v27YtYmNj4SOaaj9//nykp6cLF7GIiAjExsbigQceQFJSEsaOHYscc6068F25J06cgIeHh3A5efIk1q5da3McTUlDJhXVhrpz5cVMJrCSEgCOd+WacaIP0Ug7rDsFJVHSYK1bt8bu3buRnZ0tJNLQ0FD8/e9/x4oVK1BS+QegNuHh4YiNjRUutenbty969eqFhZUVWY4dO4bU1FTs2rXLKgnv2rULf/31F06fPm1THE2JvctbAL72uVrNH1MSdT5WXMz3vwOSzdRXiT50A32ITkGzc12FXs/PnJXhPI6Ijo7Grl27MHjwYAwdOhTbtm3DqlWrMGDAAPTu3RvvvPMOunbtCpVKhYMHD+L06dPo1auXzeeZOXMmRo8ejdmzZyMxMRF9+/bFfffdV+1xffr0QWJiIhYvXuyUONydPYUWzDiOb40WFFB3rhxMouUtUrVExaWnqDvXOagl6kr0eudfJNCyZUvs2rULubm5GDp0KMLCwnD48GHExcVh7ty56NatG3r37o2PPvoIr776KhYsWGDzOR588EHExMRg4cKF+PLLL/Hoo4/W+LhHH30U69atg16vR7t27SSPw9050p0LWBJvURGfTInzMPHMXInKmnIcJxRdsLXICmkYaokqTOXvD5NC522o5OTkare1aNECZ86cEa4HBgbio48+wkcffWRzLDWt3+Q4DqdOnQIArFq1qtbnzp49G7NnzxauR0ZG2h1HY+RIS7Tqc65cAbRax2MiNRMvb5Gy8Arn6yvs6GLMzoaqjqEUYjtKogqj+rXEmRqaRDt2fBoVFQXw8rLOklUnF3XuLHGARMBsXCM69YEHUFhaisB6uhjEdXRNN28ClEQlRUmUkEbMnEQ5DqhrgnKvXvNqvJ3WisrHqqu1AUn0jXHjGvS65g26AcCYm2tzXKRuNCZKSCNmTqI+PoDKjt92WuYiH1tbog0lTqKmmzcle13CoyRKSCOl1wM3bvDH9oyHApRE5SSMiXKcZW2RBDi1GvDgOx1Nt29L9rqER0lUZk21CDqpztn/F7KzLcsOpUiitFbfucyTf6SoVlQVp9EI52ASzdInPBoTlYlnZfdMaWlpk62eQ6xVVO7xqJaw1SFmy/KW9etbCru4jB9vWRRqLr+q11NL1JmY0QhWWZ+xoV25sVOnCru4ZCYl1flYzstLqIZkys+HOizMsYCJgJKoTNRqNYKCgoRSdr6+vk12VxECmEwm3Lx5E76+vvDwcM6vobhAgg0rmqrx9wdu3+aTKGN8byORFhNX2XLG/4fKlihASVRqlERlFBERAQBWNWFJ06VSqdCqVSunfZlytNCCmTmJ6nTAzZtAeLjjsRFr4pm5Uk4qEl5TVLzBRFUzJEVJVEYcxyEyMhLh4eHQ07hEk+fl5QWVPVNmG0icRB1piYrHUy9fpiTqDMzZSVT0muJZwMRxiibRPXv2YPHixTh06BCuX7+OLVu2YNSoUcL9jDHMmzcPn332GfLz8zFgwACsXr0a7du3Vy5oCajVaqeNgxFiJu7OdbQlanb5MtC7t/2vRWrGbFwjajPRa4pr9BLHKTo7t6SkBN26dcPKlStrvP/999/Hhx9+iI8//hj79++Hn58fhg4divLycpkjJcT9OFryz8y8JSVAM3Sdxao71wljouLuXEZJVFKKtkSHDRuGYcOG1XgfYwzLli3Dm2++iZEjRwIA1q1bh+bNm+P777/H448/LmeohLgdc0vUPMPWXrTMxfmc3hJVq/kZYYxRS1RiLrtO9MKFC8jOzkZcXJxwm1arxV133YW//vqr1ufpdDoUFhZaXQhpahiztEQdaYUC1kn00iXHXovUTDw71yktUY4TkjOjnVwk5bJJNDs7GwDQvHlzq9ubN28u3FeThIQEaLVa4RIdHe3UOAlxRbdu8bNpAccmFQF8EjZPIKYk6hwm8RIXZ7REYZlcxMrLwQwGp5yjKXLZJGqvuXPnoqCgQLhcoRXipAkSTypytCWqUlkmJlESdQ6hdahSgXPWjG1RC9dqXSpxiMsucTGvqbxx4wYiIyOF22/cuIHu3bvX+jyNRgONaGExIU2RrWtEBw/+EkajDmp1zb87AQFASQmQm8v/62hiJtaElqgNrdC18fHQ6fXQNPA5nKcnzIUmTSUlUNHmsJJw2SQaExODiIgIpKSkCEmzsLAQ+/fvx/PPP69scIS4OFurFUVFDarz/qqTi+680764SHXMaAQqVxzYMh56X5cutp1I3BKlcVHJKJpEi4uLkZmZKVy/cOEC0tPTERISglatWmHmzJn4z3/+g/bt2yMmJgZvvfUWoqKirNaSEkKqk2qNqFnVyUWURKXj9JJ/lawKLlASlYyiSTQ1NRWDBw8WrsfHxwMAJk+ejOTkZMyePRslJSV45plnkJ+fj3vuuQfbtm2Dt2h/PEJIdVLVzTUTrxW9eNHx1yMWVjNznTSpCLBu5Zoqi90TxymaRAcNGlTndlAcx2H+/PmYP3++jFER4v5snVh07douYUy0pq5dSqLOY7KzJbrn2DFhTLRBXbs0scgpXHZMlBBiP3MS9fTkiy3UZ+fOCTVuhWYmTqIXLkgUJAFg/xrRaUuXNngrtKqvzcrKbAuS1KrRLXEhpKljzLL3p3iNpyPEXcKURKUl15ioeOYvtUSlQ0mUkEamoIBfhgJItxRFrba8FiVRaTHR+KQzx0Qh2vSC0ZioZOxKoufPn5c6DkKIRKQstCBm7tLNzQVoNy3p2DsmaiuO44TXp4lF0rEricbGxmLw4MH48ssvaUcVQlyMuEiXFDNzzQIDLcf0PVo6Vi1RZ3bnAkISpe5c6diVRNPS0tC1a1fEx8cjIiICzz77LA4cOCB1bIQQO4iTqJQtUUqiziHbmCgAztylq9dT/VyJ2JVEu3fvjuXLl+PatWtYu3Ytrl+/jnvuuQedO3fG0qVLcfPmTanjJIQ0kLO7cwFKolISWqJqNd/l6kw0Q1dyDk0s8vDwwJgxY/DNN9/gvffeQ2ZmJl599VVER0dj0qRJuH79ulRxEkIaSI7uXFGhMeIgYUzU2V25oGUuzuBQEk1NTcX06dMRGRmJpUuX4tVXX8W5c+ewY8cOXLt2TdhMmxAiHzm6c8+dk+51mzJmMAAVFQBkGA8FaIauE9j1qS1duhRJSUnIyMjA8OHDsW7dOgwfPhyqyi18YmJikJycjDZt2kgZKyGkAcxJtKGFFhrK25t/vYoKaolKxSqRUUvULdn1qa1evRrTpk3DlClTrLYpEwsPD0diYqJDwRFCbGNvoYWaqhRVxXF8azQ3ly9CX1EhbZJuiuytVgSgQVWKqhEnUWqJSsKuJLpjxw60atVKaHmaMcZw5coVtGrVCl5eXpg8ebIkQRJCGiYvDzA3MKQcDzXTavkkajLxRRc6dJD+HE2JSe6WqKg710QtUUnYNSbarl075ObmVrs9Ly8PMTExDgdFCLGPs8ZDzcT7OJ85I/3rNzWyVSsyo+5cydmVRGvbeaW4uJi2KSNEQc6amWsmTqIZGdK/flNj1aUqaiU6DSVRydnUf2De75PjOLz99tvwFe32azQasX//fnTv3l3SAAkhDWdvS/TQoX+joqIAXl5a9Oo1r9bHURKVliN7iS7csAGFpaUI9PXFG+PGNeg54u5cSqLSsCmJHj58GADfEj127Bi8RLMKvLy80K1bN7z66qvSRkgIaTB7W6KnT38mbIVWVxINCrIcUxJ1nCOzc5O2bxe2QmtoEqWWqPRs+tR27twJAJg6dSqWL1+OQPHCMUKI4pzdnevlBfj48JOXTp+W/vWbGvHkHjnWiXIqFT/NmjFKohKxa0w0KSmJEighLujyZcuxMyYWAUBwMP/vzZv8TF1iP7nXiYrPQ0lUGg3+1MaMGYPk5GQEBgZizJgxdT528+bNDgdGCLGduSWq0VjtwSypoCDg2jX++NQp4N57nXOepkBIohzHtxJlwKnVYHo9JVGJNPhT02q1QnFkrVZb50UqRqMRb731FmJiYuDj44N27dphwYIFtc4OJqQpM5mArCz+2BlduWbmlijAJ1FiPyGJytUKFZ/LYADT6+U7byPV4E8uSVQdI8meShl2eO+997B69Wp8/vnn6NSpE1JTUzF16lRotVq89NJLssRAiLvIzgbMfxOd1ZULWCfREyecd57GjjEmJFFZ6uaaiWfolpfLsz61EbPrkysrKwNjTFjicunSJWzZsgV/+9vf8MADD0gW3J9//omRI0dixIgRAIA2bdpgw4YNtHcpITUQj4fK1RKlJOoAnY7vPgBkbYlyHh4w9+WxsjLrPe6IzezqhB85ciTWrVsHAMjPz0ffvn2xZMkSjBw5EqtXr5YsuP79+yMlJQVnKkujHDlyBHv37sWwYcNqfY5Op0NhYaHVhZCmwNkzc818fPgLABw75rzzNHbikn+KtURpXNRhdiXRtLQ03Fs5m+Dbb79FREQELl26hHXr1uHDDz+ULLjXX38djz/+ODp27AhPT0/06NEDM2fOxPjx42t9TkJCgtX4bHR0tGTxEOLK5GqJApbWaE4OfyG2U2RmLmgnF6nZ9cmVlpYioLILYPv27RgzZgxUKhXuvvtuXLp0SbLgvv76a6xfvx5fffUVOnXqhPT0dMycORNRUVG1FrefO3euUFkJAAoLCymRkibBkSQaGTkQ5eW58PZu1qDHh4RYZugeOwYMGWLb+UiVurl2JNF7O3fGrcJChNq63JBaopKyK4nGxsbi+++/x+jRo/HLL79g1qxZAICcnBxJ14++9tprQmsUALp06YJLly4hISGh1iSq0Wig0Wgki4EQd+FIEr3//vU2PT4kxHJ85AglUXs42hJNeuUV+05MSVRSdnXnvv3223j11VfRpk0b3HXXXejXrx8AvlXao0cPyYIrLS2ttt2aWq2GyTwYTwgRmJMoxwGistZOERpqOT5yxLnnaqxkLz5fibpzpWVXS/Qf//gH7rnnHly/fh3dunUTbh8yZAhGjx4tWXAPP/wwFi5ciFatWqFTp044fPgwli5dimnTpkl2DkIaC3MS9fMDnL1uPzhYqB6H9HTnnquxYjKX/BNQEpWU3Z9cREQEIiIirG7r27evwwGJffTRR3jrrbcwffp05OTkICoqCs8++yzefvttSc9DiLsrLbWU4HP2pCKA/zscFATcvg2cPMmv1qBRFNvIvSG3GVdlnShxjF2fXElJCRYtWoSUlBTk5ORU6149f/68JMEFBARg2bJlWLZsmSSvR0hj5ejylh9/vB9lZTfg49McDz30W4Oe06wZn0QNBuD4caBXL9vP25Q5OrFo2BtvICc/H+FBQdi6cGHDn0gtUUnZlUSfeuop7N69GxMnTkRkZKRQDpAQogxHl7cUFJxBSUkWKioKGvyc0FDg7Fn+OC2NkqitHJ1YdPbaNVy7dQsF4tdpCFFfv4mSqMPsSqJbt27FTz/9hAEDBkgdDyHEDuKVZXJ05wJ8S9QsLU2eczYmQhJVqWQrPg+Ab/R4ePC1cymJOsyuTy44OBgh4jnuhBBFKZ1EDx2S55yNiSLF580qx0UpiTrOriS6YMECvP322yi1tRuBEOIUclYrMvPyAsybNh09ClRUyHPexoCJNsWWdWZuJeGcOh2Y0Sj7+RsTuz69JUuW4Ny5c2jevDnatGkDzyq7AKRR3w4hslKiJQoAYWFAQQE/O/f4caBnT/nO7c5YeTm/PghQtCVqjoVz5rY/jZxdn96oUaMkDoMQ4ghzS1Sj4VuIcmnWDMjM5I8PHaIk2lBWM3NlLLQgnLPqTi6URO1mVxKdN2+e1HEQQuxkNFqWuMjZCgWA8HDL8cGDwNNPy3t+d6VU8fmazsloWM4hdk8Jy8/Px5o1azB37lzk5eUB4Ltxs7KyJAuOEFK/69f5tZqA/Ek0NJSvXATwSZQ0jKNrRB3FUf1cydj16R09ehRxcXHQarW4ePEinn76aYSEhGDz5s24fPmysNcoIcT5lBoPBQBPT0vlomPHgLIyy16jpHaspMRyhVqibs2uTy8+Ph5TpkzB+++/L2yJBgDDhw/HE088IVlwhJD6iZOo6NfRJj17vg29vhienrZn4bAwPokajXwd3cr9KEgdpGiJzh07FiXl5fDz9rb9yVS1SDJ2fXoHDx7EJ598Uu32Fi1aIDs72+GgCCENJ0VL9M47n7H7/OHhwJkz/PH+/ZREG0KKurlPPvig3ee36s6llqhD7BoT1Wg0KCwsrHb7mTNnEBYW5nBQhJCGU7I7F6g+uYjUT+kxUXHiNlESdYhdSfSRRx7B/PnzodfrAfBlpC5fvow5c+bg0UcflTRAQkjdpOjOdURIiGXZ4YED8p/fHSk9O5f2FJWOXUl0yZIlKC4uRlhYGMrKyjBw4EDExsYiICAAC23ZTYAQ4jBzElWrAXuGxwCgtPQ6iouvorT0us3PVaksm3RnZgKVk/VJHaRIotfz8nA1NxfX7fmB08Qiydj16Wm1WuzYsQN//PEHjhw5guLiYvTs2RNxcXFSx0cIqQNjwMWL/HFAgGW5ia22bOmDkpIs+Pm1wPjxV21+fng4kJPDHx88CAwdal8cTYXJPDtXrbZ7F6x7X3kF127dQlRoKDKTkmx6LqdS8d9+TCZKog6yOYmaTCYkJydj8+bNuHjxIjiOQ0xMDCIiIsAYo23RCJFRbi6/rARQZjzUTDwueuAAJdH6KFp83szDA6iooCTqIJu6cxljeOSRR/DUU08hKysLXbp0QadOnXDp0iVMmTIFo0ePdlachJAaKD2pyEw8n3D/fuXicAfMYBCq9XNV6o7LyTwuysrKwEwmxeJwdzZ9DUpOTsaePXuQkpKCwYMHW93322+/YdSoUVi3bh0mTZokaZCEkJopPanILDCQr9ur0/EtUcbs71pu7BQvtFD13IzxReh9fZWLxY3Z1BLdsGED/vWvf1VLoABw//334/XXX8f69eslCw4AsrKyMGHCBISGhsLHxwddunRBamqqpOcgxF2Zx0MBZZMox1m6dG/etE7uxJriy1tqOLdVYic2sSmJHj16FA/WscB32LBhOHLkiMNBmd2+fRsDBgyAp6cntm7dipMnT2LJkiUIDg6W7ByEuDNXSaIAdek2lMnVWqKgGbqOsOkTzMvLQ/PmzWu9v3nz5rh9+7bDQZm99957iI6ORpJo5llMTIxkr0+Iu3OlJFq16MLYscrF4srErT6XaYkWFysWh7uzqSVqNBrhUceHrlarYTBvJyGBH374Ab1798Y///lPhIeHo0ePHvjss8/qfI5Op0NhYaHVhZDGypxE1WrlC7+LW6JUdKF2LjMmKprUZKLuXLvZ9AkyxjBlyhRoNJoa79fpdJIEZXb+/HmsXr0a8fHx+Ne//oWDBw/ipZdegpeXFyZPnlzjcxISEvDvf/9b0jgIcUXiNaL+/spP5PHx4VvDRUX8Bt0Gg7I5wlWJE5YrzM4FaEzUETb9F68tcYlJOTPXZDKhd+/eePfddwEAPXr0wPHjx/Hxxx/XGsvcuXMRHx8vXC8sLER0dLRkMRHiKvLyAHMvnNJduWZhYXwSLS0FTp4EunZVOiLXo3TJP4EogVMStZ9Nn2CSjVUxHBUZGYm//e1vVrfdeeed+O6772p9jkajqbWlTEhjcuGC5djRNaIjRqTAZDJApXLsj3p4OHD+PH988CAl0ZpINSb684IFMJhM8FDZVb3V6tzUnWs/l+5sGTBgADIyMqxuO3PmDFq3bq1QRIS4DiknFQUFdXDsBSpVHRd98klJXrZRsZrE40ASvaNlS8cCoYlFkrDvK4xMZs2ahX379uHdd99FZmYmvvrqK3z66aeYMWOG0qERojhXmplr1qyZZWyWlnPXTGj1eXgoWiaVU6mERGoqKlIsDnfn0km0T58+2LJlCzZs2IDOnTtjwYIFWLZsGcaPH690aIQoTtyd6ypJ1NMTCArij48eBcrLFQ3H5TDGhFafkpOKzITSf8XFYIwpHI17cunuXAB46KGH8NBDDykdBiEuR8okmpn5FQyGUnh4+CI29gmHXissDLh9m5+de+QIcNddjsXWmLDycsBcp9bBSUWbdu9GqU4HX40GYwcOtO9FPD35bzpGI1/6T+l1Um7I5ZMoIaRm5iQqxRrR/ftnC1uhOZpEmzUDzpzhjw8doiQqJh57dLQl+kZysrAVmr1JlPP0hLn9yYqKlF9s7IZcujuXEFIzqfYRdQbx5KJDh5SLwxVJNalIMuKCCzQuahdKooS4oRs3LOONrjIeahYaSpOLamOSsCUqBc7LSzhmVN3NLpRECXFDrjipyMzDAzDvEXHiBE0uEmPi1p4rJFFqiTqMkighbshc0ADg9/J0NaGh/L9GI3D8uLKxuBJxonKFlqhV1SJqidqFkighbsiVW6IAP7nILC1NuThcjZQTi6Qg7s41URK1CyVRQtyQuCXq6kn08GHl4nA1JhfrzoWHhzCAbSooUDgY90RJlBA35C7duQC/VpTwhJaoSgVOrVY2GICvmFTZGqUkah9KooS4IXN3rkYj/A10KV5elhby0aP82GhTxxizJCoX+tCELl2dji8GQWxCSZQQN6PTAVeu8MdStUJ9fCLg59cCPj4R0rwgLK3RkhIgM1Oyl3VbrLycL+MEacZDmwcHIyo0FM3NU6HtZDUump/vYFRNjwus9iWE2OLSJb7YAiDdeOiYMdIv6AwNtRSEOHoU6CDNRjFuSzz7lZOgJfrH0qUOvwZQPYmqI6T7ItUUUEuUEDdz7pzl2BXHQ83E46JHjyoXh6uwGnN0hUlFZuIkevu2goG4J0qihLgZV59UZBYSYjmmyUXWS0ikaIlKhdNohGNTXp6CkbgnSqKEuBl3aYkGBFgaXNQSBZhovNFlkyiNidqMxkQJcTPOSKJ79jwLnS4PGk0I7rvvE0lek+P41uiNG/w4bmGhayd9ZxN350qRRF9YuRK3i4oQHBCAFTNm2P9CHh6ASgWYTDDduuVwXE0NJVFC3Ix5pqtaDfj5SfOaV678JGyFJqXgYD6JAnz5v/79JX15t2LVypMgiW5LTRW2QnMEx3HgNBqwsjKY8vPBjEaXWMPqLqg7lxA3YjJZxkRdbQu0mojHRY8dUy4OVyAkUQ8PcCrX+tMrdOkyRpOLbORanyQhpE7Xrll2RXGHrlFxEm3KheiZXi9UKxKPQboMb2/hkLp0bUNJlBA3cvas5ZiSqPsQt+5cMYlaTS7KzVUwEvfjVkl00aJF4DgOM2fOVDoUQhQhrvyj1SoXR0N5ewM+PvzxsWOWIhFNjdXSEReamWvGmT8kAEZKojZxmyR68OBBfPLJJ+jatavSoRCiGHFL1B2SKGBpjd66ZZlk1NSIkyi1RBsXt0iixcXFGD9+PD777DMEO1gnkhB35o5JVPwre+KEcnEoySgaZ+RE44+uglOrhRay8eZNsKbaZWAHt0iiM2bMwIgRIxAXF1fvY3U6HQoLC60uhDQWZ87w/0q5vMXZaFzUerKOKyZRQBSXTgcm3veU1Mnlk+jGjRuRlpaGhISEBj0+ISEBWq1WuERHRzs5QkLkYTRaxkQDA/n18e6Akqioi9TDw2XXYIqTu/HmTQUjcS8uXWzhypUrePnll7Fjxw54N/Db29y5cxEfHy9cLywspERKGoWLF4GKCv5Y6q7cdu3GQae7DY1G+uGSoCDLcVNMoqaSErCSEgDStkIfu+8+3C4uRrC/vySvJ55cZMrJAdq1k+R1GzuXTqKHDh1CTk4OevbsKdxmNBqxZ88erFixAjqdDuoq3+o0Gg00LjhwT4ijMjIsx+LEJIW7714s7QuKmDfoLirikyhjrl8kQkomUatOnKgc9e7UqZK9FlClJZqTI+lrN2YunUSHDBmCY1XKnEydOhUdO3bEnDlzqiVQQhozcRJ1l0lFZsHBfBItLgYuXwZat1Y6IvmIE5KUSVRq1VqipEFcOokGBASgc+fOVrf5+fkhNDS02u2ENHanTlmO3W2SekgInzwBvjXapJKoaF2Pq04qAsCXItRoAJ0OxpwcMJPJ5coTuiL6CRHiJsRJVOruXGcTJ/2mVkPXlJ0tHLtySxQQJXmDgWroNpBLt0RrsmvXLqVDIEQR5iTq6yt90ZtNmzqitPQafH2jMHbsaWlfHE23ED0zmSzduRqNpDNzuz//PK7n5SEyJATpq1dL8pqcjw9Y5ZZtppwcqB3cIaYpoJYoIW7g5k2+4g/gnFaowVAMvb4IBkOx9C8OPmbzZKKmlERNubmAwQAAUEncCi0uL0dRWRmKzTsSSEAco7GplpeyESVRQtyAeGmIuFXnLtRqS/I/fRrQ6xUNRzbG69eFY87XV8FIGsaqhi5NLmoQSqKEuAFxEnW3SUVm5uSv1/OJtCkwXrsmHLtDEoVGI3QZmKgl2iCURAlxA+Kas+6aRMXDa0eOKBeHnNyuJcpxQmvUlJcHZq7uQWpFSZQQN3D0qOXYHbtzAeu4m0ISZSYTjOaZuV5e4DzcYx4ndenahpIoIS7OZLIk0YAAl9yOskHELdH0dMXCkI3p1i1h8NcdWqFmVkUXqEu3XpRECXFx588DlaVX3bYVCvBLc8zLENPTG/8G3eKuXJWbJlGaoVs/SqKEuDhx16c7L9vjOKBZM/44NxfIylI2Hmdzu0lFlag71zaURAlxcYcOWY7NSchdib8EpKUpF4ccjG5UqUiM8/AAPD0B8C1R2qC7bu4x0k1IEyZONs5Kovfc8zGMxjKo1c79Yy+OPy0NeOQRp55OMYwxSxL19ARXmZSk9OH06SjX6eDthF2rOB8fML0eKC8HKywE5247HsiIkighLowxS0vU2xvw83POeVq3fsg5L1xFWJjlODVVllMqwnT7NqDTAXBeV+7wPn2c8rpAZRItLATAt0ZVlERrRd25hLiwCxf48UOAT0Duvg9nQAC/nh8ADh5svJOL3KnofE1oclHDURIlxIXt3285Dg9XLg6pcJylNZqTA1y5omw8ziIeD3WnmblmtMyl4ag7lxAXJlcSvXnzEEymCqhUXggL6+W8E4FPolev8sf79wOtWjn1dIqw2kPUSS3RtMxM6A0GeHp4oGdsrKSvzXl78994GKOWaD0oiRLiwv74w3IsHk+U2vbtI1FSkgU/vxYYP/6q804E6y8D+/YB//ynU0+nCKElqlI5rTrGYwsX4tqtW4gKDUVmUpKkr81xHDhvb7CyMphu3QLT650yOaoxoO5cQlxUSQlw+DB/HBxsKVTg7qom0caGlZUJk3I4Hx9wbjqQLUyIYozWi9aBkighLmrfPsBo5I+bN1c2Fin5+ADmyZ6pqcIk1kZDjq5cOViNi4rGeIk1SqKEuKjduy3HkZHKxeEM5i8FFRXWxSQaA3GrzZ0qFVVlNUOXkmitKIkS4qJ27rQcR0UpF4cziFvWe/cqF4czWLVE3bgPnpa5NIxLJ9GEhAT06dMHAQEBCA8Px6hRo5CRkaF0WIQ4XXGxZWauVuu8IgtKEbes9+xRLg5nMDWW7lwPD2FSlDE7G8xkUjgi1+TSSXT37t2YMWMG9u3bhx07dkCv1+OBBx5AiXlLC0Iaqd27hV200KKFsrE4g1ZrmSi1d69l7NfdMZPJ0p3r5QVOrVY2IAcJ3dF6PUx5ecoG46JceonLtm3brK4nJycjPDwchw4dwn333adQVIQ43/btluPGmEQ5DoiIAC5eBAoKgGPHgO7dlY7Kcabbty17iLpxK9RM5eMDY34+AH5rN7W774DgBC7dEq2qoKAAABBSx6aKOp0OhYWFVhdC3M3Wrfy/HNc4kyhgPc67a5diYUiqsXTlmoknRom3diMWbpNETSYTZs6ciQEDBqBz5861Pi4hIQFarVa4REdHyxglIY7LzATOnuWPIyKctlZfceJx0caSRN293F9VVklUtMk4sXDp7lyxGTNm4Pjx49hbz1S+uXPnIj4+XrheWFhIiZS4lf/7P8uxXP91//nPUwAYAPkKA4SE8MXodTp+DNhoBNx8CFHWNaKHV650+ifGeXry3+IqKmC8dg3MZAKncpu2lyzcIom+8MIL+PHHH7Fnzx60bNmyzsdqNBponLC/HiFy+eEHy3Hr1vKc08srQJ4TiXAc3xq9eBHIzweOHAF69pQ9DEnJUe7PLECmlq7K1xemigp+ctHNm1A3psofEnDprxSMMbzwwgvYsmULfvvtN8TExCgdEiFOlZtrWfIRGAgEBSkajtOJx0XF62LdkamkxFLuz9fXbcv9VcWJ1lcZrzq3rrI7cukkOmPGDHz55Zf46quvEBAQgOzsbGRnZ6OsrEzp0Ahxih9+AMzL8WJi3H//0PqIJ0399ptycUjB6OZ7iNZGnEQNlESrcenu3NWrVwMABg0aZHV7UlISpkyZIn9AhDjZN99YjuXseDl6dCkqKgrh5RWIrl3j63+CRIKC+Fq6ZWV8C1yvB9x1sxDx7FU5yv19+P33KCwtRaCvL14aNcpp5+F8fS3bol2+7LTzuCuXTqKssW57T0gN8vKAX3/lj/39nbv1WVXHji0VtkKTM4lyHN+le+4cX6UpNRXo10+200tKnETlmJn74f/+J2yF5tQkqlKB8/UFKymBKS8PpqIiqALkH0N3VS7dnUtIU/Ldd4DBwB+3bdv4u3LNxF26KSnKxeEoIYmqVI1n37pKnL+/cGy4eFG5QFwQJVFCXMT69Zbjdu2Ui0Nu4slF7ppETYWFjXJSkZm45Wk4f17BSFwPJVFCXMClS5atz7RaoClVVwsMBMx/o//8EygtVTYeexivXBGOuca2WwAqW6KVXwwM587RUJsIJVFCXMAXX1iO27dvOl25ZuYu3YoK4PfflY3FHgbRhBuVqOuzseBUKnCV33RYURFt0i1CSZQQhZlMQHKy5Xr79oqFohjxuOiOHcrFYS9xEm2MLVEAUGm1wrH+1CkFI3EtlEQJUdjOnfzsVIBPJk1x4qM4iYp3sHEHptJSoWXG+fjw+3A2QipR5Q/9iRPUpVuJkighCvv4Y8txx47KxaEkb2/Lkp5jxwB32jDEcOGCcMw14m9AnKen8P5MeXlW48BNGSVRQhR09SqwZQt/7OMDtGmjaDiKEpfF/uUX5eKwlSEzUzhWBQYqGInzqUTbUFakpioYieugJEqIgj76iN+9BADuvFO5XUxCQ3siPPxuhIYqVwFevGPNzz8rFoZNmMkEg3nfOpXKaj2ls3Vv1w59O3RAdxnXQ6mCg4X/pPoTJ2Cq3OO5KeNYI+/YLiwshFarRUFBAQIb+bdE4l7y8/ldWgoL+fX5TzwBNIItKO1mMvGzlHU6ftnLzZuuv5eq4dIllFTOCuO0Wng2gQW+hmvXhDFgr1694PPQQwpHpCxqiRKikA8/5BMoANxxR9NOoAD/RcLcGi0stKybdWX6EyeEY1VwsIKRyEcdHs5/WAAqDh+GMTdX4YiURUmUEAXk5gJLlvDHHAd0765oOC5DPCa8ebNiYTQIMxqhP36cv8JxVktAGjPOw8Oyp6jJhPKtW5v0TF1KooQoYN48Syu0Qwe++5LwLVHzuPDmzZZawq7IcPo0WOW2jKqgIHBKDWgrQNW8udDXbjh/HvpjxxSOSDmURAmR2YEDQOUuf/DwAHr1UjYeANi27RF8/30/bNv2iKJxeHoCrVrxxzk5ll1tXJFONDtVFRoq+/n/8Z//YNBrr+Ef//mP7OfmVCp4iKZTl2/bBlNRkexxuAJKooTIqLgYmDQJMPd+9eoFuEKBm1u30pCTsw+3bqUpHQpiYy3HiYnKxVEXY3Y2jObdTDQaRdaHpp87hwMZGUg3V+qQmSooSBgHZmVlKPvhhybZrUtJlBCZmEzAlClARgZ/PSwM6NJF0ZBcUqtW/JpZAPj+e34traspFxX4VYeHN7pdWxpKHR3Nd6eAXy9b8ddfCkckP0qihMjAYACefprfMxTguy0HDxYmORIRtdpSuclgAJYuVTaeqozXrsFw8iR/xcNDka5cV8F5eMBDNBus/NdfoRcVn2gK6FeYECc7dw6IiwPWruWvcxwwZAggKkVKqujUyTLBaNUqwFW2sGSMoWzbNuG6OiICXBP/JqQKDIQqIoK/whhKv/66SW3c3bQ/fUKcxGAAdu3iu2/vvNOy5lGl4hOoefIMqZmvL9C5M3+s0/E/x4oKRUMCAFTs22epGavRQGUu+NvEqSMjwZm/Fer1KPniC+j++APMladXS8QtkujKlSvRpk0beHt746677sKBAweUDokQAPwf+GPH+Pq3y5YBL7/MJ8mQEL679vPPAb2ef6y/P/DQQ0DbtoqG7DZ69OB/ZgC/x+hjj/FVnpSiP3sW5aJ92jxatWqyY6FVcRwHjzZtwJnXaplMKP/1VxQtW4ayn3+G/vRpmEpKlA3SSVx+z55NmzYhPj4eH3/8Me666y4sW7YMQ4cORUZGBsLDw5UOjzQxJhOQmgr89BO/72VqqiVJ1sbLi29Vde3q+mXsXImXF/+F5Mcf+frC//sfEBMDjB4N9OzJb58WHMyvsQ0N5SdqOVr1iVVUgFVU8HVwNRpApQIrLkZFWhp0e/YI06pVzZtD1Yh3bLEHp1LBo107GLOyYMrJAQCwkhJUHDyIioMHAQCqsDB4tG0LzzvugLp160axttbla+fedddd6NOnD1asWAEAMJlMiI6OxosvvojXX3+93udT7VxiK8aAlBTg8mWgqIiv4Xr+PJCWZplZ2xAhIXxt3NathQmMLuunn1qirCwLPj4tMGKEa02HzcoCpJj0GaDRYegdZ6BRGxASWIF2kaWIbV6AVgG3EOGZAy80rL+Y8/aGuk0bfnBbQXc8/zyu5eUhKiQEZ8wLj10EKy2FMTub76qpAxcYCHWLFlCHhoLz8+O/uKjV8GzfHpx5iraLc+lf7YqKChw6dAhz584VblOpVIiLi8NftfxW6XQ66EQfXEHlLgOF5vIwhNTj0UelWeSfl8dfDh92/LWczwQAKCsz4dtvG9/vSmxoLn6Y/Dn8vWpIlEag3AiUN/TFysuB9HQJo7OPqXKQ2FRRgTz3+E9WXXk5X1WjKk9P+E+YALV5wpKEAgICpO2GZy4sKyuLAWB//vmn1e2vvfYa69u3b43PmTdvHgNAF7rQhS50oUu1S05OjqR5yqVbovaYO3cu4uPjhev5+flo3bo1Ll++DK0bFoguLCxEdHQ0rly54rbd0e7+Htw9fsD934O7xw+4/3toLPF7STwxwaWTaLNmzaBWq3Hjxg2r22/cuIGIWpr5Go0GGo2m2u1ardYtP3izwMBAt44fcP/34O7xA+7/Htw9fsD934O7xy/1jGqXXuLi5eWFXr16ISUlRbjNZDIhJSUF/fr1UzAyQgghxMVbogAQHx+PyZMno3fv3ujbty+WLVuGkpISTJ06VenQCCGENHEun0THjh2Lmzdv4u2330Z2dja6d++Obdu2obl5U9h6aDQazJs3r8YuXnfg7vED7v8e3D1+wP3fg7vHD7j/e6D4a+by60QJIYQQV+XSY6KEEEKIK6MkSgghhNiJkighhBBiJ0qihBBCiJ0aRRK1Zau0EydO4NFHH0WbNm3AcRyWLVsmX6C1sCX+zz77DPfeey+Cg4MRHByMuLg4l9gazpb3sHnzZvTu3RtBQUHw8/ND9+7d8cUXX8gYbXX2bre3ceNGcByHUaNGOTfABrDlPSQnJ4PjOKuLt7e3jNFWZ+tnkJ+fjxkzZiAyMhIajQZ33HEHfv75Z5mirc6W+AcNGlTt589xHEaMGCFjxNXZ+hksW7YMHTp0gI+PD6KjozFr1iyUlze4CrHkbIlfr9dj/vz5aNeuHby9vdGtWzdsE2243mCSFhFUwMaNG5mXlxdbu3YtO3HiBHv66adZUFAQu3HjRo2PP3DgAHv11VfZhg0bWEREBPvggw/kDbgKW+N/4okn2MqVK9nhw4fZqVOn2JQpU5hWq2VXr16VOXILW9/Dzp072ebNm9nJkydZZmYmW7ZsGVOr1Wzbtm0yR86zNX6zCxcusBYtWrB7772XjRw5Up5ga2Hre0hKSmKBgYHs+vXrwiU7O1vmqC1sjV+n07HevXuz4cOHs71797ILFy6wXbt2sfT0dJkj59ka/61bt6x+9sePH2dqtZolJSXJG7iIre9h/fr1TKPRsPXr17MLFy6wX375hUVGRrJZs2bJHDnP1vhnz57NoqKi2E8//cTOnTvHVq1axby9vVlaWppN53X7JNq3b182Y8YM4brRaGRRUVEsISGh3ue2bt1a8STqSPyMMWYwGFhAQAD7/PPPnRVivRx9D4wx1qNHD/bmm286I7x62RO/wWBg/fv3Z2vWrGGTJ09WPIna+h6SkpKYVquVKbr62Rr/6tWrWdu2bVlFRYVcIdbJ0d+BDz74gAUEBLDi4mJnhVgvW9/DjBkz2P333291W3x8PBswYIBT46yNrfFHRkayFStWWN02ZswYNn78eJvO69bdueat0uLi4oTb6tsqzZVIEX9paSn0ej1CQkKcFWadHH0PjDGkpKQgIyMD9913nzNDrZG98c+fPx/h4eF48skn5QizTva+h+LiYrRu3RrR0dEYOXIkTpw4IUe41dgT/w8//IB+/fphxowZaN68OTp37ox3330XRqNRrrAFUvweJyYm4vHHH4efn5+zwqyTPe+hf//+OHTokNBlev78efz8888YPny4LDGL2RO/TqerNoTh4+ODvXv32nRul69YVJfc3FwYjcZq1YuaN2+O06dPKxRVw0kR/5w5cxAVFWX1n0dO9r6HgoICtGjRAjqdDmq1GqtWrcLf//53Z4dbjT3x7927F4mJiUh3gT0lAfveQ4cOHbB27Vp07doVBQUF+O9//4v+/fvjxIkTaNmypRxhC+yJ//z58/jtt98wfvx4/Pzzz8jMzMT06dOh1+sxb948OcIWOPp7fODAARw/fhyJiYnOCrFe9ryHJ554Arm5ubjnnnvAGIPBYMBzzz2Hf/3rX3KEbMWe+IcOHYqlS5fivvvuQ7t27ZCSkoLNmzfb/EXMrVuiTd2iRYuwceNGbNmyRfFJIbYKCAhAeno6Dh48iIULFyI+Ph67du1SOqx6FRUVYeLEifjss8/QrFkzpcOxW79+/TBp0iR0794dAwcOxObNmxEWFoZPPvlE6dAaxGQyITw8HJ9++il69eqFsWPH4o033sDHH3+sdGg2S0xMRJcuXdC3b1+lQ7HJrl278O6772LVqlVIS0vD5s2b8dNPP2HBggVKh9Ygy5cvR/v27dGxY0d4eXnhhRdewNSpU6FS2ZYW3bolas9Waa7Ekfj/+9//YtGiRfj111/RtWtXZ4ZZJ3vfg0qlQmxsLACge/fuOHXqFBISEjBo0CBnhluNrfGfO3cOFy9exMMPPyzcZjKZAAAeHh7IyMhAu3btnBt0FVL8Hnh6eqJHjx7IzMx0Roh1sif+yMhIeHp6Qq1WC7fdeeedyM7ORkVFheR7RtbFkZ9/SUkJNm7ciPnz5zszxHrZ8x7eeustTJw4EU899RQAoEuXLigpKcEzzzyDN954w+Zk5Ah74g8LC8P333+P8vJy3Lp1C1FRUXj99dfRtm1bm87t1i1Rd98qzd7433//fSxYsADbtm1D79695Qi1VlJ9BiaTCTqdzhkh1snW+Dt27Ihjx44hPT1duDzyyCMYPHgw0tPTER0dLWf4AKT5DIxGI44dO4bIyEhnhVkre+IfMGAAMjMzhS8wAHDmzBlERkbKmkABx37+33zzDXQ6HSZMmODsMOtkz3soLS2tlijNX2qYzCXZHfkMvL290aJFCxgMBnz33XcYOXKkbSe3cQKUy9m4cSPTaDQsOTmZnTx5kj3zzDMsKChImK4/ceJE9vrrrwuP1+l07PDhw+zw4cMsMjKSvfrqq+zw4cPs7NmzbhH/okWLmJeXF/v222+tpsgXFRUpEj9jtr+Hd999l23fvp2dO3eOnTx5kv33v/9lHh4e7LPPPnOL+Ktyhdm5tr6Hf//73+yXX35h586dY4cOHWKPP/448/b2ZidOnHCL+C9fvswCAgLYCy+8wDIyMtiPP/7IwsPD2X/+8x+3iN/snnvuYWPHjpU73BrZ+h7mzZvHAgIC2IYNG9j58+fZ9u3bWbt27dhjjz3mFvHv27ePfffdd+zcuXNsz5497P7772cxMTHs9u3bNp3X7ZMoY4x99NFHrFWrVszLy4v17duX7du3T7hv4MCBbPLkycL1CxcuMADVLgMHDpQ/8Eq2xN+6desa4583b578gYvY8h7eeOMNFhsby7y9vVlwcDDr168f27hxowJRW9gSf1WukEQZs+09zJw5U3hs8+bN2fDhw21eHyc1Wz+DP//8k911111Mo9Gwtm3bsoULFzKDwSBz1Ba2xn/69GkGgG3fvl3mSGtny3vQ6/XsnXfeYe3atWPe3t4sOjqaTZ8+3eYkJCVb4t+1axe78847mUajYaGhoWzixIksKyvL5nPSVmiEEEKIndx6TJQQQghREiVRQgghxE6URAkhhBA7URIlhBBC7ERJlBBCCLETJVFCCCHETpRECSGEEDtREiWEEELsREmUEAX89ddfUKvVGDFihNKhEEIcQBWLCFHAU089BX9/fyQmJiIjIwNRUVGKxCH3jieENDbUEiVEZsXFxdi0aROef/55jBgxAsnJyVb3/9///R/69OkDb29vNGvWDKNHjxbu0+l0mDNnDqKjo6HRaBAbGyts5pycnIygoCCr1/r+++/BcZxw/Z133kH37t2xZs0axMTECPvQbtu2Dffccw+CgoIQGhqKhx56COfOnbN6ratXr2LcuHEICQmBn58fevfujf379+PixYtQqVRITU21evyyZcvQunVrq51WCGlsKIkSIrOvv/4aHTt2RIcOHTBhwgSsXbtW2Drqp59+wujRozF8+HAcPnwYKSkpVps1T5o0CRs2bMCHH36IU6dO4ZNPPoG/v79N58/MzMR3332HzZs3Iz09HQC/r2V8fDxSU1ORkpIClUqF0aNHCwmwuLgYAwcORFZWFn744QccOXIEs2fPhslkQps2bRAXF4ekpCSr8yQlJWHKlCmy7itJiOwcLJpPCLFR//792bJlyxhj/E4YzZo1Yzt37mSMMdavXz82fvz4Gp+XkZHBALAdO3bUeH9SUhLTarVWt23ZsoWJf83nzZvHPD09WU5OTp0x3rx5kwFgx44dY4wx9sknn7CAgAB269atGh+/adMmFhwczMrLyxljjB06dIhxHMcuXLhQ53kIcXf0FZEQGWVkZODAgQMYN24cAMDDwwNjx44VumTT09MxZMiQGp+bnp4OtVqNgQMHOhRD69atERYWZnXb2bNnMW7cOLRt2xaBgYFo06YNAODy5cvCuXv06IGQkJAaX3PUqFFQq9XYsmULAL5refDgwcLrENJYeSgdACFNSWJiIgwGg9VEIsYYNBoNVqxYAR8fn1qfW9d9AKBSqYRuYTO9Xl/tcX5+ftVue/jhh9G6dWt89tlniIqKgslkQufOnVFRUdGgc3t5eWHSpElISkrCmDFj8NVXX2H58uV1PoeQxoBaooTIxGAwYN26dViyZAnS09OFy5EjRxAVFYUNGzaga9euSElJqfH5Xbp0gclkwu7du2u8PywsDEVFRSgpKRFuM4951uXWrVvIyMjAm2++iSFDhuDOO+/E7du3rR7TtWtXpKenIy8vr9bXeeqpp/Drr79i1apVMBgMGDNmTL3nJsTtKd2fTEhTsWXLFubl5cXy8/Or3Td79mzWu3dvtnPnTqZSqdjbb7/NTp48yY4ePcoWLVokPG7KlCksOjqabdmyhZ0/f57t3LmTbdq0iTHG2K1bt5ifnx976aWXWGZmJlu/fj2LioqqNibarVs3q3MbjUYWGhrKJkyYwM6ePctSUlJYnz59GAC2ZcsWxhhjOp2O3XHHHezee+9le/fuZefOnWPffvst+/PPP61eq3///szLy4s999xzEv3UCHFt1BIlRCaJiYmIi4uDVqutdt+jjz6K1NRUhISE4JtvvsEPP/yA7t274/7778eBAweEx61evRr/+Mc/MH36dHTs2BFPP/200PIMCQnBl19+iZ9//hldunTBhg0b8M4779Qbl0qlwsaNG3Ho0CF07twZs2bNwuLFi60e4+Xlhe3btyM8PBzDhw9Hly5dsGjRIqjVaqvHPfnkk6ioqMC0adPs+AkR4n6o2AIhRDILFizAN998g6NHjyodCiGyoJYoIcRhxcXFOH78OFasWIEXX3xR6XAIkQ0lUUKIw1544QX06tULgwYNoq5c0qRQdy4hhBBiJ2qJEkIIIXaiJEoIIYTYiZIoIYQQYidKooQQQoidKIkSQgghdqIkSgghhNiJkighhBBiJ0qihBBCiJ0oiRJCCCF2+n9kxJP+Wm6p+AAAAABJRU5ErkJggg==", + "text/plain": [ + "<Figure size 500x300 with 1 Axes>" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "---Cypher-RAG based mean and std---\n", + "0.51674\n", + "0.02966020828577499\n", + "\n", + "---KG-RAG based mean and std---\n", + "0.7417666666666666\n", + "0.0259921141887304\n" + ] + } + ], + "source": [ + "cypher_rag_vs_kg_rag_fig = plot_figure(cypher_rag_correct_frac_list, kg_rag_correct_frac_list)\n", + "\n", + "fig_path = '../data/results/figures'\n", + "os.makedirs(fig_path, exist_ok=True)\n", + "cypher_rag_vs_kg_rag_fig.savefig(os.path.join(fig_path, 'cypher_rag_vs_kg_rag_mcq.svg'), format='svg', bbox_inches='tight') \n", + "\n", + "print('---Cypher-RAG based mean and std---')\n", + "print(np.mean(cypher_rag_correct_frac_list))\n", + "print(np.std(cypher_rag_correct_frac_list))\n", + "print('')\n", + "print('---KG-RAG based mean and std---')\n", + "print(np.mean(kg_rag_correct_frac_list))\n", + "print(np.std(kg_rag_correct_frac_list))\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5b623550-d435-48cc-8c91-801262989bce", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python (cypher_rag)", + "language": "python", + "name": "cypher_rag" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} |