path: root/notebooks/disease_extraction_comparison.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "860ebc4a-63e5-462d-b6ab-9bae23d10afb",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.chdir('..')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "851d771c-15b4-4168-acf5-86bdd15d9610",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/kg_rag_test_2/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "from kg_rag.utility import *\n",
    "from tqdm import tqdm\n",
    "import pandas as pd\n",
    "import spacy\n",
    "import scispacy\n",
    "from scispacy.linking import EntityLinker\n",
    "from transformers import pipeline\n",
    "from transformers import AutoModelForTokenClassification\n",
    "from IPython.display import clear_output"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f242aeb6-99f7-496a-8fd8-1f0d964a2556",
   "metadata": {},
   "source": [
    "## List the NER methods to compare"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "45fdafb8-65cc-44dd-b22d-f17e5e807b49",
   "metadata": {},
   "outputs": [],
   "source": [
    "method_list = ['gpt', 'biomed-ner-all', 'scispacy']\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ddc073a0-2508-410e-8e39-4bd94020bf8a",
   "metadata": {},
   "source": [
    "## Load spacy and bert based models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "07a1ccc1-826a-4986-b7a7-f7bf26dd1d8c",
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = spacy.load(\"en_core_sci_sm\")        \n",
    "nlp.add_pipe(\"scispacy_linker\", config={\"resolve_abbreviations\": True, \"linker_name\": \"umls\"})\n",
    "\n",
    "\n",
    "biomed_ner_all_tokenizer = AutoTokenizer.from_pretrained(\"d4data/biomedical-ner-all\",\n",
    "                                         revision=\"main\",\n",
    "                                         cache_dir=config_data['LLM_CACHE_DIR'])\n",
    "biomed_ner_all_model = AutoModelForTokenClassification.from_pretrained(\"d4data/biomedical-ner-all\",                                             \n",
    "                                    torch_dtype=torch.float16,\n",
    "                                    revision=\"main\",\n",
    "                                    cache_dir=config_data['LLM_CACHE_DIR']\n",
    "                                    )\n",
    "clear_output()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4cf3589a-6dec-41e5-9f43-703b0171e79c",
   "metadata": {},
   "source": [
    "## Load evaluation dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "108796a6-5887-464b-8394-04e04b017d0b",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv('data/dataset_for_entity_retrieval_accuracy_analysis.csv')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ed5fca24-2b5d-4696-bcd4-ba911dce6624",
   "metadata": {},
   "source": [
    "## Custom functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "6e262f71-eac1-4894-8390-f1f4c2e8f84f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def entity_extraction(text, method):\n",
    "    if method == 'gpt':\n",
    "        start_time = time.time()\n",
    "        entity = disease_entity_extractor_compare_version(text)\n",
    "        run_time = time.time()-start_time\n",
    "    elif method == 'scispacy':\n",
    "        start_time = time.time()\n",
    "        entity = disease_entity_extractor_scispacy(text)\n",
    "        run_time = time.time()-start_time\n",
    "    elif method == 'biomed-ner-all':\n",
    "        start_time = time.time()\n",
    "        entity = disease_entity_extractor_biomed_ner(text)\n",
    "        run_time = time.time()-start_time\n",
    "    return entity, run_time\n",
    "\n",
    "def get_GPT_response_compare_version(instruction, system_prompt, chat_model_id, chat_deployment_id, temperature=0):\n",
    "    return fetch_GPT_response(instruction, system_prompt, chat_model_id, chat_deployment_id, temperature)\n",
    "    \n",
    "def disease_entity_extractor_compare_version(text):\n",
    "    chat_model_id, chat_deployment_id = get_gpt35()\n",
    "    prompt_updated = system_prompts[\"DISEASE_ENTITY_EXTRACTION\"] + \"\\n\" + \"Sentence : \" + text\n",
    "    resp = get_GPT_response_compare_version(prompt_updated, system_prompts[\"DISEASE_ENTITY_EXTRACTION\"], chat_model_id, chat_deployment_id, temperature=0)\n",
    "    try:\n",
    "        entity_dict = json.loads(resp)\n",
    "        return entity_dict[\"Diseases\"]\n",
    "    except:\n",
    "        return None\n",
    "\n",
    "def disease_entity_extractor_scispacy(text):\n",
    "    doc = nlp(text)\n",
    "    disease_semantic_types = {\"T047\", \"T191\"}    \n",
    "    entity = []\n",
    "    for ent in doc.ents:\n",
    "        if ent._.kb_ents:\n",
    "            umls_cui = ent._.kb_ents[0][0]\n",
    "            umls_entity = nlp.get_pipe(\"scispacy_linker\").kb.cui_to_entity[umls_cui]\n",
    "            if any(t in disease_semantic_types for t in umls_entity.types):\n",
    "                entity.append(ent.text)\n",
    "    return entity\n",
    "\n",
    "def disease_entity_extractor_biomed_ner(text):\n",
    "    pipe = pipeline(\"ner\", model=biomed_ner_all_model, tokenizer=biomed_ner_all_tokenizer, aggregation_strategy=\"simple\", device=0)\n",
    "    out = pipe(text)\n",
    "    return list(filter(None, map(lambda x:x['word'] if x['entity_group']=='Disease_disorder' or x['entity_group']=='Sign_symptom' else None, out)))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "65c46409-f3dd-45e6-9ea8-da84cd8db212",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing method : gpt, 1/3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "322it [03:07,  1.71it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing method : biomed-ner-all, 2/3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "322it [00:05, 63.45it/s] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing method : scispacy, 3/3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "322it [00:04, 72.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 11.3 s, sys: 1.71 s, total: 13 s\n",
      "Wall time: 3min 17s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "comparison_out = []\n",
    "for method_index, method in enumerate(method_list):\n",
    "    print(f'Processing method : {method}, {method_index+1}/{len(method_list)}')\n",
    "    for row_index, row in tqdm(data.iterrows()):\n",
    "        entity, run_time = entity_extraction(row['text'], method)\n",
    "        comparison_out.append((row['text'], row['node_hits'], entity, run_time, method))\n",
    "\n",
    "comparison_out_df = pd.DataFrame(comparison_out, columns=['input_text', 'node_hits', 'entity_extracted', 'run_time_per_text', 'ner_method'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "3fbfbfed-3dd6-4e86-8fac-e0ee40d2c363",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>input_text</th>\n",
       "      <th>node_hits</th>\n",
       "      <th>entity_extracted</th>\n",
       "      <th>run_time_per_text</th>\n",
       "      <th>ner_method</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>LIRAGLUTIDE TREATS OBESITY</td>\n",
       "      <td>obesity</td>\n",
       "      <td>[obesity]</td>\n",
       "      <td>2.214761</td>\n",
       "      <td>gpt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>disease ontology identifier for central diabet...</td>\n",
       "      <td>central diabetes insipidus</td>\n",
       "      <td>[central diabetes insipidus]</td>\n",
       "      <td>0.549497</td>\n",
       "      <td>gpt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Xeroderma pigmentosum, group G is not associat...</td>\n",
       "      <td>xeroderma pigmentosum</td>\n",
       "      <td>[Xeroderma pigmentosum]</td>\n",
       "      <td>0.926769</td>\n",
       "      <td>gpt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>cherubism is not a autosomal dominant disease</td>\n",
       "      <td>cherubism</td>\n",
       "      <td>[cherubism, autosomal dominant disease]</td>\n",
       "      <td>0.675068</td>\n",
       "      <td>gpt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>MASA SYNDROME (DISORDER) IS NOT ASSOCIATED WIT...</td>\n",
       "      <td>MASA syndrome</td>\n",
       "      <td>[MASA SYNDROME]</td>\n",
       "      <td>0.465556</td>\n",
       "      <td>gpt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>961</th>\n",
       "      <td>antineoplastic agents treats osteosarcoma</td>\n",
       "      <td>osteosarcoma</td>\n",
       "      <td>[osteosarcoma]</td>\n",
       "      <td>0.012946</td>\n",
       "      <td>scispacy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>962</th>\n",
       "      <td>timothy syndrome associates gene cacna1c</td>\n",
       "      <td>Timothy syndrome</td>\n",
       "      <td>[syndrome]</td>\n",
       "      <td>0.011308</td>\n",
       "      <td>scispacy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>963</th>\n",
       "      <td>piebaldism is a autosomal dominant disease</td>\n",
       "      <td>piebaldism</td>\n",
       "      <td>[autosomal dominant disease]</td>\n",
       "      <td>0.012271</td>\n",
       "      <td>scispacy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>964</th>\n",
       "      <td>Disease ontology identifier for Loeys-Dietz sy...</td>\n",
       "      <td>Loeys-Dietz syndrome</td>\n",
       "      <td>[Loeys-Dietz syndrome]</td>\n",
       "      <td>0.012468</td>\n",
       "      <td>scispacy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>965</th>\n",
       "      <td>NOONAN SYNDROME ASSOCIATES GENE PTPN11</td>\n",
       "      <td>Noonan syndrome</td>\n",
       "      <td>[NOONAN SYNDROME]</td>\n",
       "      <td>0.010858</td>\n",
       "      <td>scispacy</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>966 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            input_text  \\\n",
       "0                           LIRAGLUTIDE TREATS OBESITY   \n",
       "1    disease ontology identifier for central diabet...   \n",
       "2    Xeroderma pigmentosum, group G is not associat...   \n",
       "3        cherubism is not a autosomal dominant disease   \n",
       "4    MASA SYNDROME (DISORDER) IS NOT ASSOCIATED WIT...   \n",
       "..                                                 ...   \n",
       "961          antineoplastic agents treats osteosarcoma   \n",
       "962           timothy syndrome associates gene cacna1c   \n",
       "963         piebaldism is a autosomal dominant disease   \n",
       "964  Disease ontology identifier for Loeys-Dietz sy...   \n",
       "965             NOONAN SYNDROME ASSOCIATES GENE PTPN11   \n",
       "\n",
       "                      node_hits                         entity_extracted  \\\n",
       "0                       obesity                                [obesity]   \n",
       "1    central diabetes insipidus             [central diabetes insipidus]   \n",
       "2         xeroderma pigmentosum                  [Xeroderma pigmentosum]   \n",
       "3                     cherubism  [cherubism, autosomal dominant disease]   \n",
       "4                 MASA syndrome                          [MASA SYNDROME]   \n",
       "..                          ...                                      ...   \n",
       "961                osteosarcoma                           [osteosarcoma]   \n",
       "962            Timothy syndrome                               [syndrome]   \n",
       "963                  piebaldism             [autosomal dominant disease]   \n",
       "964        Loeys-Dietz syndrome                   [Loeys-Dietz syndrome]   \n",
       "965             Noonan syndrome                        [NOONAN SYNDROME]   \n",
       "\n",
       "     run_time_per_text ner_method  \n",
       "0             2.214761        gpt  \n",
       "1             0.549497        gpt  \n",
       "2             0.926769        gpt  \n",
       "3             0.675068        gpt  \n",
       "4             0.465556        gpt  \n",
       "..                 ...        ...  \n",
       "961           0.012946   scispacy  \n",
       "962           0.011308   scispacy  \n",
       "963           0.012271   scispacy  \n",
       "964           0.012468   scispacy  \n",
       "965           0.010858   scispacy  \n",
       "\n",
       "[966 rows x 5 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comparison_out_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ebb106e7-9f63-475c-acec-61dffbda4f98",
   "metadata": {},
   "outputs": [],
   "source": [
    "comparison_out_df_gpt = comparison_out_df[comparison_out_df.ner_method=='gpt']\n",
    "comparison_out_df_biomed_ner_all = comparison_out_df[comparison_out_df.ner_method=='biomed-ner-all']\n",
    "comparison_out_df_scispacy = comparison_out_df[comparison_out_df.ner_method=='scispacy']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "b6524af1-912a-44e0-8687-3e9ff65d14e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def calculate_precision_recall(row):\n",
    "    # Convert to lowercase and split node_hits into a list\n",
    "    true_entities = set([row['node_hits'].lower()])\n",
    "    \n",
    "    # Convert extracted_entity list to lowercase\n",
    "    predicted_entities = set([entity.lower() for entity in row['entity_extracted']])\n",
    "    \n",
    "    # Calculate true positives, false positives, and false negatives\n",
    "    true_positives = len(true_entities.intersection(predicted_entities))\n",
    "    false_positives = len(predicted_entities - true_entities)\n",
    "    false_negatives = len(true_entities - predicted_entities)\n",
    "    \n",
    "    # Calculate precision and recall\n",
    "    precision = true_positives / (true_positives + false_positives) if (true_positives + false_positives) > 0 else 0\n",
    "    recall = true_positives / (true_positives + false_negatives) if (true_positives + false_negatives) > 0 else 0\n",
    "    \n",
    "    return pd.Series({'precision': precision, 'recall': recall})\n",
    "\n",
    "comparison_out_df_gpt[['precision', 'recall']] = comparison_out_df_gpt.apply(calculate_precision_recall, axis=1)\n",
    "comparison_out_df_biomed_ner_all[['precision', 'recall']] = comparison_out_df_biomed_ner_all.apply(calculate_precision_recall, axis=1)\n",
    "comparison_out_df_scispacy[['precision', 'recall']] = comparison_out_df_scispacy.apply(calculate_precision_recall, axis=1)\n",
    "\n",
    "clear_output()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "09112cfd-43a3-4bdd-8128-e872f5ede03a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9549689440993789\n",
      "0.9968944099378882\n"
     ]
    }
   ],
   "source": [
    "print(comparison_out_df_gpt.precision.mean())\n",
    "print(comparison_out_df_gpt.recall.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "3814b05f-8708-428c-8c37-27160feb3ed7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.23680124223602483\n",
      "0.2795031055900621\n"
     ]
    }
   ],
   "source": [
    "print(comparison_out_df_biomed_ner_all.precision.mean())\n",
    "print(comparison_out_df_biomed_ner_all.recall.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "9496c72c-2976-4bdd-bde5-6b8612461853",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5910973084886129\n",
      "0.6428571428571429\n"
     ]
    }
   ],
   "source": [
    "print(comparison_out_df_scispacy.precision.mean())\n",
    "print(comparison_out_df_scispacy.recall.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "075fa7ce-e463-459c-88e6-00d3db62682f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAekAAAEiCAYAAADd4SrgAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/H5lhTAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA6w0lEQVR4nO3deVhU9f4H8PewoyBiCqgXI0PNpQBRcUGvJq5dvaalqbnmkuUWaoqZ5M8U65ZiuWDX7ep1KzU1UUwxS9TUVHDLLRfQAMEFBGUQ+Pz+4HJ0YlBAZuYw8349D8/DfOec4TMDbz5nzpzzPRoRERAREZHqWJm6ACIiItKPTZqIiEil2KSJiIhUik2aiIhIpdikiYiIVIpNmoiISKXYpImIiFSKTZqIiEil2KSJiIhUik2aiIhIpUzapH/55Rd069YNNWrUgEajwZYtW566zr59+9C4cWPY29vD29sbK1euNHidRPRkzDKRYZi0SWdmZsLHxwcLFy4s1vJXrlzBa6+9hnbt2iE2Nhbjx4/HsGHDsGvXLgNXSkRPwiwTGYZGLRfY0Gg0+P7779GjR48il5k8eTIiIyNx+vRpZeytt97C3bt3ERUVZYQqiehpmGWislOuPpM+dOgQgoKCdMY6deqEQ4cOmagiIioNZpmoeGxMXUBJJCUlwd3dXWfM3d0d6enpePDgARwdHQuto9VqodVqldsiguzsbFStWhUajcbgNRNRYcwyUfGUqyZdGmFhYZgxY0ah8bS0NFSqVMkEFRFRaZQmy15TIguN3d67FPeObinxz3du2gNVXh2mM3Z1zmslfhyikihXu7s9PDyQnJysM5acnIxKlSrp3fIGgJCQEKSlpSlfCQkJxiiViJ6AWSYqnnL1TrpFixbYsWOHztju3bvRokWLItext7eHvb29oUsjohIwZZYrNeuJig3alng9a6cqz/yziUrKpE06IyMDly5dUm5fuXIFsbGxqFKlCmrVqoWQkBDcuHEDq1atAgC8++67WLBgAT788EMMHToUe/fuxbfffovIyMK7tIjIeMpTlm2cqsCGDZfKCZPu7v7tt9/g5+cHPz8/AEBwcDD8/Pwwffp0AEBiYiLi4+OV5V944QVERkZi9+7d8PHxwZdffomlS5eiU6dOJqmfiPIxy0SGoZrzpI0lPT0dLi4uJT5wLDExEYmJiSX+edWrV0f16tVLvB4RPVlxsqzvwLGyxAPHyNDK1WfSprRkyRK9R5Y+TWhoKD755JOyL4iIiMwem3QxjRw5Et27d9cZe/DgAQIDAwEAMTExeo9K5btoIiIqLTZpPfTtIsvJuI3cjNs6Y/IwW/m+16KD0NjaFVrPWs9BKtxFRkRExcEmXUwZsTuRdmBdkfcnr/1Q77hLq76oHNjfUGUREZEZY5MuJiffLnD0Dijxejy3koiISotNuph4biURERlbuZoWlIiIyJKwSRMREakUmzQREZFKsUkTERGpFJs0ERGRSrFJExERqRSbNBERkUqxSRMREakUmzQREZFKsUkTERGpFJs0ERGRSrFJExERqRSbNBERkUqxSRMREakUmzQREZFKsUkTERGpFJs0ERGRSrFJExERqRSbNBERkUqxSRMREamUyZv0woUL4eXlBQcHBwQEBODIkSNPXD48PBz16tWDo6MjPD098cEHHyArK8tI1RJRUZhlorJn0ia9YcMGBAcHIzQ0FMePH4ePjw86deqEmzdv6l1+7dq1mDJlCkJDQ/H7779j2bJl2LBhA6ZOnWrkyonoccwykWGYtEnPnTsXw4cPx5AhQ9CgQQNERESgQoUKWL58ud7lDx48iFatWqFfv37w8vJCx44d0bdv36dusRORYTHLRIZhsiadnZ2NY8eOISgo6FExVlYICgrCoUOH9K7TsmVLHDt2TAny5cuXsWPHDnTt2tUoNRNRYcwykeHYmOoHp6amIjc3F+7u7jrj7u7uOHfunN51+vXrh9TUVAQGBkJEkJOTg3ffffeJu8i0Wi20Wq1yOz09vWyeABEBYJaJDMnkB46VxL59+zB79mwsWrQIx48fx+bNmxEZGYmZM2cWuU5YWBhcXFyUL09PTyNWTET6MMtExWOyd9JVq1aFtbU1kpOTdcaTk5Ph4eGhd52PP/4YAwYMwLBhwwAAL7/8MjIzMzFixAh89NFHsLIqvM0REhKC4OBg5XZ6ejrDTVSGmGUiwzHZO2k7Ozv4+/sjOjpaGcvLy0N0dDRatGihd5379+8XCq+1tTUAQET0rmNvb49KlSrpfBFR2WGWiQzHZO+kASA4OBiDBg1CkyZN0KxZM4SHhyMzMxNDhgwBAAwcOBA1a9ZEWFgYAKBbt26YO3cu/Pz8EBAQgEuXLuHjjz9Gt27dlIATkfExy0SGYdIm3adPH6SkpGD69OlISkqCr68voqKilANQ4uPjdba2p02bBo1Gg2nTpuHGjRuoVq0aunXrhlmzZpnqKRARmGUiQ9FIUfuWzFR6ejpcXFyQlpZW5O4yrymRBq3h6pzXDPr4RJaAWSZLUK6O7iYiIrIkbNJEREQqxSZNRESkUmzSREREKsUmTUREpFJs0kRERCrFJk1ERKRSbNJEREQqxSZNRESkUmzSREREKsUmTUREpFJs0kRERCrFJk1ERKRSbNJEREQqxSZNRESkUmzSREREKsUmTUREpFI2pi6AiIhILRITE5GYmFji9apXr47q1auXeT1s0kRERP+zZMkSzJgxo8TrhYaG4pNPPinzetikiYiI/mfkyJHo3r27ztiDBw8QGBgIAIiJiYGjo2Oh9QzxLhpgkyYiIlLo222dmZmpfO/r64uKFSsarR4eOEZERKRSpXonnZubi5UrVyI6Oho3b95EXl6ezv179+4tk+KIiIgsWama9Lhx47By5Uq89tpraNSoETQaTVnXRUREZFBeUyKLtVxedpbyff2Po2Bl51Cs9a7Oea1UdT2uVE16/fr1+Pbbb9G1a9dnLoCIiIj0K9Vn0nZ2dvD29i7rWoiIiOgxpWrSEyZMwPz58yEiz1zAwoUL4eXlBQcHBwQEBODIkSNPXP7u3bt4//33Ub16ddjb26Nu3brYsWPHM9dB9KwSExNx/PjxEn+VZuIENWKWicpeqXZ3x8TE4KeffsLOnTvRsGFD2Nra6ty/efPmYj3Ohg0bEBwcjIiICAQEBCA8PBydOnXC+fPn4ebmVmj57OxsdOjQAW5ubti4cSNq1qyJa9euoXLlyqV5GkRlSm2TIBgTs0zmIifjNnIzbuuMycNs5fvs5MvQ2NoVWs/aqQpsnKqUeT2latKVK1fG66+//sw/fO7cuRg+fDiGDBkCAIiIiEBkZCSWL1+OKVOmFFp++fLluH37Ng4ePKhsGHh5eT1zHURlQW2TIBgTs0zmIiN2J9IOrCvy/uS1H+odd2nVF5UD+5d5PaVq0itWrHjmH5ydnY1jx44hJCREGbOyskJQUBAOHTqkd51t27ahRYsWeP/997F161ZUq1YN/fr1w+TJk2Ftbf3MNRE9C7VNgmAszDKZEyffLnD0DijxetYGeBcNPOOMYykpKTh//jwAoF69eqhWrVqx101NTUVubi7c3d11xt3d3XHu3Dm961y+fBl79+5F//79sWPHDly6dAnvvfceHj58iNDQUL3raLVaaLVa5XZ6enqxaySip2OWyZzYGGi3dWmV6sCxzMxMDB06FNWrV0ebNm3Qpk0b1KhRA++88w7u379f1jUq8vLy4Obmhm+++Qb+/v7o06cPPvroI0RERBS5TlhYGFxcXJQvT09Pg9VHRMXDLBMVT6neSQcHB+Pnn3/GDz/8gFatWgHI/7xt7NixmDBhAhYvXvzUx6hatSqsra2RnJysM56cnAwPDw+961SvXh22trY6u8Pq16+PpKQkZGdnw86u8If5ISEhCA4OVm6np6cz3PTMDDkJQllMgPAkJ0+eLPayr7zyylOXYZaJDKdUTXrTpk3YuHEj2rZtq4x17doVjo6O6N27d7GatJ2dHfz9/REdHY0ePXoAyN+6jo6OxujRo/Wu06pVK6xduxZ5eXmwssrfCXDhwgVUr15db6gBwN7eHvb29iV7gkRmzNfXFxqNpshTKAvu02g0yM3NferjMctEhlOqJn3//v1Cnz8BgJubW4l2dwcHB2PQoEFo0qQJmjVrhvDwcGRmZipHiA4cOBA1a9ZEWFgYAGDUqFFYsGABxo0bhzFjxuDixYuYPXs2xo4dW5qnQWSRrly5UuaPySwTGUapmnSLFi0QGhqKVatWwcEhf/fdgwcPMGPGDLRo0aLYj9OnTx+kpKRg+vTpSEpKgq+vL6KiopQNgPj4eGUrGwA8PT2xa9cufPDBB3jllVdQs2ZNjBs3DpMnTy7N0yCySM8//3yZPyazTGQYpWrS8+fPR6dOnfC3v/0NPj4+AIC4uDg4ODhg165dJXqs0aNHF7lLbN++fYXGWrRogV9//bXENRMZmtomQSjKtm3bir3sX8/7fhJmmajslapJN2rUCBcvXsSaNWuUUyz69u2L/v37652sgcgSqG0ShKIUfG78NMX9TJqIDKfU50lXqFABw4cPL8taiMo1tU2CUJS/Xv+diNSr2E1627Zt6NKlC2xtbZ+6u6wku8iIzIXaJkEgovKv2E26R48eSEpKgpub2xN3l3EXGVH5kpmZiZ9//hnx8fHIzs7WuY9HWxOZVrGb9OO7yLi7jMg8nDhxAl27dsX9+/eRmZmJKlWqIDU1FRUqVICbmxubNJGJlWpaUH3u3r1bVg9FREbywQcfoFu3brhz5w4cHR3x66+/4tq1a/D398cXX3xh6vKILF6pmvRnn32GDRs2KLfffPNNVKlSBTVr1kRcXFyZFUdEhhUbG4sJEybAysoK1tbW0Gq18PT0xOeff46pU6eaujwii1eqJh0REaHMmbt7927s2bMHUVFR6NKlCyZNmlSmBRKR4dja2iqTjLi5uSE+Ph4A4OLigoSEBFOWRkQo5SlYSUlJSpPevn07evfujY4dO8LLywsBASU/BYWITMPPzw9Hjx5FnTp18Pe//x3Tp09HamoqVq9ejUaNGpm6PCKLV6p30q6urspWdlRUFIKCggAAIsIju4nKkdmzZ6N69eoAgFmzZsHV1RWjRo1CSkoKlixZYuLqiKhU76R79uyJfv36oU6dOrh16xa6dOkCIP9IUW9v7zItkIgMp0mTJsr3bm5uiIqKMmE1RPRXpWrS8+bNg5eXFxISEvD555/DyckJAJCYmIj33nuvTAskIsO5cuUKcnJyUKdOHZ3xixcvwtbWFl5eXqYpjIgAlLJJ29raYuLEiYXGP/jgg2cuiIiMZ/DgwRg6dGihJn348GEsXbpU74UxiMh4OC0okQU7ceIEWrVqVWi8efPmRV7RioiMh9OCElkwjUaDe/fuFRpPS0tjjolUoNhHd+fl5cHNzU35vqgvBpuo/GjTpg3CwsJ0cpubm4uwsDAEBgaasDIiAp7hUpVEVP599tlnaNOmDerVq4fWrVsDAPbv34/09HTs3bvXxNURUanOkx47diy++uqrQuMLFizA+PHjn7UmIjKSBg0a4OTJk+jduzdu3ryJe/fuYeDAgTh37hwnMyFSgVK9k960aZPeg8datmyJOXPmIDw8/FnrIiIjqVGjBmbPnm3qMohIj1K9k7516xZcXFwKjVeqVAmpqanPXBQRGc/+/fvx9ttvo2XLlrhx4wYAYPXq1YiJiTFxZURUqibt7e2td2ainTt3onbt2s9cFBEZx6ZNm9CpUyc4Ojri+PHj0Gq1APKP7ua7ayLTK9Xu7uDgYIwePRopKSl49dVXAQDR0dH48ssvuaubqBz59NNPERERgYEDB2L9+vXKeKtWrfDpp5+asDIiAkrZpIcOHQqtVotZs2Zh5syZAAAvLy8sXrwYAwcOLNMCichwzp8/jzZt2hQad3Fxwd27d41fEBHpKPUpWKNGjVKuluPo6KjM301E5YeHhwcuXbpUaI7umJgYfnRFpAKl+kwaAHJycrBnzx5s3rwZIgIA+PPPP5GRkVFmxRGRYQ0fPhzjxo3D4cOHodFo8Oeff2LNmjWYMGECRo0aZeryiCxeqd5JX7t2DZ07d0Z8fDy0Wi06dOgAZ2dnfPbZZ9BqtYiIiCjrOonIAKZMmYK8vDy0b98e9+/fR5s2bWBvb49JkyZh2LBhpi6PyOKV6p30uHHj0KRJE9y5cweOjo7K+Ouvv47o6OgSP97ChQvh5eUFBwcHBAQE4MiRI8Vab/369dBoNE+cS5yIiqbRaPDRRx/h9u3bOH36NH799VekpKTAxcUFL7zwQokeizkmKnulatL79+/HtGnTYGdnpzPu5eWlnGdZXBs2bEBwcDBCQ0Nx/Phx+Pj4oFOnTrh58+YT17t69SomTpyoTGVIRMWn1WoREhKCJk2aoFWrVtixYwcaNGiAM2fOoF69epg/f36JLj3LHBMZRqmadFEX0rh+/TqcnZ1L9Fhz587F8OHDMWTIEDRo0AARERGoUKECli9fXuQ6ubm56N+/P2bMmMGDW4hKYfr06Vi8eDG8vLxw5coVvPnmmxgxYgTmzZuHL7/8EleuXMHkyZOL/XjMMZFhlKpJd+zYUed8aI1Gg4yMDISGhqJr167Ffpzs7GwcO3YMQUFBjwqyskJQUBAOHTpU5Hr/93//Bzc3N7zzzjulKZ/I4n333XdYtWoVNm7ciB9//BG5ubnIyclBXFwc3nrrLVhbWxf7sZhjIsMp1YFjX3zxBTp37owGDRogKysL/fr1w8WLF1G1alWsW7eu2I+TmpqK3NxcuLu764y7u7vj3LlzeteJiYnBsmXLEBsbW6yfodVqlVmUACA9Pb3Y9RGZq+vXr8Pf3x8A0KhRI9jb2+ODDz6ARqMp8WMZI8cAs0yWqVRN2tPTE3FxcdiwYQPi4uKQkZGBd955B/3799c5kKys3bt3DwMGDMC///1vVK1atVjrhIWFYcaMGQariag8ys3N1TmmxMbGxmhzHZQmxwCzTJapxE364cOHeOmll7B9+3b0798f/fv3L/UPr1q1KqytrZGcnKwznpycDA8Pj0LL//HHH7h69Sq6deumjOXl5QHI/ydz/vx5vPjiizrrhISEIDg4WLmdnp4OT0/PUtdMZA5EBIMHD4a9vT0AICsrC++++y4qVqyos9zmzZuf+ljGyDHALJNlKnGTtrW1RVZWVpn8cDs7O/j7+yM6Olo5/SIvLw/R0dEYPXp0oeVfeuklnDp1Smds2rRpuHfvHubPn683sPb29so/IiLKN2jQIJ3bb7/9dqkfyxg5Bphlskyl2t39/vvv47PPPsPSpUthY1PqmUUB5F+sY9CgQWjSpAmaNWuG8PBwZGZmYsiQIQCAgQMHombNmggLC4ODg0OhC9FXrlwZAHiBeqISWLFiRZk+HnNMZBil6rBHjx5FdHQ0fvzxR7z88sul2kVWoE+fPkhJScH06dORlJQEX19fREVFKQehxMfHw8qq1LOXEpERMMdEhlGqJl25cmX06tWrzIoYPXq03t1iALBv374nrrty5coyq4OISo85Jip7JWrSeXl5+Ne//oULFy4gOzsbr776Kj755BODHtFNRERkqUq0/2nWrFmYOnUqnJycULNmTXz11Vd4//33DVUbERGRRStRk161ahUWLVqEXbt2YcuWLfjhhx+wZs0a5fQJIiIiKjslatLx8fE6034GBQUp16AlIiKislWiJp2TkwMHBwedMVtbWzx8+LBMiyIiIqISHjj211mKAP0zFZXkFCwiIiLSr0RN+q+zFAHPNlMRERERFa1ETbqsZykiIiKionEKICIiIpVikyYiIlKpZ7s6BhERlWuJiYlITEws8XrVq1dH9erVDVARPY5NmojIgi1ZsgQzZswo8XqhoaH45JNPyr4g0sEmTURkwUaOHInu3bvrjD148ACBgYEAgJiYGL3XZ+C7aONgkyYismD6dltnZmYq3/v6+ha6HDEZDw8cIyIiUik2aSIiIpXi7m4iIgvhNSWyWMvlZWcp39f/OApWdg5PWPqRq3NeK1VdVDS+kyYiIlIpNmkiIiKV4u5uKhVOgEBEZHhs0lQqnACByDzkZNxGbsZtnTF5mK18n518GRpbu0LrWTtVgY1TFYPXZ+nYpKlUOAECkXnIiN2JtAPrirw/ee2HesddWvVF5cD+hiqL/odNmkqFEyAQmQcn3y5w9A4o8XrWfBdtFGzSREQWzIa7rVWNTZqeiudWEhGZBk/BIiIiUilVNOmFCxfCy8sLDg4OCAgIwJEjR4pc9t///jdat24NV1dXuLq6Iigo6InLE5FxMMdEZc/kTXrDhg0IDg5GaGgojh8/Dh8fH3Tq1Ak3b97Uu/y+ffvQt29f/PTTTzh06BA8PT3RsWNH3Lhxw8iVW7acjNvQJl3S+cpOvqzcn518udD92qRLyPnLqR5kHphjIsMw+WfSc+fOxfDhwzFkyBAAQEREBCIjI7F8+XJMmTKl0PJr1qzRub106VJs2rQJ0dHRGDhwoFFqJp62QbqYYyLDMGmTzs7OxrFjxxASEqKMWVlZISgoCIcOHSrWY9y/fx8PHz5ElSo8OtGYeNoGFWCOiQzHpE06NTUVubm5cHd31xl3d3fHuXPnivUYkydPRo0aNRAUFKT3fq1WC61Wq9xOT08vfcGk4GkbVMAYOQaYZbJMJv9M+lnMmTMH69evx/fffw8HB/2n+4SFhcHFxUX58vT0NHKVRPQkxckxwCyTZTJpk65atSqsra2RnJysM56cnAwPD48nrvvFF19gzpw5+PHHH/HKK68UuVxISAjS0tKUr4SEhDKpnYjyGSPHALNMlsmkTdrOzg7+/v6Ijo5WxvLy8hAdHY0WLVoUud7nn3+OmTNnIioqCk2aNHniz7C3t0elSpV0voio7BgjxwCzTJbJ5Ed3BwcHY9CgQWjSpAmaNWuG8PBwZGZmKkeJDhw4EDVr1kRYWBgA4LPPPsP06dOxdu1aeHl5ISkpCQDg5OQEJycnkz0PIkvGHBMZhsmbdJ8+fZCSkoLp06cjKSkJvr6+iIqKUg5CiY+Ph5XVozf8ixcvRnZ2Nt544w2dx+ElEIlMhzkmMgyTN2kAGD16NEaPHq33vn379uncvnr1quELIqISY46Jyl65PrqbiIjInLFJExERqRSbNBERkUqxSRMREakUmzQREZFKsUkTERGpFJs0ERGRSrFJExERqRSbNBERkUqxSRMREakUmzQREZFKsUkTERGpFJs0ERGRSrFJExERqRSbNBERkUqxSRMREakUmzQREZFKsUkTERGpFJs0ERGRSrFJExERqRSbNBERkUqxSRMREakUmzQREZFKsUkTERGpFJs0ERGRSrFJExERqZQqmvTChQvh5eUFBwcHBAQE4MiRI09c/rvvvsNLL70EBwcHvPzyy9ixY4eRKiWiojDHRGXP5E16w4YNCA4ORmhoKI4fPw4fHx906tQJN2/e1Lv8wYMH0bdvX7zzzjs4ceIEevTogR49euD06dNGrpyICjDHRIZh8iY9d+5cDB8+HEOGDEGDBg0QERGBChUqYPny5XqXnz9/Pjp37oxJkyahfv36mDlzJho3bowFCxYYuXIiKsAcExmGSZt0dnY2jh07hqCgIGXMysoKQUFBOHTokN51Dh06pLM8AHTq1KnI5YnIsJhjIsOxMeUPT01NRW5uLtzd3XXG3d3dce7cOb3rJCUl6V0+KSlJ7/JarRZarVa5nZaWBgBIT08vsq487f1i1V9aT/rZasTXozBDvibFfT2cnZ2h0WgMVkdxGSPHALNcFvh66FLD6/G0HJu0SRtDWFgYZsyYUWjc09PTBNXkcwk32Y9WJb4euor7eqSlpaFSpUoGrUVNmGX14+uhqzivx9NybNImXbVqVVhbWyM5OVlnPDk5GR4eHnrX8fDwKNHyISEhCA4OVm7n5eXh9u3beO6558rkXUh6ejo8PT2RkJBgUf8wi8LXozBDvSbOzs5l9ljPwhg5BphlY+ProctUOTZpk7azs4O/vz+io6PRo0cPAPnBi46OxujRo/Wu06JFC0RHR2P8+PHK2O7du9GiRQu9y9vb28Pe3l5nrHLlymVRvo5KlSrxD/kxfD0KM9fXxBg5BphlU+Hrocvor4eY2Pr168Xe3l5WrlwpZ8+elREjRkjlypUlKSlJREQGDBggU6ZMUZY/cOCA2NjYyBdffCG///67hIaGiq2trZw6dcok9aelpQkASUtLM8nPVxu+HoVZwmtS3nMsYhm/p5Lg66HLVK+HyT+T7tOnD1JSUjB9+nQkJSXB19cXUVFRykEl8fHxsLJ6dBB6y5YtsXbtWkybNg1Tp05FnTp1sGXLFjRq1MhUT4HI4jHHRAZi1E0CM5SVlSWhoaGSlZVl6lJUga9HYXxNygf+nnTx9dBlqtdDIyJi6g0FIiIiKszkM44RERGRfmzSREREKsUmTUREpFJs0kRERCrFJk1PpO+4Qh5rmI+vA5UXzHHR1P468OhuPVJTU+Ho6IiKFSuauhSTysvLg5WVFe7cuYNbt27Bzs4Of/vb32BlZaXcR6RmzDJzXN7xt/MXx48fR61atRAbG2vqUkyqILynT5/GP/7xD3To0AEdO3bE9OnT8fDhQwb7f+bPn4+JEyeaugzSg1lmjotLzTnmb+gxcXFxaNu2LUaOHIlWrVoVut9SdjoUBDsuLg4tW7ZE06ZNERERgYYNG2LBggXYunWrqUtUhezsbPz55584c+YMHjx4YDF/H+UBs8wcF5fqc2zUqVNU7OTJk+Ls7CwffvihiIjk5ubK77//Lvv375eLFy8qy+Xl5ZmqRKM6d+6cVKxYUaZOnaqMnT17VjQajXz88cc6y+bk5Bi7PNU4ePCgODg4yNatW01dCv0Ps/wIc1w8as4x30kDePjwIcaMGYOsrCx89tlnEBF0794dAwYMQJs2bfDGG28ol8gri0viqVlubi5yc3Pxr3/9C/b29mjYsKFy36ZNmwAASUlJWLRoEfbu3QutVgtra2tTlWs0eXl5esdbtGiBYcOGYfHixbh9+7aRq6K/YpbzMcf6lcccm/wCG2pga2uLBQsWoFu3bujQoQNyc3Ph4OCAL7/8Evb29oiJicG8efNQuXJlTJ8+3dTlGkTBrrGHDx/CwcEBM2bMwL179xAREQFHR0ecP38eX375JcaPH4/GjRtj0aJFEBEkJCTg73//OwYPHowOHTqY+mkYTMFndzNnzoSNjQ1at26NwMBAAECHDh0QGRmJxMREVKlShQfjmJClZ5k5frJymWPTvpFXl7Nnz8rzzz8vPj4+cuPGDWU8PT1dRo0aJUFBQXLv3j0TVmgYubm5IiJy4sQJadq0qfLcExMTpWfPnlKnTh2pUKGC7Nq1S1lHq9VKWlqazJw5U3r16iW///67SWo3tAMHDsjChQvl66+/ltTUVJkxY4Y0btxY6tatKwMGDJAjR46IiEjPnj3lH//4h4mrpQKWmGXmuGjlOcds0n9x6dIl2bx5s2RnZ4vIo8+tpk+fLn5+fmZ3RZiCYMfGxoqjo6Nyzd+C8cTEROndu7f4+PjIqlWrlPUe//zK3F6TAitWrBBvb28ZPHiwLFu2THlNEhISZPfu3dKsWTNp1qyZtG7dWiZNmiS+vr5y8uRJEbGMzzvVzpKyzBwXrbznmE1aD32/mGHDhsmQIUOUwJuDx7e8HR0ddQ4uERG5c+eOiDzaEm/durUsX75cuf/hw4dGq9XY1q5dKxUqVJDvvvtOtFqt3mWys7MlJiZGRowYIW5ubqLRaOSTTz4xcqX0JJaQZea4aOaQYzbpp0hOTpapU6dK1apV5cyZM6Yup8ydPXtW7OzsJCwsTGc8PDxcgoODJTMzU0QeBbxdu3ayaNEiU5RqNDdu3JCWLVvK7NmzdcYf/4df8I+xQFxcnISGhkqdOnXk7NmzRqmTSsacs8wcF2YuObaoJp2cnCy//PKLHDx4UK5fv/7U5ffu3StDhw6VGjVqyIkTJwxfoJFlZGRIz549pWLFijqvR1hYmFSsWFF++uknEXn0h5yUlCTt27eXrl27yt27d01RslEcP35c3N3d5ZdfftF7f0HI79+/rzN+6tQpqVOnjkRFRRm8RkvHLD/CHOtnLjm2mCZ98uRJqV+/vjRq1Eg8PDykf//+OgeUiBTeqrp48aKsWrVK/vjjD2OWalB/fY4bNmyQLl26SGBgoGRkZMiCBQukSpUqOgeXPO7GjRuSkJBgjFJNZtu2bVKlShW5cuWKiOg/fzQpKUnCw8Pl3r17Oq+pr6+vzJkzx1ilWiRmmTkuDnPJsUU06TNnzkjVqlVl8uTJkpCQICtWrJAaNWooBwf81eLFi5UjP/8aBnNw7do12bJli3J7y5Yt0qFDB6lVq5Y4OTkpRzo+vlto1qxZcvjwYaPXagqnT58WjUYj8+bNU8b++tnm119/LSNGjND5+9i4caNUqVLFbI+QVQNm+RHm+MnMJcdm36Tv3Lkj7du3l9GjR+uMd+zYUdavXy+7d++W06dPi0j+L/DIkSPi6ekpPXr0kLy8PFUc3VeWsrKy5N1335UXX3xRvvvuO2V8y5YtEhQUJA0bNpT4+HgRebTl+cknn4hGozG73YT65OXlSUZGhrz55pvi7u4u33//vTJeICsrS3r16iUfffSRzrqnTp0ym3dqasQsP8IcP5k55djsm/TNmzdl+fLlEhsbq4x9+umnotFoxM/PT/z8/MTNzU12794tIvm/uHXr1qnql1TWDh06JIMGDZImTZrIhg0blPGCLfHAwEC5dOmSiIhMmzZNHBwc5NixY6Yq16CKencVGRkpPj4+4uXlJf/9739FJP+zv9jYWOnUqZM0btxYOSrWnP75qxmzrIs5fsScc2z2TVpE5Pbt28r369evFxsbG9m8ebOkp6fLH3/8IW+88Yb07NlT0tPTTVilYRT1x3v48GF5++23iwx4+/btZeTIkeLo6Ci//fabsco1qsdfm61bt8qKFStkwYIFytgPP/wgbdu2FY1GI6+88op4eXlJQECAtGnTRjl9x5LnOzYFS80yc1w0c8+xWV9PWkQKzc+bm5uLkydPws/PTxl75513EB8fj927dxu7RIMqmNbu6tWruHHjBqpXr47atWsr9x85cgTz58/H+fPnMWnSJPTp0wcAsG3bNnz66ac4e/YsfvnlFzRu3NhUT8FgHv/bmDJlCtatWwcPDw/cuXMHlStXxrfffgsvLy8kJCQgNjYWv/zyC5ycnODv748uXbrA2toaOTk5sLHhzLrGYMlZZo6LZhE5NuUWgiGcP39e1q5dqzNWsBvjr1ujBZ9TDRs2TCZMmCA5OTmq3eVRWjdu3BCNRiMajUaqVq0qQ4cOldmzZ0tSUpLk5eVJfHy8vP3229KiRQud123Hjh1y7do1E1ZuHHPnzpXq1asr7zJWr16t7D69cOFCkeupecvbXDDLjzDHT2bOOVbB7OFlJzY2Fq+88orOVUxyc3Oh0WiQnp6OnJwcneVzcnIQGhqKyMhIDBs2DNbW1mZ3ZRxXV1e0bdsWbm5u6NGjB+7evYtNmzbB19cXzZo1Q1RUFLy9vdGgQQOEh4dj8+bNAIAuXbqgVq1aJq7esG7evImLFy9i3rx58Pf3x9atW/H+++/j888/h4igX79+uHDhAoDC1x+2hCsGmRKzrIs5LprZ59i02whlJy4uTipUqKBcQ1bk0XR3V69elebNm8vOnTuV+3bt2iXvvfeeuLm5yfHjx41erzEUbCVmZmZKu3btpGPHjvLdd99Jbm6u7Ny5U0JDQ8XHx0fq1q0rNjY2otFopGnTppKRkWFW70Ke5IcffpDExEQ5fvy41K5dW/ksa+nSpaLRaKRWrVrKUbJkHMyyLub46cw5x2bRpM+ePStVq1aVAQMGiIjurrA//vhDPD09ZcSIETp/sNu2bZNJkyap5lw4QykIeEZGhrRr104aN24s27dvV/7pJSUlSXx8vMyZM0dGjBihnMJibp52jmxERIQEBQVJSkqKiOQflDRq1Ch59913y8UuMXPBLOvHHOezxByX+wPH4uLi0LJlS7i6ukKr1WLnzp1o0qQJcnNzYW1tjaFDhyIrKwtr1qwptPtLq9XC3t7eRJUbhhRxgI21tTUyMzPxz3/+E2lpaQgJCUG3bt1ga2tbaDlz8/hrsmrVKpw6dQo1atRAo0aNlGvnTpw4Ed9++y3OnTuH7OxsDBo0CE2aNMHHH38MwHxfGzVhlh9hjguz2BybcgvhWR0/flwqVKggU6dOlYyMDOndu7dUqlRJjh49qrOcJezymT17ts6Vbf7q8S3xV199VZo2bSpbt2416yvgiOj+7qdMmSLOzs4SFBQk/v7+UqVKFZk1a5aIiPz555/i6ekpVatWFW9vb2nUqJHZXCWpPGCW8zHH+llyjsttk87JyZHAwECZNGmSMlZwzdTHw23uoRbJv9Ta2LFjRaPRyLp164pc7vGAd+zYUerUqSPbt283VplG9/jv/vDhw9K1a1c5cOCAiOSHOTw8XGxsbOSLL74QkfyLNnz55ZfyzTffKP/0zP2fnxowy/mYY/0sPcflsklnZGRITk6OzqT6Bb/I5ORk6dOnjzg7O+udu9Zcpaeny9SpU8XKykrWrFlT5HIFAb93757885//lMuXLxurRJP5z3/+I127dpXAwEBJS0tTxjMyMuTTTz+V+vXry7lz5wqtV14/wypPmGVdzHHRLDXH5a5JnzlzRjp27Cje3t7i7+8vS5YsKXSpscfDbe5b4Y8fSHHlyhUZN26cWFlZ6Uy8/1dqnwbvWZ08eVIOHjyo3F68eLHUrl1bnJyc5Ndff9VZdv/+/VK5cuVC42R4zPIjzHFhzHG+ctWkY2NjpVKlSjJo0CCZO3euNG3aVNzc3JTp8B7/Q09OTpb+/fuLRqMxy9My/ur777+XJk2aSK9evcTa2lqsrKyeuMvMXK1Zs0b8/Pxk7NixOkf7bty4UerVqye9e/fWmb84ISFBateuLT/++KMpyrVYzLJ+zHE+5viRctOkz5w5I87OzjJlyhSd8Vq1askbb7yhd53ExEQZOnSo3l0g5uTYsWPi4OAgS5YskeTkZDl27JiMGjXK4gK+bNkycXJykoULFyoXFnj8Xcby5cvFz89P2rVrJ2vWrJHIyEjp2rWrNGrUqNzvEitPmGX9mON8zLGuctGk8/LypFevXmJvby8//fST5ObmKrt6hg4dKv/85z8L7SYrYG7XkNVn48aN4uPjI5mZmcpYamqqjBgxQqysrGTbtm0mrM44jhw5Is8//7ysX7++0H13795Vvv/vf/8rtWvXFhsbG+nevbtMmjSpXEyyby6Y5aIxx8yxPuWiSYvkX/2mbdu20qpVK9m6dauIiKSkpIiDg4N89dVXJq7OtLZu3SrW1tbKJfkKtjpjYmKU+X4fv+asOSn4B7969WoJCAjQOaBk586dMn78ePH19ZXXX39dOThp7dq14uPjI2PHjpWTJ0/qPA4ZHrOsH3PMHOuj6rm7r1+/jjVr1mDRokVwdHTExo0bISKYN28eVqxYAV9fXwwbNgxjxowBUHheVnOk7zm2bt0azZs3R1hYGK5du6ac8F+zZk307t0boaGhaNiwobFLNbi7d+8qV6/RarXIyspS5ugdN24cZs2ahaNHj6J9+/Y4deoUevToAQDo27cvxowZg5iYGHz99dc4ceKEuq+CYwaYZV3M8SPM8VOYcgvhSU6fPi0+Pj7y9ttvy4cffqjs6rpz5460a9dONBqNdO3a1Wx3cejz+JZ1eHi4fPjhh7Jr1y7Jzc2V//znP9K8eXMZPHiwnDlzRpKTk2Xq1KnSsmVLs7u2rojIqFGjpEePHsrt2NhYadCggTRo0EA8PDzk+eefl2XLlkliYqKIiBw8eFDs7Oxk3759yjqrV6+W2rVry5gxY0Sr1Rr9OVgKZlkXc/wIc/x0qmzSp0+fFldXV5k2bZrObo/NmzfLwYMH5f79+9KhQwdp3ry5REZGKqE311MRHrdp0yblqNhXX31VGjduLEOGDBERka+//lrat28vGo1GGjRoIK6urnLixAnTFmwgsbGxyj/1e/fuiUj+RAcrVqyQefPmSVpams7fw65du8TX11cuXbqkM75u3TqLOMfUVJhl/ZjjfMzx06muSd+6dUvatGkjo0eP1hmfM2eOaDQaadOmjRw4cEAyMjKkbdu2EhgYKJs3bzb7UIvkX1+3du3aEhERISIily5dkooVK8rEiROVZe7fvy+7d++WPXv2lNurvpTEypUrxc3NTdnS1ufBgwfSvXt3ef3115UmYO4HIakBs6wfc1wYc1w01TXps2fPyosvvih79+5VfgGLFy8WW1tbWbhwoXTo0EE6duwoBw8elMzMTHn55Zelc+fOkpGRYeLKy07B8y7Y7Vdw++eff5aXX35ZREQuX74stWrVkuHDhyvrHTlyxCwPnHjcX0O5f/9+CQgIkEaNGklycrKIiM6WeUxMjHTu3FlnDl9LCLYaWHqWmeOiMcfFp7omvXr1arG2ttbZmk5ISJBffvlFREROnTol7du3Fz8/P7l586bcunVLrly5YqJqy17B8z5//rwEBwfr7ML5+eefpUOHDnLx4kXlkn0F/wB+/fVXmTBhgnJkqDl6PJT79u2TCxcuiEj+c2/durXUr19fCfj9+/dl+PDhEhQUJK+//rpZzOFb3lhylpnjojHHJaO6Jr1//36xt7eXTZs2iYjuZ1MFv9xvvvlGmjZtKgkJCSap0dDu3LkjL730kjg4OMi7776rnNCfmJgorq6uotFoZMyYMTrrjB8/Xl599VW5deuWKUo2uMf/DiZPniwvvfSSrFixQu7duyd5eXly4MABad26tTRo0EAJ+MmTJ2XPnj3K340lBVsNLD3LzHFhzHHJqa5JJyQkiJubm3Tv3l2uXr2qd5kJEybIm2++aZZHO4rkhzsgIECqVasmPXr0kGHDhsnFixdFRGT37t3i6uoqI0aMkFOnTslvv/0mEyZMEBcXFzl16pSJKze8zz//XNzc3OTnn38utFv0wIED0qZNG2nYsKHOBRtELGfXmJpYepaZ46Ixx8WnuiYtkj/zjp2dnQwYMEDOnDmjjKelpcmkSZPE1dVVTp8+bcIKDafgj3Dv3r3SrFkzGT58uLRq1UqGDx+u7DLbtGmTVKtWTTw9PaVevXri7+9vtkd/ZmVlKd+np6dL27ZtJTw8XGeZx0/ZOXz4sNStW1f69esnIuZ/lLDaWWqWmWNdzHHpaUTUN2tAbm4uli5ditGjR8Pb2xstW7aEra0tbty4gd9++w07duyAn5+fqcssE3l5ebCyskJubi6sra2VSQ6uXLmCkJAQDB48GLdu3UJ4eDgaN26MqVOnwsvLCykpKbhy5QqcnZ3h5uaG5557zsTPpOz9+OOPiIuLw9///nc0a9YMd+7cga+vL8LCwtCvXz/lNQOArKwsXL9+Hd7e3jh79izq1aun3EemYylZZo6Lxhw/G1XOOGZtbY2RI0ciJiYGDRo0wLFjx3DmzBk0atQI+/fvN4tQA/mzDllZWeHixYuYPHky9u/fD41GA41Gg9q1a+OFF17AhAkT8NZbb2HkyJGIi4vD7Nmz8ccff6BatWpo1qwZ6tevb5bBXrFiBYYOHYorV67Ayir/z9TV1RXPPfcctmzZAiD/7yQvLw8AcP78efz3v/9FSkoKGjRoAGtra+Tm5pqqfPofS8gyc1w05rgMmPJtfHGY++xDqampUrduXdFoNFKhQgWZOHGifP311yKSv4vojTfeUA68mTdvngQGBspbb70l165dM2XZBrVu3TqpUKGCbNiwQZkAo2B31/r16+XFF1+UsWPHKuMPHjyQzp07yz/+8Q+L3i2mduacZea4MOa4bKh+otOCrS8gf4u1YD5bc+Hg4IDXXnsNp06dgpWVFfLy8rBp0yasWrUKw4cPx+3bt7Fnzx707NkT48ePR3Z2Nvbs2QNbW1tTl24QKSkpWLJkCT7//HP07t1bGc/MzMQff/yBihUr4r333sOKFSvg5+eHWrVqITk5Gffv38exY8eg0WjM8u/EHJhzlpljXcxx2VHlZ9KWouCP8N69e5g5cyZOnjyJl19+GTNmzMAXX3yB69evY9myZRARnDt3DnXr1gUA3LlzB66uriau3jBSUlLQtm1bzJo1S5lIf/Hixdi7dy82bdoEb29vODs745tvvsGaNWuQnZ0Nd3d3hISEwMbGBjk5OeY5yT6pFnNcGHNcdvgqqICzszM++ugjzJ49G9HR0ahcuTKmT58OEUHHjh1hZWWFunXrKgenmGuwC6SnpyMyMhKVKlXCokWLcOHCBQQGBiIqKgppaWkICQnB/v37MXfuXJ31cnNzGWwyGeZYF3NcNvhKmMDjW94VK1ZEZmYmXFxcEBISAmtra2zZsgWZmZmYMWMG3njjDWW9x3cXmqtq1aph5cqV6NWrF/bu3QtnZ2eEh4fDx8cHzz33HO7cuYPZs2fj1q1bhda19KNAybiY46Ixx2WHTdrICoK9fft2fPXVV7h16xaee+45jB49Gt27d0dISAhEBHv37oVGo8GMGTMsbquyffv2uHjxIjIyMvDCCy8Uur9SpUrw9PQ0QWVE+Zjjp2OOy4b5b9KpjEajwY4dO9CrVy+0bt0a3bt3h6enJ3r06IEFCxbA2dkZkydPRlBQEDZu3IjZs2ebumSTqFatWqFgp6SkYMCAAdBqtXjnnXdMVBkRc1xczPGzs6xNOxPQarWwt7dXbmdnZ2Pp0qV477338PHHHwPInwihYcOGGDt2LF588UV06dIFkyZNgr29PQYMGGCq0lUjNTUVS5cuRUxMDG7evIkDBw4o509y1xgZA3P87Jjj0uE7aQOaOnUqhg8fjscPoM/OzsaFCxfg4uICAMp948aNQ9++fbF8+XLls61p06bBy8vLFKWryvXr13HgwAF4e3vj4MGDsLW1RU5ODoNNRsEclw3muHT4TtqAevbsCTs7O2g0GmVr0cnJCW3atMGePXswYsQI1KhRA0D+wSTVqlVDXFwcKlasCAA8R/B/fH19sXr1ari4uCivpaV9vkemwxyXDea4dPhO2oCaNGmCV155BXv27EGfPn1w9+5dAPkHVOTk5GDu3LlISkpSQqzVauHm5oasrCzw9HVdlStXViY44JY3GRNzXHaY45LjZkwZKjj/8a+cnJywZcsW2NjYKKclXL58GZs2bcJPP/2E1q1bIzk5Gdu3b8fBgwfh4OBggurLB74rIUNjjg2POS4+zjhWRgqCHR8fj19//RVJSUkYMWKEEtSjR4+ic+fOaNu2LdavXw9bW1ts374d+/btw2+//YbatWsjODgYjRo1MvEzIbJczDGpDZt0GSgI9smTJ9GjRw+4urri8uXL8PDwwPHjx+Ho6AgAOHz4MLp06YJ27dphzZo1SvCzs7NhY2NjEZMcEKkVc0xqxL+mZ1QQ7Li4ODRv3hz9+vXDjh07cPToUWRkZGD79u3KsgEBAdixYwd++uknDB48GHfu3AEA2NnZMdhEJsQck1rxM+lnZGVlhUuXLqF58+aYOHEiZs6cCQBwd3fH888/j7i4OERGRqJTp05o2bIlmjdvjp07d6JFixZwdHTE8uXL+fkMkYkxx6RWbNLPKC8vD8uXL4ezs7PORdvnzJmDQ4cOoVatWrh8+TLWrVuH9957DzNnzkRAQACOHj0KJycnBptIBZhjUis26WdkZWWF0aNH4/79+1i/fj0cHByQnp6OuXPnKlveGo0GY8aMwbJlyzBu3Dg4OTnB39/f1KUT0f8wx6RW/AClDNSoUQNTpkxB06ZNER4ejqlTp2L9+vXo3LkzsrKyAABdunRBtWrVoNVqTVwtEenDHJMa8Z10GfHw8MC0adNgZWUFe3t7nDhxAq+++qpyROiPP/6IatWqwc3NzcSVElFRmGNSGzbpMuTu7o6QkBDk5eXhu+++Q05ODiZPnoxPP/0Uy5Ytw4EDB8z+Qu9E5R1zTGrC86QNICkpCbNmzUJcXBy0Wi1OnjyJmJgYfn5FVI4wx6QG/EzaADw8PPDRRx/B29sbt2/fxqFDhxhsonKGOSY14DtpA0pJSUFeXh7c3d1NXQoRlRJzTKbEJk1ERKRS3N1NRESkUmzSREREKsUmTUREpFJs0kRERCrFJk1ERKRSbNJEREQqxSZNRESkUmzSREREKsUmTUREpFJs0kRERCrFJk1ERKRS/w/YHtTpevyjUAAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 500x300 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "# Assuming you have these dataframes already loaded\n",
    "# comparison_out_df_gpt\n",
    "# comparison_out_df_biomed_ner_all\n",
    "# comparison_out_df_scispacy\n",
    "\n",
    "# Create a list of dataframes and their labels\n",
    "dfs = [comparison_out_df_gpt, comparison_out_df_biomed_ner_all, comparison_out_df_scispacy]\n",
    "labels = ['GPT-3.5', 'BioMed NER', 'SciSpaCy']\n",
    "\n",
    "# Function to calculate SEM\n",
    "def sem(data):\n",
    "    return np.std(data, ddof=1) / np.sqrt(len(data))\n",
    "\n",
    "# Calculate mean and SEM for precision and recall\n",
    "precision_means = [df['precision'].mean() for df in dfs]\n",
    "precision_sems = [sem(df['precision']) for df in dfs]\n",
    "recall_means = [df['recall'].mean() for df in dfs]\n",
    "recall_sems = [sem(df['recall']) for df in dfs]\n",
    "\n",
    "# Set up the plot\n",
    "fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(5, 3))\n",
    "x = np.arange(len(labels))\n",
    "width = 0.35\n",
    "\n",
    "# Function to remove top and right spines\n",
    "def remove_spines(ax):\n",
    "    ax.spines['top'].set_visible(False)\n",
    "    ax.spines['right'].set_visible(False)\n",
    "\n",
    "# Plot precision\n",
    "ax1.bar(x, precision_means, width, yerr=precision_sems, capsize=5)\n",
    "ax1.set_ylabel('Precision')\n",
    "# ax1.set_title('Average Precision')\n",
    "ax1.set_xticks(x)\n",
    "ax1.set_xticklabels(labels, rotation=45, ha='right')\n",
    "ax1.set_ylim(0, 1)\n",
    "remove_spines(ax1)\n",
    "\n",
    "# Plot recall\n",
    "ax2.bar(x, recall_means, width, yerr=recall_sems, capsize=5)\n",
    "ax2.set_ylabel('Recall')\n",
    "# ax2.set_title('Average Recall')\n",
    "ax2.set_xticks(x)\n",
    "ax2.set_xticklabels(labels, rotation=45, ha='right')\n",
    "ax2.set_ylim(0, 1)\n",
    "remove_spines(ax2)\n",
    "\n",
    "# Adjust layout and display\n",
    "plt.tight_layout()\n",
    "plt.show()\n",
    "\n",
    "fig_path = 'data/results/figures'\n",
    "os.makedirs(fig_path, exist_ok=True)\n",
    "fig.savefig(os.path.join(fig_path, 'ner_extraction_comparison.tiff'), format='tiff', bbox_inches='tight')      \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "9af751da-eee0-4d03-9bba-137baf429eae",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.9549689440993789, 0.23680124223602483, 0.5910973084886129] [0.008286258373808576, 0.022466879308773186, 0.025950253677613028]\n",
      "[0.9968944099378882, 0.2795031055900621, 0.6428571428571429] [0.003105590062111801, 0.025047065948613282, 0.02674395944460631]\n"
     ]
    }
   ],
   "source": [
    "print(precision_means, precision_sems)\n",
    "print(recall_means, recall_sems)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "5f2faf89-cdc6-492c-9372-8f1ff6233dd5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5822619658819637"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comparison_out_df_gpt.run_time_per_text.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "c3c44796-55f0-4027-8651-f53fdce6629c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.015508739844612453"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comparison_out_df_biomed_ner_all.run_time_per_text.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "302251b3-4748-4cd2-950a-d4e25ffec4bf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.013423655344092327"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comparison_out_df_scispacy.run_time_per_text.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "362efc28-28c0-46ad-95b2-3d78ca7a6540",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Print all labels\n",
    "# # print(model.config.id2label)\n",
    "\n",
    "# # Or, if you want a list of just the label names\n",
    "# label_names = list(model.config.id2label.values())\n",
    "\n",
    "# set(map(lambda x:x.split('-')[-1], label_names))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "64c523d7-3ed8-4e4a-a1a0-089bd84dd554",
   "metadata": {},
   "outputs": [],
   "source": [
    "# method = method_list[0]\n",
    "# text = data.iloc[25].text\n",
    "# entity, run_time = entity_extraction(text, method)\n",
    "# print(text)\n",
    "# print(entity, run_time, method)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1a60d8c0-fd66-4700-911d-a3e8ac51115e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}