diff options
Diffstat (limited to 'fine_tune')
| -rw-r--r-- | fine_tune/bert/tutorials/06_attn_01.ipynb | 696 | ||||
| -rw-r--r-- | fine_tune/bert/tutorials/06_dive_into.py | 49 |
2 files changed, 696 insertions, 49 deletions
diff --git a/fine_tune/bert/tutorials/06_attn_01.ipynb b/fine_tune/bert/tutorials/06_attn_01.ipynb new file mode 100644 index 0000000..e933ea9 --- /dev/null +++ b/fine_tune/bert/tutorials/06_attn_01.ipynb @@ -0,0 +1,696 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-23T14:15:27.053371Z", + "start_time": "2022-08-23T14:15:27.036312Z" + } + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABBoAAAEICAIAAAB+mAA/AABqo0lEQVR42uy9Z1QUyff43UMOIhJUBAwgKoJZwIA5Z1ZcRTHHrwFzWFHMmDCguGBaZUVFRURRUUB0VUBUJIMoSlBAWEmS08A85799Tv/qqR6GnsgM3M+r6Zru6uo7PVV1q25Q4HA4BAAAAAAAAAAAAP/IgQgAAAAAAAAAAAB1AgAAAAAAAAAAUCcAAAAAAAAAAAB1AgAAAAAAAAAAUCcAAAAAAAAAAABAnQAAAAAAAAAAANQJAAAAAAAAAABAnQAAAAAAAAAAANQJAAAAAAAAAABAnQAAAAAAAAAAAAB1AgAAAAAAAAAAUCcAAAAAAAAAAAB1AgAAAAAAAAAAUCcAAAAAAAAAAAB1AgAAAAAAAAAAANQJAAAAAAAAAABAnQAAAAAAAAAAANQJAAAAAAAAAABAnQAAAAAAAAAAAAB1AgAAAAAAAAAAUCcAAAAAAAAAAAB1AgAAAAAAAAAAUCcAAAAAAAAAAAB1AgAAAAAAAAAAANQJAAAAAAAAAABAnQAAAAAAAAAAANQJAAAAAAAAAABAnQAAAAAAAAAAANQJAAAAAAAAAAAAUCcAAAAAAAAAAAB1AgAAAAAAAAAAUCcAAAAAAAAAAAB1AgAAAAAAAAAAANQJAAAAAAAAAABAnQAAAAAAAAAAANQJAAAAAAAAAABAnQAAAAAAAAAAANQJAAAAAAAAQNK8f//+1atXIAcp5+7du7m5uSAHUCcAAAAAAACkCH9//9GjR+fl5YEopJykpKRhw4ZlZGSAOgEAAAAAAABIBefOnbO1ta2oqABRyASpqanDhg1LTk4GdQIAAAAAAABoYnbs2LFhw4b6+noFBQUjIyMQiJTTrVs3giCys7NHjBgRHR0N6gQAAAAAAADQZGzfvv3EiRPkZw8Pj4EDB4JMpJz58+evX7+eIIj8/PzRo0eHhoaCOgEAAAAAAAA0ATt37jx58iT1eeXKlSATmcDV1XXatGkEQZSUlEycOPHp06ctUAgsDocDrwIAAAAAAEBT4eTkdPjwYfKznZ3drVu3WCwWiEVWKCsrGzFiRExMDEEQioqKN2/enD17NqgTAAAAAAAAgCQ4derUtm3byM/W1tYhISEqKiogFtnix48fgwYNysrKIghCTk7Oz8/PxsYG1AkAAAAAAABAvDx79mzy5Ml1dXUEQZiYmEREROjq6oJYZJG4uLhhw4aVlZURBKGhofHu3buePXuCOgEAAAAAAACIi7S0NEtLy8LCQoIgVFVVo6OjTU1NQSyyi5+f36xZs8jP3bp1e//+fZs2bVrCg4MrNgAAAAAAgKQpLy+fOXMmqUsQBOHi4gK6hKxja2u7aNEi8vOXL1/s7e3r6+tbwoPD7gQAAAAAAICksbOz8/HxIT9PmDAhMDAQ3K+bAcXFxX369Pn+/Tt5uHPnzqNHj4I6AQAAAAAAAIiS48eP79y5k/yspaWVmJior68PYmkevHjxYty4cdQE+86dO3PmzGnejwzGTgAAAAAAAJIjKCho165d1OH58+dBl2hOjBkzZsOGDdTh0qVL4+Pjm/cjw+4EAAAAAACAhEhNTbW0tCwqKiIP7e3tb968CWIRE9evX8/PzxemhoEDB44YMYLfqyorKwcMGPDp0yfy0MjIKDIyUkdHB9QJAAAAAAAAQHDKy8sHDx6cmJhIHhoaGiYkJLSQ4D+Sp6KiQltbu7q6WphKvL29582bJ8CFkZGRQ4cOZbPZ5OHYsWODg4Pl5JqnWRAYOwEAAAAAAEiCvXv3UroEQRCXL18GXUJ8vH79mtQltLW17ezsrly5Eh8f/+PHj+Li4vLyctTAzMvLq/o/cnJywsPDly1bJi8vTxAEi8UaO3asYHe3tLT8448/qMPnz5+7u7s3V1HD7gQAAAAAAIDYiYuLs7CwoJarJ02a9PTpUxCL+Ni6devp06cHDx786NEjLDlgcnKymZkZ+VlOTu7ff//FTiB95fv27RsbGytwA8rLy01MTHJzc8nD1q1bJycnN0s/GdidAAAAAAAAEC8cDmfNmjWULiEnJ+fi4gJiESvBwcHDhw9/8eIFPdH4s2fPqM8DBw6kn7Bp0yYFBYXx48cL0wB1dfX9+/dThyUlJZs2bWqWogZ1AgAAAAAAQLxcvnw5IiKCOly6dGnv3r1BLOIjJyfn48eP586dU1VV5appUJ8nTpxIP0FZWblz587jxo0TshkrVqzo2bMndXj37t3AwMDmJ20wdgIAAAAAABAjeXl5PXr0oKI5qampffnyBYLDipXr168/e/bMy8uL/lVNTY2Ojk5ZWRl5GBYWZm1tTT/NyMjo48ePXLURvnj48KGNjQ11aGxsnJiYKHy1UgXsTgAAAAAAAIiRbdu2UboEeQi6hLiZPn26m5sb168iIiIoXUJTU3Pw4MFcT3N1dRXJpH/GjBloqNm0tDRnZ+dmJm3YnQAAAAAAABAXr169GjVqFHWop6f35cuXVq1agWSaCicnp8OHD5OfZ82a5evrK+47vn//ftCgQdShoqJiXFwcagQl68DuBAAAAAAAgFiora1ds2YNWnLgwAHQJZoW1HFi0qRJErijlZXVnDlz0Ldi9erVzUmksDsBAAAAAAAgFo4ePbpr1y7q0MzMLD4+nsxpIG0UFRV9//499z/q6+v1/sPMzExZWbk5/SKFhYVt27atr68nD79//96xY0cJ3Dc1NbVnz561tbVUiaen55IlS0CdAAAAAAAAALiTkZFhZmZWWVlJldy8edPe3l56WlhXV/fmzZugoKDAwMDo6Gj6nFBdXX3ChAmzZ8+2s7MTJqNzdHT0iRMnmJ9vaWm5ZcsW6jAiIqIhRwgUc3NzJycn3ufcvXuX2igwMzNLSkqSmLRXrVp1+fJl6lBXVzclJUVLS6sZvOoK8G8HAAAAAAAQOYcPH0Z1iQ4dOsyePVtK2lZTU3Pt2jUXF5evX79SmoPJf+jr66ekpERHR+fl5ZWXl9//j6NHj548eXLChAmC3c7AwGDixIlZWVnh4eEhISFU/g0MRUXFGTNmWFhYWFlZoeWdOnUaM2ZMQkJCSEhIcnIy/UJdXd3JkycPGDCg0ZagGSe4hogVHxs2bEDVifz8fDc3t3379jWHd50DAAAAAAAAiJSsrCwlJSV0xnXgwAFpaBibzT579qyBgQHVsN69e3t5edXW1mJnpqamYnncnJychG9AdHS0hoYGfUY6duzY7Oxs3tdWVlbSU86tWrWqpKSE4d27dOlCXRgUFCRh4Y8ePRptORmvthm87eCKDQAAAAAAIGJcXV1ramqoQyUlJWnwvs3IyBg1atTGjRuzs7MJgtDS0vLz84uPj1+4cKGCAm6xYmxsHBgYeOjQIcrZw9nZedmyZUK2oX///mioK5KBAwc+fPiw0fi5nz9/zs/Ppw719PSCgoIuXrzIVT+h8+XLl4yMDPKzqqoqGr9VMqxfvx49LCgouHTpUjN420GdAAAAAAAAECVFRUUXL15ES+zs7Nq1a9e0rbpx40bfvn3DwsLIw27dur19+3bmzJm8polyck5OTqtWraJKPD09z5w5I2RLqqqqsBInJyc1NTXeV/369WvhwoXU4ZgxY2JjY/myv0ItnUaOHKmioiLhn2DGjBmdO3dGS06dOoWqnaBOAAAAAAAAAIS7uzuVKI1k48aNTduk7du3L1y4sKSkhDy0trZ++/Zt9+7dmVy7d+9edK6/Y8eOmJgYYRqTmJiIHhoYGEyfPp33JRUVFTNmzEhISCCVnL179z579qx9+/Z83RcNESthxwkSeXn5tWvXoiXZ2dnXr18HdQIAAAAAAAD4v4nv2bNn0ZKhQ4cOHDiwCZu0b9++kydPUoedOnV68OCBtrY2w8v19PRQK53a2to9e/YI3JikpKScnBy0ZOrUqbyD5+bm5o4aNSo0NJQgiHbt2gUGBh44cIDfSFNsNvuff/6hDiWTcYLOihUrsGTbLi4uVOBaUCcAAAAAAABaOleuXEHt+8mQPk3YnhMnThw8eJA6VFFRuXfvHt2hmTfjxo1DDwMCAuLj4wVrT1BQEFbC22ApMTFx0KBBkZGRBEGMGDEiJiYGcxBnyLt376jNmc6dO5uamjbJz6GtrT1//ny0JCUlxc/PD9QJAAAAAAAAgGCz2adOnUJLDAwMZs2a1VTtOX/+/I4dO9ASDw8PCwsLfusxMTHBSv7++2/BmhQYGIgeysvLjx07tqGTb926NXTo0O/fv7NYLEdHxxcvXjTqrt0QTRgiFgNzyCbTHYI6AQAAAAAAAPy/6e+3b9/QkjVr1tCDJkkGb2/vdevWoSUTJkxYunSpAFV17NgRi3sbEhIiQD0VFRWkzRKFpaVlmzZt6GdWVlauWrXK3t6+tLRUR0cnICDgyJEjwiQUR9WJprJ0IunTp8/IkSPRkujoaNSvA9QJAAAAAACAlgiHwzl+/DhawmKxhI+sKhjZ2dlr1qxBE13Ly8ujHhR8IS8vj6aqIG2QMJsuJrx69QoL68TV0ik5OXnQoEFk0rehQ4fGxMRMnjxZGGkUFxe/e/eO/KygoMBjP0Qy0KMGy/QGBagTAAAAAAAAIuDRo0dJSUloyZAhQzp06NAkjXFwcKBcBUiWLVvWu3dvgSvEauNwOFQOB+YwcZy4cOHCwIEDySBO27Zte/XqVceOHYWUxj///FNXV0f9KK1bt27aV2XatGnKyspoycuXL9++fQvqBAAAAAAAQMvFw8MDK7G1tW2Sljz4D7SkVatWqEM2v9TU1BQUFGCFeXl5QqoTrVu3HjRoEHWYn59vY2OzZs2ayspKLS0tf3//EydOiMRUDDUlalpLJ+rnoDuU//XXX6BOAAAAAAAAtFDy8vKeP3+OFfJOEicmSkpKHBwcsEIHBwc9PT2B68zNzaUXFhYW8lXJt2/fPn36hJaMGTOG0haCgoJ69+798OFDgiCsrKyio6NnzJghEoHU1dU9evSIOkQVmCaErmreu3dPRlPagToBAAAAAAAgLHfv3mWz2WhJv379jI2NJd8SJyen7OxstITFYqGZrUWlTmhoaPBVSUOWTtXV1Zs3b548eTJ5lw0bNoSFhXXp0kVUAvH19c3KyhK42WJixowZmGf5r1+/nj59CuoEAAAAAABAS+TWrVtYSZNYOpWUlNBtZiZMmGBkZCRMtZmZmfTCtm3bCq9OJCUlWVlZnTlzhsPhaGpq+vr6nj17VlFRUSTSqK2tvXHjxooVK9DCv//+WxrSxuno6GDxnbi+RaBOAAAAAAAANH++f/8eHh4uDerE3bt3KysrsUIhtyYIgnjz5o2Q6gSbzcaMwYyMjJ4+fWphYUFmxDM1NY2KihJVjo64uDgrKysdHZ2FCxeWlZWhX50/f15HR8fKymrNmjVN+9rQ35BHjx5hrZUJWGgEMQAAAAAAAIBfTpw4gWWL6969++fPnyXfkhEjRmCJHfT09DIzM4V0aB44cGB0dDRaoqysXFxcjIUn4kFYWNjw4cPREnl5eSraEkEQ+vr6KSkp6urqLee1yc7O7tixIzYVv3HjBpY2W/qB3QkAAAAAAAChkBJLp7S0NEyXIAhi3rx5QuoSxcXFsbGxWKGVlRVzXYKrpROWafvHjx+ynhyaXwwMDKysrBp9l0CdAAAAAAAAaM58+vQpJiZGGtQJLy8veuG4ceOErDYsLIzubDBs2DC+KsHUCUVFxcePH2Ne0adOncJyijd76O9JcHAwPSYvqBMAAAAAAADNFvpycseOHS0sLCTfkidPnmAlioqKI0aMELLaly9f0gtHjx7NvIb8/PyoqCi0ZMiQISYmJrt370YLq6qqMJuxFqhO1NbW+vr6gjoBAAAAAADQUrh9+zZW8ttvv7FYLMm3JD09HSsZNGhQq1athKmTw+HQZ7cGBgZjxoxhXsmzZ8+w/Q0yROymTZuwWLo+Pj5hYWEt5+UxMTGhpyqXOXsnUCcAAAAAAAAEJCoqKiUlBSsU3r5IAMrKyvLz80XekufPn2dkZGCFS5YswXIm8IbuODFx4kTSn/vEiRPYV5s2bZKGQK4Sg54e+/Xr12iiDFAnAAAAAAAAmi30hWQWi4WFMJIM9Em/AB4OdK5evUp/wGXLljGvgcPhYOqEjo7OgAEDyM+2trajRo3CNLRr1661nFdoyJAhdInduXMH1AkAAAAAAIDmj5+fH1bSp08fLS0tKVEnOnbsKEydRUVF9+/fxwrnzp3LV7bv+Ph4LKn2uHHj5OT+bwrq6uqKHhIEsWvXrtLS0hbyCg0ePJhe+OjRI1AnAAAAAAAAmjmZmZl0dwXhXZ8Fbgy90MDAQJg6vb29q6qq0BIlJaXDhw/zVQnXZNjoYb9+/bDtjtzc3CNHjrSQt8jwP7DCd+/e1dTUgDoBAAAAAADQnKEneSAIYuTIkU3SGG1tbaxEU1NTmKxwtbW1Z86cwQodHByMjIz4qicwMJC3OkEQhLOzc+vWrdESV1dXuqrWXKFvUFRVVb1//x7UCQAAAAAAgObM69ev6YVNtTtBn+Xr6+sLU+H58+e/fv2KlnTs2HHPnj18VVJWVhYeHo6W9OzZk74Y3759eyxobHV19fbt21vIi0R3n2hIWQV1AgAAAAAAoDmrE926dWvbtq2UqBNKSkoC11ZcXHzw4EG0RF5e3tvbu02bNnzV8/LlS8xoh4zpRGfjxo1du3ZFS+7du/fq1auW8CJxdZ/gqqyCOgEAzRkOhwNCAOA9AYCWQ35+/qdPn7BCKmCR5Gnbti2WYkIYY6EjR45guZn37NkjQJwoJpZOJC05aOyAAQPout+bN2/q6upAnQCAlkJOTs6xY8dADkCjbN68uUXFUweAZkxoaCh9gWDgwIFN2CRsg6KkpKSwsFCAetLS0tzc3NCS6dOnOzk5CVAV5oetrKzMw7dk5syZWLLt2NjYK1euNPt3SUVFpV+/flhhSUlJXFwcqBMA0CIoKCgYP358Tk4OiAJolIcPH65cuRL2KACgGcDVFqUJdycIgrCzs6MrBvxWUlFRYWtriwZ0GjVqlI+PD19566i7Y94X1tbWampqPC45ffo0FjTWycmppKSk2b9OXN0nZMXeCdQJABCKkpKSiRMn5ufnYz5kAMAVNzc3T0/PjRs3gigAQNbh6inbtOqEg4ODpqYmWhIWFsZvJUuXLkUXxS0tLR8+fKiioiJAexoNEUunX79+ixcvRkt+/vzp7Ozc7F8nru4TsuKNDeoEAAhOfX39nDlzYmJibty40b59exAI0CjTpk3bvHnzuXPnTp8+DdIAANmltLQ0NjYWKzQyMmqSBHYUmpqaDg4OaMmpU6f4Sl9w5MgRHx8f6nDq1KkhISEaGhqCtYfuODF+/PhGr3J2dsZ2MM6ePZuamtq83yiZDu4E6gQACI6zs3NQUNCmTZvGjRsH0gAYcuzYsd69e//xxx8CrBoCACAlhIeH091km3ZrgmTTpk1oromsrKxr164xubCurm7Hjh3oTvv27dsfPnyIpYNgTkVFRUhICFqirq5O9xCgo6+vv2jRIrSkpqZmw4YNzfuN6ty5s56eHlaYl5eXnJwM6gQANFtCQkIOHDigr6+/f/9+kAbAHEVFRXd3dzabbWdn9/PnTxAIAMgiXJeNe/Xq1eQN09XV9fb2Rm2TDhw48OXLF95XFRUVTZ06lQqspKure+vWLRcXF8yNgS8CAgIqKirQkrZt2zKscMaMGVjJkydPbt++3bxfKq66lkxsUIA6AQCCkJOTY29vX19ff/LkSYF3gYEWy/DhwxcuXPjjx4958+aBWzYAyCJcfWS7d+8uDW2bMWNGcHAwlSAiOzt78ODBDSVwKCsrc3Nz69u3L+nnwGKxli9f/vnz57lz5wrThqqqKnokqMzMzNzcXCaX0w3JCIJYvXp1REREM36puKYblwlvbBaMZAAgAHZ2dj4+PoMHD27eXRsgVo3U2Ni4qqrqzp07c+bMAYEAgAzBZrNbtWpVXV2NlX/48KFpA8WiJCQkTJo06cePH+ShkpLSggULhv1H27ZtU1NTv379GhkZeeXKlV+/fpHnjBw58vDhw9bW1gLftL6+PjU1NSIi4ujRo/SkHARBmJqabtmyZcSIEV26dFFWVsa+/fXrV0xMjL+//5UrV8rKyuiXq6mpLV68ePbs2T169OjQoQOLxWpO79WJEyd27NiBFXbr1i0lJQXUCQBobjx79oyMTeHv70/fkAUAhqxbt87Dw6Nr167JycmKioogEACQFb5+/dqtWzd6eUlJiVTtV5eUlNy5c8fT05P3ypeGhsb06dPXrl0rjCJB4uDg4O7uzuRMOTm58vJy1Chr6dKlf//9N/N7LVq0iKFbiKzg6+s7e/ZsrFBBQaGyslJBQQHUCQBoPlRXV/fu3fvLly+9evWKj49vZksjgCTJyMjo1q0bm80+d+4cFowFAABpJjAwcPLkyVhhhw4dqK0AaePz589nz549f/48QRCqqqqGhoYdOnQwNDTs06fPgAEDhg8fLlgcWEC0REVFWVhYcFVfu3btKs0tV4AfDwD44sSJE6RP244dO0CXAIShS5cudnZ2N2/ePHjw4OLFi8EJp0Xx69cvDofTtEFFAYHBUrORSInjBFfU1dXJUHKdO3cODQ3t2LEj/IhSCFffCYIgvnz5IuXqhCRcsT98+LB9+/aoqCh4UQBZp6Sk5OTJk2Rsb/qOpJgoKiqKi4sLCgq6du2ap6fn06dPY2Ji6Da7gCyyYsUKMhSgi4sLSKPZw+FwHjx4MG7cuLZt22ppaWn/x8KFCz9+/AjCkS24Bkrq0aOHdLY2Pj5+8ODBCQkJenp6ISEhQuoSlZWVa9asSUhIgNdA5Ghra3MNy8tVfZUqxL47UV9fv2rVqpiYmFatWkmPfxIgDFeuXAkJCenWrdu2bdsEDkcto7i7uxcXFxME8fvvv4t1a7iuru7NmzdBQUGBgYHR0dF0o0R1dfUJEybMnj3bzs5OmEB+QhIdHU0FFmSCpaXlli1bqMOIiAg3N7dGrzI3N6dHCBESX1/fe/fuMTmzf//+dN84bGKxd+9eHif06dPH0dGR61cjR47s1KnT9+/fT58+vXbt2g4dOkAP01xJS0uztbWNi4szNDTcv3+/hYVFdHT02rVrb9y4ce/evWfPnglvtg5IDK7TO2NjYylsamBg4Jw5c0pLS3V0dEJCQkxMTISp7devX9OmTfvx4wdESBcTXbp0iY+PF1idaKqhTezqxNWrV2NiYgiCSEpKgrekGbBv376DBw+Sn+/duxcVFdVyDC4rKyvPnDlDfl6wYIGY7lJTU3Pt2jUXFxeq+1BXVzf5D319/ZSUlOjo6Ly8vPLy8vv/cfTo0ZMnT5Ku4ZLHwMBg4sSJWVlZ4eHhISEhbDab62mKioozZsywsLCwsrJCyzt16jRmzJiEhISQkBCumXp0dXUnT54sjsxQpqamPXr0CAgI+PDhA9cTVFVVp02b1qdPn0GDBvGuSldXd8SIETExMX5+fgUFBehXcnJykyZN4hGKnsVi2dvbHzt2rKKiYv/+/RcvXoROplny5s2bGTNmFBQU9O3b9/nz5zo6OgRByMvLU33L6tWrYblX1tUJAwMDaWvnmTNntm3bRqbb09PT27t3r46OjqGhoYGBgYmJycCBA1u1asW8tqioqMWLF+fk5Lx586Z9+/bwGogDIyMjYdSJJhvaOOKkuLiYeuHMzc05gIxTX1+PdT3nz59vOY9/7tw58qkNDQ3r6+tFXj+bzT579iw6IPXu3dvLy6u2thY7MzU1dfz48egP4eTk1OTyiY6O5mr9P3bs2OzsbN7XVlZW6urqYheuWrWqpKRErG2urq6eOXMmvc2WlpZFRUX81lZUVNSzZ0+0Hnd390avoiaR8vLyycnJ0M80P9LS0sjXW11dPT09nSqnlmZIysvLQVYyAZvN5hqK7cWLF9LTyMrKyuXLl/OeL8rJyZmbmy9duvTy5cufP3/mUduPHz82bNggLy+voqISFhYG74D42LRpE/2X6tGjh5QPbeJVJ7Zv344uT9JnRYBsgamnZKTLlvP4VFjA5cuXi7zy9PT0YcOGUYLV0tLy8/PjcX5dXd2hQ4eo1U0yxF6Ti2j69OnYGzJw4EAmkyQsY5Genl5QUJBk2pyTk0OfGdy7d0+w2tDxe+PGjQyvMjQ0JC9ZuXIl9DPNjLq6uv79+5O/7+HDh9Gv0HiaPXv2BFnJCqmpqVxn57xn5JLky5cvXPMr80ZPT2/WrFk7d+68ePHi06dP/fz8Ll68uG/fvkGDBpFxR9q0afPkyRN4AcTK2bNn6T+NkpISm82W5qGNEOvbrKSkhD7Gx48f4UWRdbS1tfldfG0evHv3jnrqmzdvirby69evo14o3bp1YzgsrVmzBv05XF1dm1ZK2J4JQRD3799nsvLRu3dv6pIxY8bk5uZKstkjRozAmh0eHi5YVZSHmLGxcWVlJcOrFi1aRKmR1dXV0M80J7y8vMgfV0NDg4zmRFFVVbV8+XJVVdWePXvGxsaCrGQFMns0nbKyMil55cQRJq5///6pqanw64ubhw8fcpU/urEphUObGNUJGxsb7DHu3r0LL4qs8+eff6ILz1VVVS3kwdevX089uGgnu9u2bUP/JtbW1gUFBcyXH9TU1NA9QNJvu6nAPIkNDAwaXVApLy8fPnw4tfO+d+/euro6CTd7z549AmhBdNAUsI8fP2Z+IZqJifeuFCBb1NbWUuEd16xZw/WcmpoaEJRswTVNm6amZpM3rKioyM7OjiCIVq1azZ8/39nZ2dvbOyIiIjw83NfX99ChQ6NHjxbA3VFbW9vFxYX5+gggDA35UD179kyahzZxqRPPnj2jy+LAgQMiv9G///5bWlratDXIKAI/+MuXL/ft23flypWWo0vU1ta2a9dOHF5AWNiETp065eXl8VXDH3/8gdYwderUppJSYmIi3fmhUXXI0tKSPLldu3bBwcFN0vIHDx5gLXdxcRGgnlWrVpGX29jY8HVhVlYWdetZs2bBgNpsQH3rX7582eTjHSASNm/eTJ/hmJmZNXnDwsLCevXq9eeffxYXFzd0TmVl5fPnz/ft2zdq1ChVVVXeioS5ufmhQ4d41AaInNLSUq6/hYeHhzQPbWJRJ9hsNtdIJnPmzBHtjT5//qyjo3P27NkmrEFGabEPLhhPnz5lPkVmDpZqQEVFJTIyUnjVPS4urkmkdOrUKXrEOt5rMJ06dSLPHDFiRKPu2uIjIyMDa/mKFSv4rSQtLY00VFVTU8vIyOD3ckoUysrKmEkMIKNUVVVRXjHt27cXctsNOm3pYdq0afQZzvjx42XuQaqrq8PCwtzc3BwdHZcuXTp58uSxY8fOnz9/27Zt7u7uaWlp8Fs3CfTAJARBbNmyRZqHNrGoEx4eHqSmPnbsWEzNFeFdiouLTU1NCYKIiYlpqhpklBb74AKzbt066jU+d+6cCP8mWFRlAepJT0/H6tm8eXOTSAlznJCXl+cRQcLb25u07mWxWI6Ojvw6mYkczNR45MiR/NawePFi8tojR44I0ICpU6cKuR8NSBu3bt2iflN7e3votJsN5G+BMXfuXJAMIBK4plefMWOGNA9tolcnCgsLSb3q6dOnW7duxTzTRRXcqaqqasqUKaSBhGAhO4WvQUZpsQ8uDOhum/AWCxwO5+bNm2SgDIoJEyYIVhWbzcZiHvTu3VvyIiovL8dMcgcPHsz1zIqKipUrV5Ln6OjoSEmckMGDB6ON79ChA1+Xf/r0iYyy1aNHD8F8qVGjtU2bNsGfrhlga2tL/aZ//fUXdNrNBmVlZfpsb/Xq1SAZQCRQseBQ+vXrJ81Dm+iT6R48eDA/P3/Sf2AmTzU1NVzz0vNLWVnZ1KlTnzx5QhDEuHHjsGmZZGqQUVrsgwtDfn4+moSRR0oyhmRnZ5NOmehC/smTJwWrTV5eHsudlJiYmJ+fL2EpvXr1qqqqClOQ6KclJycPGjTo8uXLBEEMHTo0JiZm8uTJ0vArm5ubo4c5OTllZWXML3d0dCQTRbm7u2PaHUPQ2FavXr2C/10z6GxRI8mRI0dCp908qKmpqa6uppe3adMGhAOIBHV1dXphQz4VUjK0iVid+PTpk7u7u7y8PGlFjT0GGStWyFt8//599OjRz58/5zFlEXcNMkqLfXAhCQ0Npab+HTp0INPZCoODg0NJSQlasmzZMnQ2yS9YbVwtJsUNPXIi/QW7cOHCwIEDybAV27Zte/XqVceOHaXkVzYzM8NKmK99hISE3L9/n7R2wCw8mYOqqXFxcb9+/YK/nkzz+PHjyspK8rO2traJiQl02s1GUeRarqmpCcIBRAIasLHRF09KhjYRqxNbt26tra393//+Rz6AmZkZtpSCrvIKgJeXV58+fdDM4fQ49+KuoSFIQy90yVkwqBGIycSxqR68vr6e6/KMYLDZbNFOnkQrw5cvX1Kfe/ToIWTbHvwHWtKqVSssOS5f1NTU0NML5uXlNa060bp1azSBf35+vo2NzZo1ayorK7W0tPz9/U+cOKGgoCA93bfAfS6bzSaTmGpoaNCd0ZmDGsvW19eHhobCmMqDoqKimpoa+h8/OztbSnqku3fvUp+pkO1NO1rV1dXRhYb2JAx79fr6eoFHIt5UVFQ01HtXVlbyaDzDypk0u/Y/BFAnYHcCEBVcdyckrE7wO7SJcjh/+vTpkydPNDU1Dxw4QEnEyMgoLS2NOkew3YnKyso7d+64ubnFxMSg5bq6umFhYdRhr1696IITVQ1c+fXr1+3btx89epSamvrt27eqqiplZWVTU9Nt27bZ29vLyckx+cE+f/4cFxcXGxsbExMTHR29c+dONJs42btdu3YtODg4NDQ0NzdXV1d3xYoV+/btYxI9WlQPXlhYmJSUlJiYGB8fHxcXFx8f7+7uTjnoCEB0dLSnp2d0dPT3799zcnLq6uratWtnbm7ev3//bdu2YekLmlaG6HDOV8O4ajIODg5YoYODg56ensB15ubmcv29JNn3ffv2DY1LTaaio7SFoKCgJUuWkO20srK6c+dOly5dpK377tmzJ1aSkpLC5EIPDw9yleTAgQP6+voCN0BVVVVTU7O4uJg8jI2NpacYb+EUFhYePXr09evXKSkp5HTf2Nh45MiRo0ePtrKyqqio+OOPP0xNTd3c3CTfI3348AEd6bBwcIqKij4+Puj52tra48aNE/doVVVVlZCQEBMTQ/aN8fHxQUFBaPZ9slO6fPnys2fPQkNDq6ur+/fv7+7ubmVlxVWL8PX1PX36dGxsbHV1dZcuXU6fPj1z5kwhf9by8nJPT8+AgIDk5OTv378TBGFoaLhq1aotW7aoqakdOHDA398/MzMzPz///Pnzq1ev5rd7DAgICA4Ofvv2LVn57Nmzvb29ua5lhIaGOjk5RUZGqqmpLVmypCEDVFAnAHHDdXeioqKCw+Hwa+4ouaFNVI4jtbW1ZKyDkydPouUzZsxAb9erVy9+a37y5ImWlhaTh798+bKYaqATHx8/Z84c0h9rzJgxFy5cePTo0e7duykVwtHRsaFri4qKzp8/v3LlSgsLC/p0FkuHHBUVxXXMcHBwELfoUlNTbWxsLC0tuZr3ZGVlCeY37O7uTib/V1dX37Jli5+fX2xs7O3bt0lfQ3Jh+9y5c7yDKkpMhhwOp23bttQlAkRqQ0Fz4ZGwWCwhg/Gh6bop/P39Jek3hgbXp3oi0oV006ZNVPe3YcMGqc3YVV9fjy0ILVq0qNGr8vLyyDlEnz59hI8zgYaLWbhwIfgjoly5coXqzWbOnPn333/fv3//wIEDmC+Tm5tbk/RI9OyzvJk9e7aYRquMjIzTp08vXLiwV69e2KRZU1MTe0sDAgLoEwUdHR16Js20tDT6HguLxYqKihImcIujoyP5vG3btt21a5ePj8/Lly/PnDnTvn37iRMn1tTUtG/fnrod8/jLFRUV58+fHzVqFNdFvRs3btAv2bdvH+lySvH+/XvmXS65ogr/U0AkUHke6Kqs1A5tItudcHd3//TpU9euXbEJk7m5OZowPCUlhc1m82XkUFBQsGTJEmoKi+4gz58/n0ouRoauF1MNmIJ44MCB06dPs9lsDQ0NHx8fSmWaNm3a8+fP3759SxDEsWPH1q5dSwUdR/n586e/v39hYeHHjx8x71VTU1PU5sHX13fx4sUVFRUKCgrkYEZ9dfHiRVdXV96SFPLBtbS07OzsysrKUlJSLl++TK2bkj8r5v7LhIyMjPnz579584Y0xSPHDPKrvn37zpkzZ/PmzWfPni0pKVm/fn18fPylS5caqkpiMiwuLkYNh4TZnSgpKfnrr7+wwgkTJhgZGQnz18vMzKQXoiqQBODqOJGUlGRvbx8fH0/OY65cuTJr1iypXQ1isVg9e/ZEd6KYLOHs3r37169fLBbL3d1deNutDh06UJs8Iola0Wx4/fr1ypUr6+vryaUyKvnjb7/9tn379hUrVnh7e5Ml3bp1a5Ieafjw4ehs+8mTJ58/f6YO169fj70eDW1NCD9affnyJSwsrKCgICcnh81mN7RnSBCEq6vrtm3bSKlibXj8+PGiRYvQibK9vT3dAIzD4Vy4cIGHWHgQGhq6YMGC79+/k7nwt2/f3qpVK/KrkSNH2trampmZubi4/Pvvv9Qv27lzZ4aVl5aWhoaGzp8//8yZM23atHn48OGGDRtQTWz+/Pno+Xv37j106BBWSU5ODuxOANKzO0G+e1ztoKRiaBOJIpWfn08uMPj5+dEDYmJ3TE5OFvhGZ86coepRUVERIOW7kDVERUVRdhrGxsaJiYk8lp8fPHjAu7bq6mpsp3jHjh3Ut97e3nJycsbGxrdv3y4qKvr48SMWP5ivVFlCPrizszN6awEyG3h7e1Oeajt37uR6TllZGZq9JTAwsNFqxS3DyMjIRpe1GELXJQiCuHfvnpD/vi1btnCdUkhsHaW2thbzQTQyMjp37hy1a2Rqavr161fpXxBauHAhtkbL+/zo6Ghy7XPJkiUiaYC9vT3zuwtAampqd4mwcuVKETa7qqqKyvGnqqpKXyqrqamhumXmb76YeiQSVLXo2rVrk4x3dXV1WIzICxcuUN8ePnyY9Njx9PTMz88nDymcnZ2pMwMDA5WUlNTU1I4fP56ZmYkacREEMXr0aAGWS6mtABUVlYaSXa5ZswbdeSaj4QnMxIkTUduz8vJy6itPT09yv+jhw4dbtmwhlRZra+uGtlL9/f25zqYgJQggKnbv3s31HRNsJJXM0CYadWLt2rUNZceIi4vDxME7US5vUGPisWPHSriG4OBgavnE0NDw58+f9HPmzp1L1f/o0aNG68RymYWHh1MdloKCwuTJk0n3bnrjFRUV+bIbEVJ0aFB8fnd1Kysr0YWuhkZuEtL1hxIyaSzYhDKkVj1Jnj9/LvDbO3z4cOy/oKenJ7yFzIABA7BqlZWVq6qqJNbx0Z2GMZsBfX19AbZoJc/Ro0fpy7Q8zidt0LW0tLh2BcJrhuh7KxKSk5Mls7Q2ZcoUETb7/fv3VM19+vTheg45I1RQUGDyhxJ3j1RVVUUmkSVZsGBBU413aOMJgkhPTyfL//zzT4Igpk+fXlxcTJZgK/FXrlyh/t1qampmZmZJSUlUta1bt6bOnDp1Kr+6BGXIIS8v/+zZs4bOPHv2LNok+mIlX3h5eaG1UfcNDw9XVFRctWoV2sJv377xqIq+SEpCX14EAME4cuQI13csNjZWaoc2EUR2SkxMvHjxopycnKurK/3bHj16YBslAseKZbPZaDh2AQIyClPDrVu3pk6dSu5yKigo3Llzh25PUltb+88//6BWN41Wi5o0tGvXjlxMSkhIsLe3nz59+uPHj1E7WnTnulu3buiIJW7RodaiysrKfMVQX7ZsGdWVjx07FlsGw0DDKWZlZdHVUQnLEHWvFCYUYFpaGn3aPW/ePCEtZIqLi2NjY7FCKysrrlmWJGbphMXE/PHjB707k0L4ioBx69Yt0i/28OHDojItw94uer7zlgn6hjcUsszOzk5BQaFLly5M/lDi7pFiY2PR0EBDhw5tkvGO9CKjPnfv3p3cwwkJCdm4ceO6dev8/f0pxQCzmSR3VzIzM3/77TcTE5PQ0FDq35GTk4PGR6KHg+dtDbFu3TrKOOrEiRMNGX1hNkXy8vKjR48W5i3CBEhauJWVlS1cuHDEiBHnz59HjUOo3TC+jJ2Yj8gAILCxk9QObSJQJzZv3lxXV7d48WKuafyUlZWxuYXAsWIjIyPRXoxHNyTyGl6/fr148WJqhDh8+DDXEeLGjRuUoefQoUOZBBoPCQmhPk+bNk1OTq6goMDGxsbc3PzmzZuYGxk6NvDViQspuqqqKtIhhGTYsGGqqqoMr3V2dr516xY117916xbvgFdYP05a3jehDDFzYSaRoLiCrY0J/A5jhIWF0U2fscgtElYnFBUVHz9+jBmVnTp16tu3b1LegzOPgFFeXr5jxw6CICwsLP73v/+JqgHY2yWmWJwyBxr7NScnB7M/JFFVVe3VqxcTxwkJ9EjodgpBEEOGDGmS8Y4giKioKOozae2Tnp5uZ2c3a9asc+fOoSFiUNNqDQ2NXr16sdnsuXPntmvXLiQkRFtbG5uIC9YwR0dHauJOuqbwOBndMLGwsBDSM0FfXx/1qSN/o02bNpWUlHh5eTEJwwjqBCAxGnKQEEydkMzQJqzv4MOHD0NCQtTV1Xks8PTq1QsNIinw7gQ6a9TS0hIgkrdgNWRlZc2ePZvSJaytrbEgpNQ8FXX2opwFefDjxw8ypReJjY0NuXJWX1//6NEjbMpeUFCArpSjQf3FLbqwsDDU3Rk1QuXNvXv3UDkwUXaxoOONDt7iliH27xVYnSCT2mJjD7/RYOigOTEohFzG44v8/Hx0ykJOnkxMTHbv3r1z505UI92xY8edO3ekuQc3NjZWVlZGQ+83tIRz7NixrKwsOTk5Dw8PviYivMH2lAQbOXjQvXt3NKCC+BBtRhFjY2P08MiRI2RmJYxBgwY1mo9cAj0SqQlQn1u1aiVYhkrhx7vv37///PkT7bfZbPa8efP69u17/fp1LNxkREQE9XnMmDHy8vI7duxISUmJiorCRIT2OSoqKnQbTh59oIuLC6WxYLZMvNUJ4VdeyDU+ahYVGRnp7+9/5coVX19ffuM7l5eXgzoBiJWGdicaevekYWgTqtOvqanZunUraXvKI+JNr169fH190Wk3v8GdSKjMoOSESYBRXIAa6urqZs2ahXbK+/fvp8f9raio+P3336nhf/fu3Uzm3M+ePUNX18aPH3/+/PmAgIDQ0FA0ggfXFS/mc3rhRRccHIweMszMGhsbS8YjIw/79eu3bNmyRq9CRU3OVptWhqJSJ+iGK4MGDaJccQSDw+Gg/ywSAwODMWPGSKzXe/bsGbY9Qr4emzZtunTpEqq8+fj4rF+/XsI7J3whLy/fo0cPdL7IdQknPT2dDEi/atUqS0tLETYAe7tErk7IycmhVu+ywoQJExQUFCgzxQcPHvj7+5MLByjHjx+Xhh4J62esrKwwVyKJjXeohqCkpDRq1KiDBw9mZ2fHxcXR9S40o8WkSZOeP3/u6ur69OlTutkPatA7fPhwhl1iTk4O6dZJHu7Zs6fRZDvR0dHUZ5Fkmx06dOjff/9N/ayLFi2aMmWKAOHmGvpjNqrNioqqqqp169bBnFvy2NjYYMkPxIRodyckNLQJ4yxy4sQJgiA6duzI2zuNPuP59OkTv/cqLy9H/6tkVHsJ1IAF1B8yZAjX6B/oG7Z69er6+np+A7nMmDEjOTlZVVX1wIEDXE92cnKiTtbX15ek6Mig7CTt27dn+HTYetI///zD5Ko9e/agV23fvr1pZfjbb7+h7cnLyxPgn1JaWkr/6+3fv19Iby1UlaLYvXu3JD3G6KkMIyMjya/u3buHfTVw4EDewfubHDs7O7TBAwYMoJ9DRhJr27atyF2lqbkOCfM0OM0ebAjX0dHhK66dJHskMroidb6Tk5MAzyt8p01GRkI3HCIiIpSUlF6+fEk/E3Oc+PjxY5cuXQ4fPkw/E0ua6eLiwrAxkydPpq7q2rVrdXU17/NTU1PRdaJGz2dCYmIipr2npqYKHHuGjsg7hIaQzB4jQGffvn2S+YmxNVwKd3d3qR3aBN+d+PnzJxmn2cjIiLefJT14c1JSUo8ePfi6XWhoKJpgX4CtTwFqKCsr27dvH1pCN2FKSUmZM2cO6Z+npKR05swZtBPnrcih08Fp06YtW7Zs4MCBDQUIQ09muD8gEtH9/PkT9T4cP348k6SMkZGR6Gb9wIEDR40aJcAqPu/0yRKQoUh2JzIyMuiFwq/TX716FSthsVhMFlxFBYfDwRwndHR0qEhTtra2o0aNQk0joqKirl27tnTpUqkdLTCXNfqOcEhICGlp4+LiwjDdGHPEbewkuxw8ePDx48fUPlhBQYGtrW14eDjz/6NkeiTSCQFNbiOYH7bw4x1BEKgn9+jRo1esWLFjxw6uUTTQDQcrKysPDw99fX1HR0f6mZh1JcNNg3/++QcNL7t58+ZGF/LRH8va2lokC/9mZmZo4vk5c+ZgdnRC7k6AsRMgnbsTEhraBFaeVq5cKczYwO/tUHeFTp06CdBgAWrYv38/9nug3/78+XPTpk1UD2JjY8NXsH90J1dOTs7R0VFdXb2hJbfCwkJ0x/zWrVsSEx0WFM/Ly4vJVba2tuhVf/31F8PbYaM170DvEpAhZhksQKoTDofz6NEj+r8AS93NL4WFhfS51Lx58yS5ZkwPKmVnZ4eeEBMTgxlp6OnplZSUSO0qOJo1jFoNob6tra0l3fetra0Z7tEJc/fjx4/DvgTF8uXLsZ9m8eLFzC+XTI+ERXhksViCrfMJP979/PkTXfextbW1sLBgs9lcT6YS55HRY1VUVD5+/Mj1TDRZb7t27Rj+C1Bn9DZt2jAJGz1t2jTqkiNHjojqLUINXCdNmiRYJQsWLOA6sZFYzwa7E81+dwILeEBx9OhRqR3aBNydiI2NvXLlCovFWrBgARPD0AcPHqARcgTwxkbXKgRbquG3Bg6Hc/nyZbSEXPetrKwMCQnx9PR8/PhxbW2toqKinZ2dg4MDv4vN6LJu+/bt3dzcDhw40FDWzxcvXtTV1VHzZr4kIKToMIsaJstRnz59Qn0l27RpM2/ePCb3ysrKQhfyW7duzTsirQRkiC0SVFdXC7BBwTVxtQBpxVG8vb2xdOBKSkq8Q16KHK7JsNFD0jwdzd+Xm5t75MgRqY0bSw+ol5KSQhl5e3h4JCUlycvLe3h4MNmj4xfUVY7HAlXL5NSpU8+fP0f7h2vXrllZWTVkedIkPRLmOGFqairYOp/w4x1pykUdBgQEvH79uqHB+sWLF2iHv3PnTnooGPo+xrhx45j8C548eYJ6caxYsaLRF/vbt29o7AoRxpYYOnQo1WtxDRHGhIZ83srLy7GIdmJCTU2Nik4GSJJevXpJ5kYNuVwL7G8piaFNMC2E7FKZL4Vic9DevXvzdbv8/Hz0CW/evMlvgwWoAXVNI9mxY8f48ePJ2aSysvL48eNPnz7977//CiZDrIvs169fQ0tH2JqQhYWFJEWHznobyiGFsXr1avTRFi5cyPBeWBfZ6OqjBGT4+++/o7cQ7Oe+ffs29i5pamoKs3RRU1NDD0O8ZcsWCS8Y04f5zMxMurE15v6rrKyclpYmnUvg1dXVWJQIahk7Ly+PDFW5adMmMd39ypUr6K3//vtv2JTAVuywX0dZWTklJUV6eiSsw1y2bJkAjyl8p83hcDCTwqVLlzLxUiAIQldXF80YzcPFwtPTk0lLSINsivj4+EYvQbOmtmrVSvhcnxRofgnB3Dg5HM62bdu4zqZkIvc/IBM8ePCA6zvG8E/XJEObICEO79279+rVK0VFRdJ3gglYQreUlBRqnZgJL168oBZaWCyWAAl9BKgB2xtSVFQMCAjQ0dHZs2fP06dPCwsLg4ODN2/eTA8fxFD1DA8PR0uOHTvGY59HYMcJIUX38eNHNOg7w1ujS2t8rS1hMue9gigZGWKLAdiGAEOMjIywEn5DE2KcP3/+69evaEnHjh0xn1FxU1ZWhsm/Z8+ehoaG2Gnt27fHXFmqq6u5hlqWBpSUlDA9jYqA4eTk9OvXL319/YMHD4rp7tjbJWTgL67USwR0XVyEDBkyBDNAra6uXr9+vZT0SGTcarTDFMxxQvjxjjTKoj6ThqAMhbNx48aGglSiWxMMd6rLy8vRlnTq1KnRsLmVlZXofqa1tbWo4g7n5OSg0ThIHxUB6mlod0WwIJ4AQKeiokK0uxOSGNr41T8qKyvJudGaNWuYX0Wm9Bd4VQDNpsHvzobANfTt2xdtsKurqwhVz4CAALTy/v378zgZc5p59eqVxER35swZbEbe6CV0z3uGS9G5ubmoK5upqSmPrQaJydDBwQG9UDCHByzWJEEQffv2Ffjl+fXrl46ODlqbvLx8aGiohJdP6A4hDa1tVFVVde3aFTuZa4QZaQAzsp85cybppUM6gfDltsQvp06dQm8dFBQk2vqTk5MlMxZOmTJFTCKqq6ujG//wfvkl1iPR1xSTkpIkM1phYE5NNjY2DF94DQ2NoqIiJjsemCchQ4ttJtMGLPyJqBwn6uvrJ0yYICcnh3aeZOBgfiFjWtJ58+YNLKsDIgGztKd4+vSp1A5tfO9OnDp1Kj09XU1NjUmaNgp6HCe+3Cck7zhB5gDidyWGOVgUMDINIZNldQ0NDb5yrAopOrSdqqqqTPxDsHjG+vr69LV5rly7do3KFUjqb7zdciQjQ2wxALNuZ0jbtm2xeuhpKJhz5MiRgoICtGTPnj2Sz+eALjry3vNRVlamD8CbNm2i5/OWBjCrcfJ93rhxY319/dixY+fOnSu+W0tgd0LWkZOTu3r1KrZ8fu3aNWnokTDHCS0trYY8EMQ93mH/TTS/KgabzUYTXCxYsIBH8ml0d4LhgIj10o2mxElPT8fyhwif65PEzc0tODh4165d6F9YsN2Jhv6YDa0oAwC/iNx3QhJDG1/KR2ZmJrnNt2vXLr4uxKbmBEEcOnSI4bVYkM2AgAB+dSYBaqBH4xIspE9DoNZf2travINqo6kPZsyYIdYHR6mtrUXf3YkTJzK5CkszYmVlxdCwD43Zx2R1UzIyxBaM3717J9gvTt/iLygoEKCe1NRUzBd8+vTpjS6aigNs51RZWbkhk2sSuoXJpUuXpHBZCAtlpqKiQpYoKSklJyeL9daYJQZfkeKa2e7E2rVrf/z40dC3mJ1tjx49pKFHwlJbCBY4SPjxjnJuJOncuTOPUC3YfDoiIqKhM9GUlMwbhmkdZBRd3h01mhJXRUVFJBknEhISyATebDYby82flZXFb23Xr1/n+tr7+/vDsjogEtAAcSgxMTFSO7TxZ5K4Y8eO8vJybW1t3ivBdAwNDdXU1FDdPSkpieG16NqJoqKiAGsVAtRAt04RYcLLzMzMT58+UYezZ8/mUTmbzUbXhAROhi2A6CIjI1G1iqG/AbZCw9C3xNXVlRquVFVVXV1dpUSG2KSZSU5crtjZ2SUkJGAWF9ra2nxVUlFRYWtri65hjxo1ysfHR7C0u8KQlpaGOW9YW1s3ZHJNcvr06YEDB6I7Ek5OTnZ2dtKWpxmLgFFVVUUavG3duhXzARM5eXl56B+20RQH/KKgoCCk0w5DMGM8fqmtrb1w4cKIESOw1EsU69atc3Z2prYKs7KymrxHItfmPnz4QB0K5jgh/HhXVFSEOjXNmzePR6gWNDhb586dBw8ezGRrQklJqdEIVyTYSiI9xza2HfT8+fOtW7dSHjKWlpbCj7zV1dXz589XV1f39vaWl5fHRBoaGsrvuiyPyE6wrA5I7e6E2Ic25poHFUtOgKwRWGZlMt4WwwtR77fhw4ejXy1btiw2NlYcNdTU1GApaZjEDyHt2hMSEnifg/qZkVvbPE5GQ+xhC5ZJSUm8I4cIKbpjx46ht0afKy0traHsjFiDmcQ2ycrKov4kLBbL19e30UskJkPMKo95uHr6i6GpqYnNV/itZM6cOWgNlpaW/EY6F1VSag8PD6wnOXbsWKNX0RPYNZpjWKxPwZWKigosVwY50+K99yIS0NzP3bt3b7Erc6Sm6uDgwOMcNC8B790JyfRIHA7n8+fPzDsl8Y133t7eaDOio6N5nGxpaUmdyTuuy/z586kzR44ciX7l6Oj44MEDrldhXss89lETEhLU1NRu3ryJprT6448/0H+9j4+PACLdvHkzQRCPHj2iSrp3707dYu3atdj5jx8/5j2IY8HTKSCNPSAqNm3axPUdQ5NFSNvQxtR3oqamZt26deT+CMOszxiYL+anT58YmqFjga7R6UtWVlafPn3EUYOioiLa4xAEgWaGboiEhAQLC4uzZ8/yPg01J9XR0eGdnBVdrOrSpQu1WF5UVGRjY8NbVRVSdOgwbGBgQEVcrqiomDlzJma+35AGjOZ2bYgNGzZQ2yCHDx+eNWtWo5dITIbGxsboP5Du1skQTU1NzKv71KlTTIRDceTIER8fH+pw6tSpISEhzMOch4SEWFlZqamp6evr//7771h0SH6hO04wsaV2dnbGdjDOnj3LV0tE+xRcUVVVpW8LnDlzhvfei0hA365u3bq12JU50uAHixuGgfbPVCL2JuyRMMcJeXl5KysrAZ5d+PHu8ePH6LDbv3//hs7Mz8+PioqiDlFtlg6aDxsNNuXl5eXm5tZQemlsAxYNe4X94lOmTJk7d669vT2azBt1cnNwcDh16hS/3msBAQFnzpzZvHkzqn+iGxSvX7/GZibz5s0rKSmB3Qmgme1OiH1o49eod+7cuYIpLljkb4IgwsLC+F3voS559OhR27Zts7OzxVcDtgHar18/3tGv//77bzU1tenTp9fU1PBeWEV72EYDmaOWOVQYitra2nHjxllaWvKwKxVedKidz8qVK6n2z549e+jQoTzWmTp27EhdOHjwYN53QTOaMQzTLjEZkqD/QPpSFnPy8vKwtTqGzgNsNhuLrLp9+3a+VuijoqKwYIutWrUSOBhUeXk51gGpq6szbA+9H2BuZy/ap2C48k1qbpJZkUL/OOLLbiH9kHuPLBaLx2L8ihUrKFm5uLgwF6w4eiQS1OO5X79+Ajy48J02m81GE+dt27aNx8moD0CrVq14jFyY3k5Fxnv9+rWysvLt27cbuhBbZeC6ifHt2zdjY+O+ffuWl5djZsaUY8POnTuNjIz4TfuTnJzcunVrCwsLrJP38vJC70LF6yspKTEzM0O3RBraSGloxQeW1QGRgG4GUrBYLIFTVktgaGOkTrx+/ZoaxQWOl0qXzubNmxu96tKlS+glFRUVHA4nPDxcTU2N4b6nwDW8f/8e2xhav3491w43NTWVnLCOHTu2qqqKd3vQFSyCIPz8/Hifj1ohOzs7k5Pp+fPnt2vX7vv37+ITXV1dHWq06uHhQRYuX75cQ0ODd6RFNC6whobGr1+/GjrTz8+PEvLu3bsZ/lUkJkMS1GmEjK0mMP7+/qgjtYGBQaNGdIWFhag6pKurK0BAN647iqampoK5OaKbJNSeD8Nr0WS3FAyfSLRPwQNUeVNRUUlNTZXA+FFfX48aWF64cKHFDqVUohIeEU4pK39dXV0eTtuS6ZGwJjEc3cQx3qFL++QOD4+TUcMq3gEqsEScpaWlHA7n7du3bdq04W2viNlsDBgwAPu33rp1S0tLq3///vn5+RwOB932VFdXJ89xdHRs06bNx48feRtyODg4GBkZrV69+uXLl3V1denp6SYmJm3atKH/fzGPDtKsrry8fMKECb179260P8G80il2794N82BAJKDBY+j/COkc2hpXJyIiItq2bUu1wM3NTYDbFBYWorlCqeUQsgfhwfLlyzHTiOPHj6uqqu7du5fhrYWpYePGjfR9cy8vr7i4uNLS0oyMjCdPnsybN09ZWZnFYm3evLlRXQKLSaKsrFxWVsb7fGVlZep8CwsLHx+f0aNHa2ho8AjBIRLRFRYWopcvXbo0NDR03LhxZDq/RidGaNzShhZ7Ll26RPoQKysrX79+nfnrJDEZkqBx0C0tLYX8y71+/RoNxaitrd1QBobS0tKzZ89SC6ssFmv58uWCxYOaPn16Q7ZD/FZVWVmJ2QGSph0MbTq5BqzQ1NRkErJdhE/BG09PT6ryAwcOSGb8+PHjR0OuSi15Ze7Jkyf0E9BQqg1Z7UuyRyLd7dCVgsePHwvw4MKPd2jC5vbt2/PYM2Sz2ege77lz53hUi/nReXh47N+/X1VVdeXKlby3JdPS0rBIdObm5h4eHo8ePTpw4AD5o1hYWBQWFpLnY+H2N27cOGTIEHV19UbT1GAeI3p6eqqqqnJycg3F6cfMxqytrXV1dTU1NZmkCqFHaoEdRUC0cDUebt++vTQPbbzUiYyMjGPHjqmqqmIDv4eHR0ZGBpPYlFVVVSkpKZ6enpTZPd0w/cGDB7m5uQ3VMGXKFPpVfGXQE6aG0tJSJhlJ9fX1meR3Ixk+fDh1IZNIgnRjWXV1dSYGHsKLjp77U05Oztvbm8m1X758MTc3J69SUFA4ePAg+sKkpKRQtmSzZs36+vUrX2+txGRIQgUhIHVgIXcbORxOfHw8GmBHSUlp2bJlV69eTUlJKSoq+vDhw+3bt7du3YpqHSNHjmRiHNgQWARS1G6SYQ11dXUpKSnXrl1rKASEqanppUuXPn36xFWpLioqevHixcaNGxsy/VRTU1uzZs2LFy+ys7MbkrDwT8GQd+/ekTWbmJgwWSMQCah/p66urvCvmexibW1NiUJBQQGzCXzy5An591FQUNi/fz/DOsXaI5GWeGib+Y2RIKpOG/17Ll++nMeZYWFh6F14r/37+/vTG7Zz504mTeKdrd/GxgbdLLp//z52QuvWrZl0fVzTfp8+fbqh8w8fPkx3mGS4MNFQfgnKJBgARNgHoq5Q0jy0NahOoOl7uKKoqMi76szMTB7x6TA8PT25VoJlZdbQ0Dhz5gxfVuPC13D58uWGMvtYW1t7enoy94svKSlB7Rn+/PPPRi+5e/cuanNlZWXVaOhuUT34li1b0BqMjIz4yshYW1vr6upKhQHV09ObNGnSvHnz+vbtKy8vr6mpOW/ePN578U0uQ2o9Ht3fYJhSlzfFxcWXLl1qNJuehoaGvb29MIoEJTSuYSuZG/uSkRiYICcnhyVpWbJkCV8OXQ2lqhX+KZj/OmTNwqQg5Rc0FKmtrW1LHkoNDAz09fXXrl27cuVKMgWqrq7utGnTZs2aRWViGjt2bGJiIl/ViqlHIjl//jz18w0bNkywSoTstLHc/w8fPuRx8q5du6gzO3TowLvm8vJydBDs3bt3YGAgw1bV19efPn0a26Mg3UtevHiBnVxWVoZ2tgMGDOAdmYoiNjYWvVBeXp63uWB5eTnqPt62bVs09FOjcI3NbW9vD/NgQCRgoVBJ+vfvL81DG+v/qRRSDJlnJygoqLy83NzcfOrUqfzGTRe+BoIgSkpK4uLiEhISEhMTFRQUOnfu3KlTp759+9KtPsRBVFRUUFCQhoaGmZnZmDFjGCppIhHdtWvX3r59q6Wl1bt375kzZ2JbVQxFl5iYmJKS8vnz53///bddu3YdOnQwMzMbNWoUFopXCmWIRgKh8j09fPiwIasbAfj8+fPZs2fJuYiqqqqhoWGHDh0MDQ379OkzYMCA4cOH04dhgYmNjU1PT9fR0WnVqtXkyZN//vwp2meRDJJ5io4dOw4aNAjLgCZWli9ffvXqVcrKhUcm4+ZNfX396NGjL168SC20p6amBgYGJiUlVVZWdu3atWfPnmZmZoIlnBZfj4T+fG5ubuvXr2+S8U58lJWV3blzp7q6um/fvkOGDKFHnORNenr6y5cvY2NjFRQUzM3Ne/XqZWFhwbWStLS0+/fvl5SU9OvXz8bGhvmNfv78+fjx44iICH19/cWLFzcUbIqisLDQ3d09MzPTzMzM3t6eYUISanENSzhI7rQ8ePAAohIBwtO9e3dsdYB03fTz85PaoU3a1QkAkAb27t1LOWwcPnwYXdgTkqysrClTpiQkJHTu3Dk0NBQNQSNWDA0Ns7OzMzIyOnfuLLu/i/ieorS0VEVFRZIa76BBg6gYA/Hx8fQ06oA0Y2ZmRiYdl5OTy8rKQlM7A82PCRMm0LNPDBky5M2bNyAcQHi0tbWLioqwwu3bt7u4uEjt0KYAPxsANIqtrS2lTjDJQMKQ+Pj4KVOmZGdn6+nphYSECKlLVFZWbtmyZe3atY3ORJOSkrKzs6dMmSLTuoRYn4J5Qg+RwGazExMTyc89evQAXUK2yMnJIXUJcicTdIlmj4mJCV2daCjiE0AQRGZm5u+//y5trZo8eTKVgl16KC4upusS5FsnzUMbqBMA0Dj9+vUzNzdPSkoiY66LpM7AwMA5c+aUlpbq6OiEhIQI2VP8+vVr2rRpP378YNI5btmyhcViOTs7y/SP0jyeguT9+/eUfyfXiOOANINGmlq7di0IpCWoE/TCf//9t6KiQgL5LmWRq1evYhHepQHJ2KvzC92OjkTKc5vKwVsOAExYsGABNWZQC8kCc+bMmWnTppWWlpIuoXv37l21atXBgwevXLny6tUrKiMvQ6KiooYNG5acnPz06dP27dvzPnnv3r3BwcFr167lkS5X+mkeT0GBJm4HdULmoBaqO3fubGtrCwJpmeoEj4lgC4fM8wtyYEh6ejpfb52UALsTAMAIe3v7Xbt2kb5Gz58/byj2caNUVVU5ODhcuXKFKkn6j/+fli8n17NnTysrq6FDh44YMYLHCkpOTs6xY8fc3d3JKIdkGBweODs7Hzp0aM6cOW5ubrL7WzSPp+CqTgwZMqRRF1JAqqirq6N2JzZv3sw15g/QQtSJ9PR0MzMzkA9d3wZFS0h1QkVFxdDQENQJAJB5OnXqNHr0aDIHxfPnz+kpDpnw9evX2bNnx8bG8j6tvr6e1DHIpDN6enrW1tbdunUzMjLq1KlTZWVlXl7ejx8/AgMD379/z+Fw2rRp4+3tzTVSNUVJScnSpUv9/Pxmz559/fp1fgOzSAnN4ykwKioqIiIiyM+LFy+G/5oUUl9ff/jw4cDAwKlTpzo4OFDRZgmC8Pf3z8nJIZ1ewNKphWBsbCwnJ1dfX89kIghQy2f/+9//mMcclwAN5QCQQnWia9eu/EaklDQQ3xcAGBIeHk7+a9TU1ARIU+Xl5SUOL6j+/fszSZV/6NAhPT29u3fvyvRP0DyeAsPHx4f8Kdu3b4+l7ACkhODgYOofd/z4ca4Jp4KDg0FQLYdOnTrRe+PNmzeDZDAKCgqolCDJyckgkEaZNm0a/dWysbGR8maD7wQAMGXo0KEzZ84kl5P5Cv/869evuXPnktnZ5s+f7+zs7O3tHRERER4e7uvre+jQodGjRwuQX0JbW9vFxeXNmzdMzGPWr1+fnJwshbE1+KJ5PAXGjRs3yA9btmwRYZoRQIRQKh9BEC9fvqQ+h4aGkqsMW7ZsGT9+PAiq5cDV3gl2J+h4eXlVV1eTaX/RrO0AX7sTUu6HDcZOAMAfR48effToEZvNvn79OnO7FNJy6c8//1y4cCFqJkHh5ORUVVX15s2b169fv3r16t27d5WVlTwqNDc3nzt37oYNG7jWxhVNTc1mIP/m8RQoBQUFT58+Jbfd16xZA38x6QRdih43bhz5ITMz087OjiCIadOmnThxAqTU0tQJ0vYV1AneUOkdV6xYAdIQWJ2Qcj9sUCcAgD969OixfPnyixcv/vPPP9nZ2QYGBkyusra2TkhI4H2OiorKmP8gCKKmpiYyMjI6OjonJyf3P2pqavT09Dp06GBkZDR58mQjIyP4LZoHPj4+tbW1BEE4ODhIONkFwJwVK1Zcvnw5MzOTIIjc3NyCgoLg4OBDhw7l5OSsW7fO1dW1ebjxAHypEwwngi2Zd+/ekWOfhobG7NmzQSCN8vPnTypouGypE5AVGwD4Izc318TEpLy83NHR8ciRIyAQQGA4HI65uXlycnKHDh0+f/4M6oQ0k5qa6ujo6OvrSw2a48eP37Zt24QJE0A4LZD79+9zDQpcUFCgra0N8iFZtWrV5cuXCYJYuXLlpUuXQCBMFLDBgwfTy799+8bVXUd6gAUVAOAPPT29rVu3EgTh7u5eXFwMAgEExt/fn8ymfPr0adAlpJyuXbv6+PiUlZXF/UdZWVlwcDDoEi2WhmzZITc2RUVFxZ07d8jP4rZ0qq2tjY+P9/b2rqqqkmmhyWiUWDB2AgBB2LZt24ULF37+/Onu7r5r1y4QCCAYR48eJW3x586dC9KQCdTU1Pr06QNyAIyNjVksLvYd6enpFhYWIB+CIO7cuVNSUkI6+1lZWYmqWjabnZ6enp2dnZWVlZaW9vXr1/j4+OTk5JqaGnl5+alTp8p0QIuGHCek35wS1AkA4BsNDY29e/c6ODi4urquXbtWOmNXA1LOkydP3r9/r6Sk5O7uDtIAAJlTLHv37h0fH89kOtgyodJNiHZrIjY2dujQoaTLGcaAAQNkPVwH1/dn6NCh0t9yMHYCAEFYs2bN2LFj8/Pz9+zZA9IA+KW6uprMhHjq1CkeWc8BAJBahg8fTi/8+vUrSIYgiM+fP5MxlJWUlBYsWCDCmi0sLGpqasrLy9+/f49tBJGxTGQarsZyXN80UCcAoDkgJyfn7e1tYGBw/vz5RrNcAwDGyZMnv379amdn5+DgANIAAFlkxIgR9ML379+DZAiC+Ouvv8gPv/32m66ursjrV1NTs7S0nDdvHlo4evRomRYah8OJiYlh+KaBOgEAzYR27dr5+PjIycmtWrWK68YrAHDl8+fPR44c6dGjBxnzBAAAWYTrmnFiYmJZWVkLlwyZmon8vHz5cvHdqLy8nPqsqKg4bNgwmZZbSkpKYWEhVti5c2cpj+kE6gQACMvQoUNdXV0jIyMdHR1BGgATqqqq5syZo6io6OvrC9GcAEB26dChAz0bQF1dXWRkZAuXzMOHD//9918y/yOV9lEcREdHU5+trKzU1dVlWm5v375lqLWCOgEAzY1169YdPHjw9OnTAQEBIA2gUbZs2ZKamvrkyZNevXqBNABApuFqhRIREdHCxUI5YS9btkx8IYnq6+tfv35NHTYDxwmu6oRMWDqBOgEAImDPnj3bt2/38/MDUQCNEhAQ8ODBA5mI1AEAAG+4rhxznRS2HLKzs4OCgkgPwyVLlojvRlFRUahpUDNQJ7gqorKyOwGBYgFABBw/fjwsLAzkADTKzZs3Zd3AFwAAEq4rxy1cnfD09KyrqyMz6nTu3Fl8NwoODqY+q6ioDBkyRKblVlZWlpiYiBW2a9fO1NRUJtoPuxMAIBpgjgjAewIALQpjY2MDAwOsMC8vLzU1tWUKhMPheHp6kp/FnQmb3AMhGT58uLKyskyLLjIyklTDUGRlawLUCQAAAAAAAAHhOuFrse4TL168IDMn6Ojo2NjYiO9GJSUlqJAnTJgg66KTaT9sUCcAAAAAAAAEBOydUCgn7IULFyopKYnvRv/88w+bzW726oSs+GGDOgEAAAAAACAgsDtBUVRUdP/+ffKzWNNNYJZOenp6ffr0aX7qhKamZt++fUGdAAAAAAAAaM6Ym5vr6OhghfHx8RUVFS1NFDdu3KiqqiIIYtCgQaINhF1bW1tSUoKWoH7Y48ePl3XRpaWl/fz5Eyu0trYWX5hdUCcAAAAAAACkAhaLNWXKFKyQzWZ/+PChpYni6tWr5AeRbE2w2WwfHx9bW1tTU1M1NTVNTc22bdsePXqUIIjU/6DObK6WTjNnzpShR4BAsQAAAAAAAAIyb96869evY4Vv3ryRIcN34fnw4UNsbCxBEOrq6nPnzhWmqurq6jNnzvz5559ZWVkqKirDhg1bunSpiYmJr6/vrl27dHR0UK8JFovVDHYn6NZxioqKs2bNAnUCAAAAAACg+TN+/HhdXd38/Hy08PHjxzt37mw5QqCcsOfMmaOhoSFwPTExMYsWLSIzMAwZMuT27dudOnUiv5o1a1br1q0vXrzYsWNH6vy+ffu2b99e1qWHuoKQTJo0SUtLS4YeAYydAAAAAAAABERBQeH333/HCiMiInJzc1uIBCorK2/dukV+FsbS6fjx44MGDUpMTGSxWDt27Hj9+jWlS5A4OjrGxsY+e/aMKmkGlk6Jif8fe/ceFNV5+H/8sIAgohECRhoqlJDgNAlpuIQhMDRgIwmFuiJJK+mgEHNpLMNYAjHNYJN0WrAxqVZGtKIVlZsUUShFW5UCglbAqkM74CJIEajKRQii3HZ/8+uZ7+mZXV0FgV3g/frrPI9nDw+HM8757HOrU6lUWpWrV6+eXr8FcQIAAGD8IiMjtWrUavXRo0dnya+fl5fX29srCMLSpUv9/PzGcQWNRhMXF7dx48bh4WFBEH7xi19s3rzZzEx7BI2zs7OFhYV8mvsMiBPSclgSKyurH/zgB8QJAACA2cLf39/R0fGBr4kzlTTSaXxdE2q1+r333vvd734nFkNCQjZt2nS/M4eGhuSv3f7+/tP97h0+fFirJiwsbN68ecQJAACA2cLExER3/nFpaan4nb2x6ezsTEtLa21tnZCrqVSq8vJycfZwVFTUOK6wYcOG3bt3i8cuLi4HDx40MTG555k9PT2jo6NSMSAgwMLCYlo/OVevXhWnsMtNu5FOxAkAAIBHpfsKODQ0VFxcbGztrK2t9fT0/OCDD6TegEckdU2EhYUtWrRorB/Pzc2VWqJQKPLy8vRMQb5586a8OCNHOi1cuPD1118nTgAAAMwuHh4ezzzzzANfFg0rIyPD39//3//+tyAIBw4ckK+4Oj4jIyP79+8Xj8cx0qm+vn7dunVSce3atR4eHnrOv3z5srwYHBw88+JEeHj4nDlziBMAAACzjm4HRUlJibhRtMENDw/HxsauXbtWas/169f//Oc/P+Jli4uLOzo6BEF48sknx/pyr9Fo3nrrrf7+frE4d+7czz//XP9HqqqqpGNHR8dvf/vb0/qBuXHjRmVl5QOfIuIEAADALI0Tt2/flq9qaijXr18PCgpKTU3Vqpf2sR43aaTT2rVrTU1Nx/TZzMzM8+fPS8UNGzY8+eST+j8if/nWP9Lp7t271dXVaWlp69atM9oVe48ePapWq+U1TzzxRGBg4HR8+E00Gg3/BQAAADwiDw+Pf/zjH/Ka6OjoR39rfxRnz56NiIhoa2vT/Sdzc/O2tjZ7e/vxXbmjo2PJkiUjIyMmJiaNjY0uLi4P/9nBwUE3N7eWlhaxaG1t3dbWtmDBAj0f6e/vt7e3l3pXcnNz33zzTd14k5OTo1KpWltbxTf1+fPnd3d36645awxCQkJKSkrkNbGxsRM1p2WK0TsBAAAwAXTnDxQWFsoXI5piu3fv/u53v3vPLCGOgDpw4MC4L75v3z5x9kVgYOCYsoQgCLt27ZKyhCAISqVSf5YQF1SVsoRCofje976ne86qVatSU1OTk5Ol/BAQEGCcWaKvr+/kyZPyGhMTk3fffXeaPvnECQAAgAkQExOj9WV/V1eX1lvj1BgeHn73v5599tmtW7cWFhZ+9tln8+fP1zrtD3/4w/iur9FopF6XcUzCzsjIkBd19wHUJc35FgTBy8vL1tZW95yFCxe6ubmtXr3a3d1drFm2bJlxPipFRUXyPTTEzornnnuOOAEAADB7zZ07Ny4uTqtyx44dBni9Uyj++c9/pqen19bWxsXFhYWFbdq06eTJk1obNdTV1VVXV4/j+mVlZY2NjYIg2NjYhIeHj+mzjY2N8lkTdnZ2r776qv6PtLS0lJaWSkX90741Go20BpTRxomdO3dq1WzcuHH6PvnECQAAgImxfv16rU6AoqIi+cCeqWFqalpeXv7222/Lt4Tz9vb+8MMPtc4cXweFNAn7rbfesrS0HNNn8/Ly5EWlUvnA8Ui//vWv5bOW9ceJurq6vr4+QRAWLVr0/PPPG+FDcuHChdOnT8tr/Pz8pvUO38QJAACAibFw4cL33ntPXqNWq9PS0qa+JfdcaikuLk6rgyI7O3usq9n29vbm5+eLx+MY6STvZxAE4dlnn9V//pUrV+TT2R977DEfHx8950tv6kFBQffbYNuwdFfZmtZdE8QJAACAibRhwwatV/b09HQj2YDC3t5eqVTKa27dujXW7fYyMzPv3LkjCIKnp+d3vvOdsbZBq6/mqaee0n9+UlKSfMe9oKAg/b0Z0mQV4xzp1N3dnZWVJa957rnnvv/97xMnAAAA8P994xvfiIqKktd0dXXl5OQYSfOio6O1asa6lK000mkcXROCILS2tsqL+leFys/P17p1+idaqNVqqffjnqs/GdyePXvEMCZJTEw0zl6Uh8e+EwAAABNJpVItXbpUPtzfy8trfJOeJ5xarXZycrp27ZpUo1AompublyxZ8jAfv3DhwosvvijOO+/o6HjsscfG9NO7urrs7Oy0au65TJMgCO3t7e7u7gEBAfL+k8uXLz/99NPisUajyc/Pj4iIkP61pqbG29tbTClXrlwxtgdDrVa7uro2NzdLNU5OTo2Njca5mu3Do3cCAABgIj399NOrVq2S19TU1Pz97383ijc/hWLNmjVa77haK7fqkZ6eLh5ERESMNUuIMx/Mzc21MsM9z+zt7VUqlStXrly8eLFU6ejoKGUJQRASEhJ27Nghj23SNuTGOdKpuLhYniUEQYiPj5/uWYI4AQAAMPF0J9fqTsA1lOjoaK3RNfv27XuY4Sp3796Vxv2Pb6STmZmZ1uimM2fO6J7W19cXHBzc2dn529/+9uLFi1K9fPmjr7766uDBg1lZWQrF/95mT5w4IR5II506OjrCwsKWLFli+n/MzMzc3NykM6fS9u3b5UU7O7vx3UbiBAAAwAzn4eGxfPlyec2hQ4du3LhhDG176qmnAgIC5DVNTU1lZWUP/GB+fn5PT48gCK6urlpXeHhaSzklJyf39vbKa/72t7+99NJLly5dyszMtLa2ln+d7+joKB5s27YtMTExKytL3ndx586dyspKcYfpwMBAsdLBwaGoqOjUqVPq//L396+qqmpoaJj6mRUNDQ1aGSY2NtbKyoo4AQAAgHvYtGmTvDg0NLRr1y4jaZvuhOyH2YBCPgl73LOHP/74Y3l/QnNzs5eXV3p6enl5+Y4dO8LDwwMDA9vb248dO+br6ysuhSSdnJ2d/dVXX73xxhvx8fH79+8PCgqSX7miomJwcFAQBHd3d63tybds2WJqarp582Yxqxjknqempsq7gOzs7GJjY2fIs64BAADAJNCapWBjY9Pd3W0MDevv79fabs/Kyqqvr0/PR65cuSJGCFNT0/b29kf56e+8846eV1MbG5uzZ89KJzs5OWmdYGZmdujQId3LJiQkiCf87Gc/k9d/9tlnixYtKi0tNeANb21t1drvb8+ePTPmOad3AgAAYFJ88cUX8mWLenp6UlJSjKFh8+bNe/PNN+U1AwMDubm5+rsmxC/XQ0JCHBwcHuWnp6WlpaSkaO3OIbYqISGhoaFBvlGd1qR2Hx+fs2fPvvHGG7qX1Z04oVar4+LiDh06dO7cuVdeecWwXVXyvUf8/Px0O4imLxaKBQAAmCzp6enyL+MtLS1VKpU0B8CAqqqq/Pz85DW+vr5VVVX3PHl0dNTJyamtrU0QhCNHjqxYseLRG9DS0lJZWVlXV9fZ2fmtb33L1dX1lVde0RqkJAjC7du3s7KyCgsL3dzc/Pz8lErlPcdZ3bx584knntBoNObm5j09PfPmzevv74+Kiurr68vPzx/HIlQTqK6u7oUXXpBWoDIzMzt//vzzzz9PnAAAAMADaDQacfqvVBMTEyNNQjCspUuXNjQ0yGvq6+vd3Nx0zywuLg4NDRUEYfHixa2trUa4tmlubu6PfvQjcfWniooKlUqlVCo9PDz27t2rtTTt1AsNDS0uLpaK8fHxW7ZsmUkPOYOdAAAAJouJiUlaWpr8/TsjI+Nf//qXMbTt4SdkS9tNrFmzxjj3SZCPdCosLPT29q6vr9+yZYvBs0RZWZk8Szg6On766acz7CEnTgAAAEwid3f3uLg4qTg6Ovrzn//cGBoWFRVlamoqr9m/f//o6KjWadevX//Tn/4kHsfExBjnTZbiRFlZmVKp7O3tVavVUgoyoI8++khe3LZtm7W1NXECAAAAY/Dpp59+85vflIpHjx693yyFqeTg4PDaa6/Jazo6Oo4dO6Z1WkZGxsjIiCAIAQEBzzzzjBHe3sbGxqtXr4rHt27d+uSTT8TjnTt36qajqfTHP/5Rvht6SEhIeHj4zHu8iRMAAACTy9raetu2bfIarS+tDUW3t0F3vNPevXvFA6PdwlnqmvD09KysrExMTBR7AK5du3bkyBFDtWpkZETeDWVpaam1KzZxAgAAAA9r5cqV4mxm0enTpwsKCgzeqtDQUDs7O3lNUVFRZ2envJ3idO0FCxZEREQYeZxYtWrV3Llz58+fL07LFgRh69athmrVzp07VSqVVPzkk09cXFyIEwAAABin1NRU+TYUH3zwgXzLZ4OYM2fOj3/8Y3nN0NBQZmamVJSmH0RGRlpZWRnhXVWr1adOnRKPpR0nfvKTn0hx6Ny5c1Pfqubm5o8//lgqvvjii4mJiTP1wSZOAAAATAUnJ6fs7Gxp9vN//vOfn/70pwZvlZ71nfr6+vLy8sRjox3pVFtb29PTI26n7enpKVZ6eHgEBASIx1pbB07B8CeNRhMdHd3f3y8W7ezsCgoK5syZQ5wAAADAI1m+fHlycrJUzM7Ozs/PN2yT3N3dpbdw0cWLF8+fPy8IQk5OzsDAgHiOl5eXcd5SaaRTUFCQQvG/N9uNGzeKBwUFBdJ86PLy8pycnMlu0tatW8vKysRjU1PT3NxcJyenGfxUEycAAACmTkJCgjSyXxyWc/PmTcM26X4dFNJIp3Xr1hnt/fzrX/8qHkgjnUSvv/66v7+/1P7bt293dXW9//77X3zxxaS2p76+Xj4D+ze/+U1QUNDMfqTZFRsAAGBKDQwMvPzyyxcvXhSL4eHhhu2j6OnpcXBwGBwclGpsbW2PHz/u7e0tCIKFhUV7e7t81odR3UlbW1ux5SqVytXVVf6v1dXVvr6+4lqxjo6OfX19X3755aRGo5GRkZdffrm6ulosrl69Oisra8Y/z/ROAAAATCkrK6uCgoLHH39cLB4+fNiwL502NjZKpVJe093dLe2QsHLlSuPMEuJMazFLODs7a2UJQRC8vb03b94sHre1ta1fv36yu1lSUlKkLPHCCy8Ywz56U4DeCQAAAAM4ceLEa6+9Jn53bmtrW1dX5+DgYKjG/OUvfwkODr5fO5ctWzZ973NlZeWlS5d8fHw8PDwm9QdduHDhpZdeGh4eFv+gtbW1zs7OxAkAAABMli+//PLDDz8Uj319fU+dOmVpaWmQlqjVamdn59bWVq16Z2fnpqYmExMT/lj6dXZ2+vj4NDU1idOvS0pKXn311VnyuzPYCQAAwDDi4+MjIyPF4zNnzkRHRxvqe16FQrFmzRrd+piYGLLEA929e3fFihVilhAEITk5efZkCXonAAAADGlwcHDFihXHjx8Xi0lJSZ9//rlBWtLU1OTq6ip/M1QoFC0tLY6OjvyZ9NBoNJGRkdL6s3FxcQbcitswWZSHAAAAwFAsLCyOHDkiLXL6y1/+8uDBgwZpiYuLi7T1myg4OJgs8UBJSUlSlli/fv1syxLECQAAAAOztLQsLCyUdid4++23KyoqDNKSmJgYedFod8I2Hvv27fvVr34lHr///vvbt2+fhTeBwU4AAACGNzAwEBoaWlpaKgjC448/XlNTM/XrAg0MDCxevPjrr78WBMHe3r6trc3c3Jw/zf1UVFQsW7ZMXMrpnXfe2bVr1+ycZ0LvBAAAgOFZWVmVlJSIG2Z3dXXV1NQYpA0//OEPxeOoqCiyhH4nT54Us0RSUtLvf//7WTtnnTgBAABgFCwsLLKysj766CMDtkEa78RIp4dhZma2Z88eQ82eN5abwHMAAABgJExMTFJSUpydnU1NTQ3SAF9fX5VKpVAoXFxc+HPot2DBguLi4uXLl8/2h5a5EwAAAMZGrVYrFIwi4W9EnAAAAAAwc5GoAAAAABAnAAAAABAnAAAAABAnAAAAABAnAAAAAIA4AQAAAIA4AQAAAIA4AQAAAIA4AQAAAIA4AQAAAADECQAAAADECQAAAADECQAAAADECQAAAADECQAAAAAgTgAAAAAgTgAAAAAgTgAAAAAgTgAAAAAAcQIAAAAAcQIAAAAAcQIAAAAAcQIAAAAAcQIAAAAAiBMAAAAAiBMAAAAAiBMAAAAAiBMAAAAAiBMAAAAAQJwAAAAAQJwAAAAAQJwAAAAAQJwAAAAAAOIEAAAAAOIEAAAAAOIEAAAAAOIEAAAAAOIEAAAAABAnAAAAABAnAAAAABAnAAAAABAnAAAAAMxO/y8AAP//+/YfLypb24wAAAAASUVORK5CYII=\n", + "text/plain": [ + "<IPython.core.display.Image object>" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from IPython.display import Image\n", + "Image(filename='./images/qkv.png')" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-23T14:37:24.764368Z", + "start_time": "2022-08-23T14:37:17.596209Z" + } + }, + "outputs": [], + "source": [ + "import torch\n", + "from torch import nn\n", + "import math\n", + "from bertviz.transformers_neuron_view import BertModel, BertConfig\n", + "from transformers import BertTokenizer" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. model config and load" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-23T14:37:43.593937Z", + "start_time": "2022-08-23T14:37:26.027208Z" + } + }, + "outputs": [], + "source": [ + "max_length = 256\n", + "model_name = 'bert-base-uncased'\n", + "config = BertConfig.from_pretrained(model_name, output_attentions=True, \n", + " output_hidden_states=True, \n", + " return_dict=True)\n", + "tokenizer = BertTokenizer.from_pretrained(model_name)\n", + "config.max_position_embeddings = max_length\n", + "\n", + "model = BertModel(config).from_pretrained(model_name)\n", + "model = model.eval()" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-23T14:37:49.297031Z", + "start_time": "2022-08-23T14:37:49.282116Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{\n", + " \"architectures\": [\n", + " \"BertForMaskedLM\"\n", + " ],\n", + " \"attention_probs_dropout_prob\": 0.1,\n", + " \"finetuning_task\": null,\n", + " \"hidden_act\": \"gelu\",\n", + " \"hidden_dropout_prob\": 0.1,\n", + " \"hidden_size\": 768,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 3072,\n", + " \"layer_norm_eps\": 1e-12,\n", + " \"max_position_embeddings\": 512,\n", + " \"model_type\": \"bert\",\n", + " \"num_attention_heads\": 12,\n", + " \"num_hidden_layers\": 12,\n", + " \"num_labels\": 2,\n", + " \"output_attentions\": true,\n", + " \"output_hidden_states\": false,\n", + " \"pad_token_id\": 0,\n", + " \"torchscript\": false,\n", + " \"type_vocab_size\": 2,\n", + " \"vocab_size\": 30522\n", + "}" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model.config" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-23T14:38:37.060053Z", + "start_time": "2022-08-23T14:38:37.055373Z" + } + }, + "outputs": [], + "source": [ + "att_head_size = int(model.config.hidden_size/model.config.num_attention_heads)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-23T14:38:42.881098Z", + "start_time": "2022-08-23T14:38:42.874824Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "64" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "att_head_size" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-23T14:41:53.748482Z", + "start_time": "2022-08-23T14:41:53.742105Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[-0.0112, -0.0324, -0.0615, ..., -0.0383, 0.0031, 0.0059],\n", + " [ 0.0260, -0.0067, -0.0616, ..., 0.1097, 0.0029, -0.0540],\n", + " [-0.0169, 0.0232, 0.0068, ..., 0.0124, -0.0168, 0.0301],\n", + " ...,\n", + " [ 0.1083, 0.0056, 0.0968, ..., 0.0188, -0.0171, 0.0141],\n", + " [-0.0436, -0.1032, -0.1035, ..., 0.0138, -0.0488, -0.0453],\n", + " [-0.0611, 0.0224, -0.0320, ..., 0.0376, 0.0186, -0.0482]],\n", + " grad_fn=<SliceBackward0>)" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model.encoder.layer[0].attention.self.query.weight.T[:, 64:128]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. data" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-23T14:42:30.683705Z", + "start_time": "2022-08-23T14:42:29.810137Z" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/chunhuizhang/anaconda3/lib/python3.6/site-packages/transformers/tokenization_utils_base.py:2227: UserWarning: `max_length` is ignored when `padding`=`True`.\n", + " warnings.warn(\"`max_length` is ignored when `padding`=`True`.\")\n" + ] + } + ], + "source": [ + "from sklearn.datasets import fetch_20newsgroups\n", + "newsgroups_train = fetch_20newsgroups(subset='train')\n", + "inputs_tests = tokenizer(newsgroups_train['data'][:1], \n", + " truncation=True, padding=True, max_length=max_length, \n", + " return_tensors='pt')" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-23T14:43:26.668044Z", + "start_time": "2022-08-23T14:43:26.663985Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "torch.Size([1, 201])" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "inputs_tests['input_ids'].shape" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-23T14:46:49.614873Z", + "start_time": "2022-08-23T14:46:49.610494Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "dict_keys(['input_ids', 'token_type_ids', 'attention_mask'])" + ] + }, + "execution_count": 37, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "inputs_tests.keys()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-23T14:14:25.826897Z", + "start_time": "2022-08-23T14:14:25.824419Z" + } + }, + "source": [ + "## 3. model output" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-23T14:43:51.861587Z", + "start_time": "2022-08-23T14:43:51.134234Z" + } + }, + "outputs": [], + "source": [ + "model_output = model(**inputs_tests)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "- last_hidden_state (batch_size, sequence_length, hidden_size) : last hidden state which is outputted from the last BertLayer\n", + "- pooler_output (batch_size, hidden_size) : output of the Pooler layer\n", + "- hidden_states (batch_size, sequence_length, hidden_size): hidden-states of the model at the output of each BertLayer plus the initial embedding\n", + "- attentions (batch_size, num_heads, sequence_length, sequence_length): one for each BertLayer. Attentions weights after the attention SoftMax" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-23T14:44:04.984971Z", + "start_time": "2022-08-23T14:44:04.981312Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "3" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(model_output)" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-23T14:44:36.364871Z", + "start_time": "2022-08-23T14:44:36.360873Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "dict_keys(['attn', 'queries', 'keys'])" + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model_output[-1][0].keys()" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-23T14:46:08.927665Z", + "start_time": "2022-08-23T14:46:08.921389Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[0.0053, 0.0109, 0.0052, ..., 0.0039, 0.0036, 0.0144],\n", + " [0.0086, 0.0041, 0.0125, ..., 0.0045, 0.0041, 0.0071],\n", + " [0.0051, 0.0043, 0.0046, ..., 0.0043, 0.0045, 0.0031],\n", + " ...,\n", + " [0.0010, 0.0023, 0.0055, ..., 0.0012, 0.0018, 0.0011],\n", + " [0.0010, 0.0023, 0.0057, ..., 0.0012, 0.0017, 0.0007],\n", + " [0.0022, 0.0056, 0.0063, ..., 0.0045, 0.0048, 0.0015]],\n", + " grad_fn=<SliceBackward0>)" + ] + }, + "execution_count": 36, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model_output[-1][0]['attn'][0, 0, :, :]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4. from scratch" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-23T14:46:57.878825Z", + "start_time": "2022-08-23T14:46:57.868066Z" + } + }, + "outputs": [], + "source": [ + "emb_output = model.embeddings(inputs_tests['input_ids'], inputs_tests['token_type_ids'])" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-23T14:47:06.547071Z", + "start_time": "2022-08-23T14:47:06.542839Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "torch.Size([1, 201, 768])" + ] + }, + "execution_count": 39, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "emb_output.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-23T14:48:17.703437Z", + "start_time": "2022-08-23T14:48:17.698015Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): BertLayerNorm()\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): BertLayerNorm()\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + ")" + ] + }, + "execution_count": 40, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# model.encoder.layer[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-23T14:49:09.030869Z", + "start_time": "2022-08-23T14:49:09.026963Z" + } + }, + "outputs": [], + "source": [ + "# emb_output[0].shape: 201*768\n", + "# query.weight.T.shape: 768*768, query.weight.T[:, :att_head_size]: 768*64\n", + "# 201*64\n", + "Q_first_head_first_layer = emb_output[0] @ model.encoder.layer[0].attention.self.query.weight.T[:, :att_head_size] \\\n", + " + model.encoder.layer[0].attention.self.query.bias[:att_head_size]" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-23T14:51:02.471342Z", + "start_time": "2022-08-23T14:51:02.467129Z" + } + }, + "outputs": [], + "source": [ + "# 201*64\n", + "K_first_head_first_layer = emb_output[0] @ model.encoder.layer[0].attention.self.key.weight.T[:, :att_head_size] \\\n", + " + model.encoder.layer[0].attention.self.key.bias[:att_head_size]" + ] + }, + { + "cell_type": "code", + "execution_count": 48, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-23T14:53:33.339672Z", + "start_time": "2022-08-23T14:53:33.335941Z" + } + }, + "outputs": [], + "source": [ + "# (201*64)*(64*201) ==> 201*201\n", + "attn_scores = torch.nn.Softmax(dim=-1)(Q_first_head_first_layer @ K_first_head_first_layer.T / math.sqrt(att_head_size))" + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-23T14:52:07.447498Z", + "start_time": "2022-08-23T14:52:07.442701Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[0.0053, 0.0109, 0.0052, ..., 0.0039, 0.0036, 0.0144],\n", + " [0.0086, 0.0041, 0.0125, ..., 0.0045, 0.0041, 0.0071],\n", + " [0.0051, 0.0043, 0.0046, ..., 0.0043, 0.0045, 0.0031],\n", + " ...,\n", + " [0.0010, 0.0023, 0.0055, ..., 0.0012, 0.0018, 0.0011],\n", + " [0.0010, 0.0023, 0.0057, ..., 0.0012, 0.0017, 0.0007],\n", + " [0.0022, 0.0056, 0.0063, ..., 0.0045, 0.0048, 0.0015]],\n", + " grad_fn=<SoftmaxBackward0>)" + ] + }, + "execution_count": 46, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "attn_scores" + ] + }, + { + "cell_type": "code", + "execution_count": 52, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-23T14:56:15.178493Z", + "start_time": "2022-08-23T14:56:15.173311Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000,\n", + " 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000,\n", + " 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000,\n", + " 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000,\n", + " 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000,\n", + " 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000,\n", + " 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000,\n", + " 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000,\n", + " 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000,\n", + " 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000,\n", + " 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000,\n", + " 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000,\n", + " 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000,\n", + " 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000,\n", + " 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000,\n", + " 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000,\n", + " 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000,\n", + " 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000,\n", + " 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000,\n", + " 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000,\n", + " 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000,\n", + " 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000,\n", + " 1.0000, 1.0000, 1.0000], grad_fn=<SumBackward1>)" + ] + }, + "execution_count": 52, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "attn_scores.sum(dim=-1)" + ] + }, + { + "cell_type": "code", + "execution_count": 49, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-23T14:54:01.596921Z", + "start_time": "2022-08-23T14:54:01.592186Z" + } + }, + "outputs": [], + "source": [ + "V_first_head_first_layer = emb_output[0] @ model.encoder.layer[0].attention.self.value.weight.T[:, :att_head_size] \\\n", + " + model.encoder.layer[0].attention.self.value.bias[:att_head_size]" + ] + }, + { + "cell_type": "code", + "execution_count": 50, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-23T14:54:15.701673Z", + "start_time": "2022-08-23T14:54:15.698925Z" + } + }, + "outputs": [], + "source": [ + "attn_emb = attn_scores @ V_first_head_first_layer" + ] + }, + { + "cell_type": "code", + "execution_count": 51, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-23T14:54:24.153734Z", + "start_time": "2022-08-23T14:54:24.149518Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "torch.Size([201, 64])" + ] + }, + "execution_count": 51, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "attn_emb.shape" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.8" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/fine_tune/bert/tutorials/06_dive_into.py b/fine_tune/bert/tutorials/06_dive_into.py deleted file mode 100644 index 3ce3efa..0000000 --- a/fine_tune/bert/tutorials/06_dive_into.py +++ /dev/null @@ -1,49 +0,0 @@ -from bertviz.transformers_neuron_view import BertModel, BertConfig -from transformers import BertTokenizer -import torch -import math - -import numpy as np - -np.random.seed(1234) - -max_length = 256 -config = BertConfig.from_pretrained("bert-base-uncased", output_attentions=True, output_hidden_states=True, return_dict=True) -tokenizer = BertTokenizer.from_pretrained("bert-base-uncased") -config.max_position_embeddings = max_length - - - -from sklearn.datasets import fetch_20newsgroups -newsgroups_train = fetch_20newsgroups(subset='train') -inputs_tests = tokenizer(newsgroups_train['data'][:1], - truncation=True, - padding=True, - max_length=max_length, - return_tensors='pt') -# print(inputs_tests['input_ids']) -# with torch.no_grad(): -# model = BertModel(config) -# # print(config) -# embed_output = model.embeddings(inputs_tests['input_ids'], inputs_tests['token_type_ids'], ) -# model_output = model(**inputs_tests) -# print(embed_output) -# print(model_output[-1][0]['attn'][0, 0, :, :]) - -# print(inputs_tests['input_ids']) -with torch.no_grad(): - model = BertModel(config) - # print(config) - embed_output = model.embeddings(inputs_tests['input_ids'], inputs_tests['token_type_ids'], ) - print(embed_output) - model_output = model(**inputs_tests) - print(model_output[-1][0]['attn'][0, 0, :, :]) - att_head_size = int(model.config.hidden_size/model.config.num_attention_heads) - Q_first_head = embed_output[0] @ model.encoder.layer[0].attention.self.query.weight.T[:, :att_head_size] + \ - model.encoder.layer[0].attention.self.query.bias[:att_head_size] - K_first_head = embed_output[0] @ model.encoder.layer[0].attention.self.key.weight.T[:, :att_head_size] + \ - model.encoder.layer[0].attention.self.key.bias[:att_head_size] - # mod_attention = (1.0 - inputs_tests['attention_mask'][[0]]) * -10000.0 - attention_scores = torch.nn.Softmax(dim=-1)((Q_first_head @ K_first_head.T)/ math.sqrt(att_head_size)) - print(attention_scores) - |