diff options
3 files changed, 1125 insertions, 0 deletions
diff --git a/fine_tune/bert/tutorials/03_bert_embedding-output.ipynb b/fine_tune/bert/tutorials/03_bert_embedding-output.ipynb new file mode 100644 index 0000000..7c42a70 --- /dev/null +++ b/fine_tune/bert/tutorials/03_bert_embedding-output.ipynb @@ -0,0 +1,1020 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "ExecuteTime": { + "end_time": "2022-07-03T04:20:09.832759Z", + "start_time": "2022-07-03T04:20:09.828238Z" + }, + "scrolled": true + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABAAAAAFUCAMAAACN5L73AAADAFBMVEX///8iIiJdXV3p6en/8swAAADZ2dn0zM3z8/O216j29vY+Pj78/Px4eHjU1NTm5uY4PTZAQEB9k3O3rZI7ODLWy6tRUE6nxprhvL3X19dCNzfv7+/9/f0BAQGIcnP5+fnc3NzY2NhkZGRZWVkkJCP19fUJCQnw8PAFBQX+/v6IcnJoaGjo6OgSEhJVVVV6enqdnZ3yysvNzc26urqrq6vt7e3k5OS2trYmJibi4uL77skwMDDe3t4MDAxwcG8qKikPDw7Q0NAoKCey0qRzc3OTk5PvyMgzMjFyhGqiiIm11qdMTEyIiIgtLSw6OjrDw8OwsLBbW1vx8fE8OjU9PT0YGBg/PTjT09Ozs7P+8ctBQUGpoYg7Ozv468a9vb1lV1dJSUmAgH/MwaNRUVHkv7+sy5/CwsJ2dnYeHh40NTRBOjqz1KXCo6M6NTVsbGwxLixea1jg4OA4OTjy8vKFnXvrxMWkpKTZtrZXVEtKU0bBt5wbGxr6+vpbaFYzOTHLqquqyJ2RrIahoaG6sZXeurthYWE2OzRGRkZ+fX3IyMhEQTioqKienp7j17bSsLGQkJBqZVhOS0PAwMDw5MDq3rs9NzdPRUXGxsbt4b5fWk6fl4FDQ0OYgIB6j3CQqIaYmJhiclu7np43NjZWS0vLwKQ2NDFPT0+MjIyYkX34+PgyNTB4Z2cVFRWdhYVxYWFsfmSCgoI8RTeCb2+mxJmbm5vMzMywz6JOWUmXsYyPj4+qkJCPh3SwlJVqW1yiv5WTfX19a2tFUEBFQz3nwsKcuJHTyKmQenq1mZmIgm+FhYXPxadXV1fGp6iWlpbm27mAemt3jG5neV/f1LNeUlL06MN0ZGTHvaFKRkIuMS1aTk5IQEBxbF5gWlrLy8tFPj5VUUW0q5JSXk5aV00/Pz/St65TSEiBlnhETEF8dme+tJp4c2Ofmo6upo2VjnqLd3fVy6xXZFKLo4F1iG1jX1M9RDt1cGFugmY5PjaZi386MTGSi3jaz69BSD6Ff22GgG+CfWy2kVifAABW00lEQVR42uycX2gT2RrAM1vsSNc/7Mtl2mtjrXG11u3u0W2bYm006JSh62VZFEUuThcD1w0zLauM82Bh4BKposwFodMH8zKWtkg2QcL1YcMMDgQapLlPIUgCNy+XvvY9b3cmTdo0SdfMmaz/+v0opaX06znfnPP7vjmdxOUCAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAOAToW+w0w6DO4TZN2onyujhxlHabI3leOMge9vtBBnqcxqhPKV9DYYy2InFyKFG8zp+xE6MtvoAHT1Yo+k5VhfpoP0oR9oaXqzBI50Oab/bIOyhIbxgg7tPAD2TfFfzFCd7GkaZeey1EaVrermhAS58ZSdIcXWkUZCO3mkbQfjxp/UR+u1EKON5PFOf24liFw7Tww1sMjJgJ1jxxoG6ab304Y1mrKP2Oj23P6/is18bXKwhzAxVD+9BfSE4NDyOFas4277b6v/IRDZENU/ImGhkgJknPGcjCsUF/9NRP5aeG3E7QUL6wMEGJeUfOclOEKZwpqYHGLyYkynbcMUntbt2ZDISonBI8PUGOLiqi3bm5Z69VtPWvHkrYY1GWnv407ZI8zfC9uclGn85UG/I1axIOURm+2uXwb6nPIMVK2TsUOA+W0YnDYG0gxBpYICZe0XVVhRSCi7XFcxrN9btjSUQO1rXA7zoVwL2hhLl57cXyimFIjFQizU9QM+EkSSxQKYBau4ChgZi9oLRJ2e3bbiONwUZbzRkSBmrntmFKysIZ1LZ2dqrdWpSp0nHJBdqeoC+YT6B8GIJxuRu6gH6RpciSfspqjXAzHc8ZzfhXKGmB+hrfxa3uxoE/dapuuot2gxCJ64MV0cYsx2hvMA5/kn1PhmajQi4axrV9gBDR2M2vYZo97OqHmDvy7cywh2OvPZwa4vNfxXG27VCfGl7D9A+EEmSLSC0vQew6j/2VGljdhf1AL9OYqxRIbLUU1P/edV+wqW55W1F7oDd+l/uAYacV+/w4qOtCGNKCHfXqnxVD4Bf/xv0AKfs1v/SharqATp+wK7/pS2WH6v46PT/VnCrNspOVp8DtE20ov6XlsFCb1UPcOZ5AuHHEoyJ3dID9PUsYRm4RpI49d8iXbi0dbzc1/nsJM5qEPSlturqncGp3nTib8NO639tDzAyG3G2uhPF4U2bjKzarf8b83JXTgKPPcyLyMlo5IX9f92o/1+G8eclxCe2mpL2W0aSbBEU21vpAWaeLjKOpkobz3ZJDzA0idmjJo2qHgCv/pd6AH55M8qRG268sVT3APjVO8w/qkQQHaydrR5gdMLp6kYaP1w+nWwb0PGCmT1AZ+lgZDkoOdxiYv7ivtL/aVaceI3e6gHallpV/0sGWOgvH1SeGU8gZ7GSu+UkcL+Ou0ZpfWzz5Hx0UcVNOHd5tBKlF7teCrk3lSCP89jdOzPZ5ixCpQf4ZWPtXHS+uhHzvHxK8kMugF3NHlhCusmLyOlwxKDZJd1dXXE2LzpWOU/8lhXIFhJQ/r0R93nU8VTp2NSuEMDYCnaKor2bN6g9Qew9Q3V1VqJ8wWCPxf1tJch3LP7y/vKgwwjlKQWPlIbSH3a+ptNzZQH8HMeOoQ1YD1xcV1qwxRZ+dLmOT3AOozC95SdApl6TLSX2+0ZcnnMei/kCBAACAAGAAEAAIAAQAAigNWfuIAAQAAhg1wpgpP9i27sU0dYJAgABgAA+SwF0+n8Zeddjub4xEAAIAASwWwXw9AQIAAQAAviMBdDz6KfT3/9oPZ114dXg03vXj1uPwM/vdbn2dc7fPbh8YmD+OAgABAAC+FwFsH98ynP1G9/jGdfEnaXbV38b6HEdnvKZQtjbP37h+6vd55+/AgFsIatCSwSABAF9bgKg02kaWwAh1bxmtEA3m+JmBICktPCnCODdF+/TEUD/nnP77/Wf8w257uy5/a+bY92rMy+OfnPBFMDR2/8d/fnE7PVBEEDVNSW4lgggrbuTrRNASP2Dl9XtJIBQStNE64sk1XjfipqmhWwIgJrLB7AFECaiJB1mUyQZIdQWCUDI8NKfIQA6posOBIAkTVOtq492yrysaSnqvQlg6rBr75XuA6473ZdmXMf+/vW1LQF84DMAs0xaBheoAG19U17jKBkwvzR/Rn/CAlDnWKp1AtB4t30BaHxeSZBIZQuFvK7RZFgpkUKGoqzpmkAmlPx46r0KwFhkPjYBJNHmwjMbFOvDjJufkxwIgM7yCiuTdKqU+RSNcqXE66KcUZSFuIRIRimMc+9LAN3XrcW+57Trznnrn4K/n7j30QiAiq3FzTxlugpZkZTZjaQn19e6cgwts2vMJyyAJCfTLRSA7w/et2AnASQKadOh6uW38ZVIYSFAGr6IYZJGuTnDWPCumys/Pae9TwEgmQt9ZAIQYrJolMs9ikbJqPVYP0o3dWuxowB01rqH0HyKmflgLomm81biV6g0n4tneT5tVjeOV9+XAL623oCuf88r1x3CenXlve6XZQGsfnABiL6chpjbXQZL5EX1fqka0ZH7ejz/20ooejvbtAAQx8iqOxsNoXQ4FudoUmRSJa2rjNisAFCIyWajmZIAQolINrzRdQc0Q19JMbJNAVAJlSaRyMSz6ynKlgBQQFuPrHBJIZWwtpvIpEk5681FGS1p7iBmc2BNCMDcHUgPmjfutJgSSGORok0QymUoWsxYOZTtCkDSjCxDbeQlHlvnaHsCSFuXY0MAEmP2IGaazCjWyQIKJNxZtya+WwC0lpATRsRKTkkASArHY5GomRQqoZUab4mRmhYAl5fVYPlmLZBh0FrUiqAlzF4goYmbsxXUuB5PMWqzArCCZApSKfM0mo7Qpcyn58JIUMcjVqLfnwDObQrgqnW3f6n7n6YATpuJvfXhBTAdJiUiZy4E9Xy4LACqmDGTV+iiyWLzAqCzfsVHEAS7Pm5+XuRI6WzQWksC729aANKah/Cav81Z70bj8fuIguUQivV6/ISHeG33DIBnBVJWPF4f4d2hfd9BAHLG6/ES/jC1ELSskyIMMkpY5GVSLXgWxz15zoYA2Ll0+XtjcWOhmwIIkMhtFU+7AjireAjCy5rzETP+ad57NkrbEsBJX6IsACnojQrWXM0o42bVFVmP57LHw75bAIH8dN4ahS5sCEBgCcLv9QTTpKictXaVwFp/pUkBvGaF17lK5udkKmj9MVqZDplpLc+Wtl7heJnwezwEa0MAdK7yemhTAKic3bD1ZmeZDyWA7psu18zkiWsvHpwbNn/qKwlg6gMLYP2+lT2UTpYFkFR8mixQIWRPAJ5FN5dSPJd1TYoRutnZ+a1eLuHRUZMCEDLemCSqb00BJFl/VhTD0wWRFAzvQjqU1rEEgGIetyxy+g4vZ20sgGRpHKncSpUAAq/9Men/7J3Pi9voGcdHXVqFkrBXbdKaHMRmMnShb3bTaJkGDaaREa1IKO0trCgIFoxd0RShwy4IFh2yFFQoSLkYijCmFNWm6LbCxgKDRVnvSZhBhvqmq/+Gvu8r2zPjGU8leZKZ0vdLDkM8fubVo1ef53mf94cMS7CO7YAxW1Ks5AYAHfDNUJxAd9Ihl7ahXAQAoCezEhkA67SMfo2dQP/yvsm49ZpVDgCNY7Ul0NqAr0Ardk2nF2zPMCdeLw8AWKdrBDYbZAAQul3LtEKIJTBkUZcxqrNRPgBYnlefo38w7IOxJ/M9+XDg6ysAsMfwaptsgwYiN490q1IIAHSXt8NgwgAIgA5yfM9EABAi1b8uAHz/N8/f//N+7e+3/7n/yzdff/tjCICv91/95d51AgD0qGVvXgKAdtX9agqjQzEA8Oh5H7JoS7zOzWAvlNoafJKlKG8NIFJTZlkDcGttEx0mBbusZWOMi4UB0IAAEBwWxxOlSAYAG47aoeinALCqAQy5FkChhdfzA0DotqsU3+4DOuR7HlRWA0j5Ll0cADZqkUeJdKMjo4dsSImlAGDNWHSH3doAjbUXVEAPsCVg5gEAHviLrAxOagBwTH3M0Cavws7kb9ysSwAQDvipV5VDDIDQdsL5LKysAYCNh7BhShv3I6MYALRWCj2fRACo8yl0/ALXAHpNfFvfDQD4UwB4tPdqnzs6+vDlG3QW3v7To8OXEACP+f2jJ9cKAJ/CNx2sAQAU17v7UVxwCKCiL/fZEPWQmg2DC3r4XDVW8gJgyOFzZxEARNYH+NGb0i6HUwix3BBgStmVyNCKFAHBmBtmvLgAADIro0e4frZQeTkAaKAxjUqzqZ+qAahOMnWFEgDARcAWdIbIHqOWxJRfBgBjR8WHfHVZB1lJqAU95mpeZIxAHgDM0d03OqmSAUARZVuVWJiv0T22Bax53coJACBETW0yswSApwDSLhOL0EErAMwRibvUmLbqqVkcANjzBx2bOakBNO12XMEHpbwTADz45MWtvTdfoW725qeP914dPf/uF8/uoy2C9//6+XeP/v2n9/b2Hn3zqx9c6xBAzHrztLsEgO5BX4+8D80SAIgyADQhAGCHmwKP6uadBRD8qriaBRizuKNMqJgOKI8unQHQVsJRnN2zCgBACKvLZl0AAIdt1rGKAAB3xQU1OVUDSPRsUAJKAQA6A4wpKWvJuAwAahS+NFCh1MzKkNZ7LMV25EluAFi2Y2IAjAZVqT0NmwgAkZQwYnXzpKdLagDDGLRWxwTpTsPK2HEaACIEQKODA0lRAGAX+5R1qgYwXE05vhMAbOjV0eOHt5c7hB/+aLVV+OHD660BWBRMvYXgaLgCwD9CDcbBp8yOANDrdnTY1PICAFS4Lr3OAA7woIDq0S67SwZAC0a3V2flAjUAsOBaawAYGwCIOVfIYkluAIz8CI5ClR4cZoc1czULoKzLXuUAAGO3f0FL8gKgnrAopIIWt1hbEaxg6rBzLXcGUEs0DIA+m+oC0B0EACXmA0ea5JwGtOR2p5k0O21/RAuL9kxKHS5tG+cBYJTJAJQwgmM/ZdBBGYC2mgU46R/XAoAbdh4AAgA4eDr3Bl/Udfe3M1mWx8zgI2caf5EIOwJA8Lh6lsrnGwJ0ORw4EAAiHg9wKzAzNdSZjvLyUhkAQH8duJI9KlADEKt4XEyDUdKZnAAANy5kF6DIOgCUH7dReEwkCJCQi6GD5cYJAEpnANBFsVJiHcASACLTptCZw/0TK9hVVke18gJAZHtZDcCnUJ8yMQBg2+ZcW8gJAKa7OPT8mt/qa7QQDdO05aTDlnkeAKZziNrgFgIA46gJ8vxCAKqNHB+a1wyA7718cAMBAF2f1B3IYEtOoXwNdJN62ipaBDwHAHoiUfYk/0Igy5ZEa7KoQgAwSXVhMv1ax6K1HpuKYsyVygCUeGgwZiA5WgEA6MfcwjKNcCxMKc8yujYCgKHWJ7olTGx1DD8L4gJFQH08SOIemtEMBlgNejjUds0AlBhV7S23F5SbBTBjCkKWSVgPWol6fVoODdNs1JpMDgB0AssKOryRzQK0qBRNteAaAK11qGo//0Ig19Hd5pIXSoJWAYCLhgD0kLK74pQvBABgYc+7gAZT7Hjf1MPoOgHw6f3bNw4AQwEHu7NPCBihlSHCjgAQ2tQFx+RuBQDwOWnerPKoJOEe8k5ak1DjLIdjuZpDtUpkAIpTradO7VAstA4gqFVnjs1VgMtx8zovIQBobbbuxLrQleBndV7NC4CsuqWMLm4AEIyyAKAndc525odcuVkANEJjfY1u2MiKygV0zNqO06wu8tQAqMPZXJIOhAwAVpOrz2pVDgMAhNT5Q9a3A2AoC+tVALptmLPsqcQAUE4BgJGrHMfDDpUXAMJ/8/x1AOAmaBMAXwy2JZLG3SMv/0rAIES5ozFFAUnzceYvtNlGkaXAgii3e5E41fH8UJz0sjs0ilqBPqb6BQGg+10BGBUYA7wJKLYU2PLleIAWoPUH7YE4wdekw/+rMPizRPbPLsDbCgBplvYvXbObzqpFAKAd4BcpNabIM0prEMvTYJQXAO60QYO+B3OyYIrvVejBBIUZQiseTMGX4VLIA4COP5Cn6GB4oeXrNDCm7SRsjBcKxjg7BLkBAMQ+La48oB+MrHE2IhHG4YjWKosRvloUWoRGV7TczTmPrQBQ08t3EwXQ8wQAyK3R1rXcIzeydtoL0Lgw9b5sL4B2sncLKMuf0QZWwMQbhaW85wEAbVsMuGwvAPyWsJxG0k6yoqUhTdncqroNAHoQBJe+m8uCv1BkN+CGtsW3YucBrK0IiiLk2QugzOa6cMYHArP+5qij6oX2AuQT0AQAhJAVcwEANIIgunQvqAE9zxAAFFNBAGjT6kWvayh6IEiQhK1hm9t4TR45EOSyxOLtHgiyLAJerBbl0W8BAPrMG7YG1WMmFwBKiQDgigFgzZ2Lwl9RAPQ7Ei/VNhM6AoBrBICz/WVsStx03woAahLPq8nm+ygJAG4wAJRoQl8BAAQrCqJzO3sJAK4PAKDR2Lo1Wms0tLcBAKC7Qf/8kk4CgBsMgG1PCzkT8H8cACWeLfJmIAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACgLes3zuVsmqfAoDklbUSUmsAfJuUNXIwOwFAs/QFeUsANA92UygtAZBWdlZPWgLg9ay0jTgDgHqwu1QMgMGOVpI1AJyDK9XdJQD4we6eT/4/APDZ53dK6+P10SX3/lbeyutbKysvdmjL85WRD16XN/IVftg++N2dHfVNdqrjx3euQM++zK7rzQ7OeYFI/fN/XUFrXv9hb+/LT3Y282TZcz67c7V6/Siz++wqjD3ZIyIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIlrq4Q9vbeqPZ/Tr03pwRvdO63FevZdbPzmt97frZ5fo0636D3vnE9NE2sfxIlNj2OpmA5qQhcACKqQiRGDBqQg0mB6AFcSiYx2CNXkph1eiJoa0nGwI+KfhzzSh1MMmpkVIOOxBD4QbSROosSZNMx6IKcGLEAwHwsUG952/nT8FOp0Z0BefjzE1rfPNM89MP/N7npnp3N+dS3uQL5GjQv6gyN6VE3twY3capHJBxJs3bz4yPGEY53OV4T8EYxydfCo4qkXc5NMqpFLEMz6PRDzm8ZyhvLz8NZ+/Gd5xXOEzzDBL84pPs4hFjn/43GEY4lHGwF6k283RyOeyiHs89CIKRAxwHDkyvg8CqBpqK+TxluN8AndFTCbQJKIrgcHBQfJvnBcct3j8RvKJ4heS6xS3efxF8oDjJZ+eOCYOQwItIuoSMHLUcBTxSOOw7YyDxMwwI6I0gXoRtQmcEVG8M7kCtID/ex7tgwD+7AH9CgBw5CalOBXOJKU2Keyx4NlhFUCyDt6rxwRHS+GxVHisNQtw8Ek8aseP6vEjffzwz9UEglpBUEoICw2i+ughyhK2VqFqF7aQoSobqsihyh2y8OHqILY24pdMg6KSSlBxCesxQbkmqOW4Oo8t/driHBEwwKcgAX0C9wRcTkBQHHcLYWvpsjhDAtg6XFCcLwppTkBQ8r+anR2+kox3ifzN43Ui5STP+TxO5FEiz4g/j54lULkzF/dBAP3ZxGiQHRgKRo3MKJMdiLKDVWosy45seWNedozMjqPZoTY7GI+P2Inx/A7j/aoD4feknE3KtaQ83IFjSjidlAwVmE5KXlL6VUCXlJyklOwBmPgDAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAODj+HDilgEk9E3NOQUxTOtuY2Sb5IZVsyNW78ptS9pAJSb8nP+TuH2xTXjedUsyknnmGRmWX/M6Z1TEtyj8ivyUF93n7zT35qzb5hJfTMHlKLZpm++OxehV6vundzyCABv96pny+9rI7lt8iN8PSG38yUOGa7JZsck8GGpUd4qWfDESuz0vZ65OZVRHfDZcyFfPFzz4abEJ2xvb5PNaOWfJbYrrA7TYl/jnZOU8reftfddZypkpsF8afMKMxfVWet/ZTPBmooQOF5GNtZ78wT91yMzxPuWcDWmS3ZI4TwIjsEBcngOuY7JRRTgBeSDGBDlYAC7IznJwA5uW3pJ0vgA6r3BhkQiCAKAyphJMngHar8jwLEAAQABAAEICKnI3/WkQD11bNxQsPgQCAAIAADr8Abp5hb8st5f3kyOXiMSAAIAAggMMvAPLBtd3a2uPHj//zKxAAEAAQwE84B9Cf6xC9AwQABAAE8NMJQHej8bapsPo0LYCM8sLmjJz87t9MbZ0ZmpzHA1cbX/5y5z4QABAAEMDhFEBDXbHhU/3MlRxSALrXpcYxTXZPseFFvfmxRqcvNs68LM0d0AEBAAEAARxGAZweLH6VkffRVptNCKD67zM9N0ryzhcPZ+Q12GbSdfrcW0fzGtJsDUAAQABAAIdRADdsD84RL0PaK4QABmZaxomdzmYgL4pt1LYSFcC7Es2xtzOdQABAAEAAh1EArfUF5IatyNVrLmu1WtNRjaaztJj81dx6bbNOf4b46k/rgQCAAIAADqkASvXTlAAGCAH0dOVeztFUl9roX4qu1OlriYpg+h4QABAAEMDhFMBHR1cV8TKsHSKGABW/GmsrcxrMn8i3TrTeBwIAAgACONwCqHpQezNPd85QXEGeBch5Xf/g0sMHZyrz+qtu1Xd+JwGgbhZYsQDCdga3fAFYmQgrqlgAHrY5YRkCgIPs0ogCAaSYIkEAGM4EBmQJANlgG4QqFICLDQqpIIAYk4VjagkAk7TxD1gAmpulpfoyQ21jP30dwIBW3189U1/Q3ZM72P+dBLAef4BQQLEAvrAP/FmWL4BRJmI+plgAqzVM1jckdQFgK+zKwAoE4O5jQupglQQQzGISNxE5AoBH2NUKKxSAlw2yqCCANSar166WAHzsYyXXfggB2Hqo1+nW22Zz0eJZjWYx7YlGk20wPZmuvu4wOxrzNbrjxjcaTd6QYfxABWDR2lZoXCoIYGWKIqxEAN+oCItHBQHM083xQXIEULdGLexFFAmg5XMKKZIE0D5HJToheQIoWqI7BVMsgD46KKaKAOhWeV3qCeAp3Tz7jyAAjpz0+8dEb5VUXbr2/S4FtmhTHT7vJYApxXMAo2nSa7akApCyajsLAMFW/D7lcwDuvvagunMAwaw+j7Svxi4CMK6qMgeAeNPm1JsDWEteNKYsgIkfaA7gB/49ACCA3SoAIIAkFQAQABCAWACffRQeJQJYpiICERUEsEI3xyVLACYvvbSyIYDfQoVY1RsCfFilEmMyBVAzRa+WwklAogLYooPcqgjgK5UVxNQTwCjdvDAQQBIB1ExQTKEqCKDOROFUIgADFRFVYxLQSDfnq6xJwKJeemllk4BMSrt6k4A1dLOW5E4CttDLu5QJgKgAmI29rIoA6FZ14OoJgOmnbSCAJALQ5lJMRFQQQPQ9yb8nlQhgi8r44lZBAB1U1PtVeZOAI9TC28oqACZlQb0KwL9GJVpkCqBok95GHsUVwCgdFFBFAHSrFmLqCaCd3vjLQABJBDDhovAgKgjgC0yBKBFATGKGlDkACVG7zwHg9NIK5wB8KaRImwMISUrcdQ7AKbl/k80BSA+SMAeQSqukzQHAMnoezAGASUBwFgCcBVCVqgqGznTxOcEbY7v9NOg4u0PpToyd7b9w9dhPKgAUPixnARA0gu56fBN9Jk0AMKpMAHTf7p2SylmAPZK+qwBS6fn9EEBnLnOpXW2r6JPTbebdbv7vKc6h/5ExYB6v6jJmH4AAIlbySkwksByAIZiZj4Vjy3gEggO467sIAJlzqiKAgBNWqwKAI/IEEBkZWQpBSNgytb6BQCeXCTJd6Opypg+F0KWRrVjKAghuKRPAexzB7UhwDVGpAlhdUF0A7syQcgF4iJ53QzDu9TqJl1Wy51fRCNHzQaLnR0a2wvsugJZXFMP5YgG8lSaAh0MFF/dPAI6nNBarw2+HnL11WY6oO5SWSR0Ho0VZxl67Z9OxJkkAHX0UuBIBjFIRS5RxkL4pJQIw0c2xIJYtOOUKoIZuyIro+2E3pCQAI50ygWAmCx5BvO3Rzah/HVowraysTNhdWaMr8x82YBw34VIF0BulEhcQp0nWJOAHulNCUIcFmVuAM/2IzAqgnQ6KX2rnHVUggHkqazNANDEUcZNNCkVC0MaST7YAeunmTUEh0zqOYiMdE5sfDCjaMU/0/BIWexklej4M46um4L4L4O0un0gVgKY/r2QfzwIwLFhb7Gi4aCLg8jq2ww5Kyetp6yFrVhRx921JEkD8XgA35iaKBgwPwMT2JLcmJlkA8XsBkJD9ZJQQgAcPUO3dWHWF0JQEwGQtIC7ioBsJ4u5UKgBm4SIYQlx4GKLWA3MjFmM4jJJhmCQBxO8FwP67QazK/PsQGnJaoW9RDwHq8js9vtHPxP/0SxZA/F4ApwkN+MjOjfh8mPTTgPF7ATosUCwGkQJAXRgEb1C39USCgqs49qoAGNYht8eDx2gBYEH7SRiKhIhvOMrbWKncC+CZwskzHMiczwtFAhjijoTsYcoI9pMhTLoAmMjPUCiLOMYvdyxHIrEpGG33Eh2PITGT3YN/2CZW2H+AAshoPlJ51/aiPL3ZYCtsIARQutjl+DRMTgTkl5kct66cJe8XGDT/9aqOEIBubNJm6G4yj18rG7ioed7W2mY0Fhwt0fRXDzpMd5oHf9ekLxrMJv2JHHVuB7b2BqE5P3nktawzAvAWrUegoBdCJ6QIgMdI1v+ou4PQtrE0AMDq5rmUbLqhtDmUJmQ2pK5DJilMp51UHsdu6OLDbtppvFWqqoJsCtP0MA1toQjbpzElU++KkWPD2p7DQIg7DuTQQ+dQdBMYMi6VQQjpYIqM5yIbGR+MLjFx9sluWidtd2zFG3beKbFjWf719OX9f9574WR3UZbdm1Emi4JoItciAE19Ne4Oyc4kmvMonEKDiO6hMHu6DQCaaaJAmgspK4GWRwBNDf7yULAaT8ET3YyJ8nBISc/LcghrXuH328uB6wDwWKZxe9UUHjYIgBegah5tHYDmzj+hyCcoEfdyITmYag2A5gYBULMGAGFdqdIxTvHE8VVPSPFIrYwAmlo2wcn2JAkBELMyhSnhqr0A8KQcbhmAJljK8WxmWwApIqZu0OXQajTGcfJiDqCEWwk5N1sFoKnVASCwXP1NIQAw8BEIQBqg2Qp6IACMD9abH3k4bvUdumkdm1u/OTEwN3L65YDDOe6zfteDzKxY77485Dh7BPlq3TV3xWWFAFydsh77weV4UwO4bPX5lny28fPIkwXX3N8dY5au/qNW+zWn7eanHQMAjVGNa0M3AIhgC7KaRk0AQEkCgx0XWErUuD5AhOi2AShzSVoIJSMhlk9RG4BIzEdynHkANkJV/kNrQ38TAJzgCny6EGkAQGpTNB2pZKsCkxXbBgBV7XlVi+KgNm1kF146USBTcgaYAQAPOHOCFpSiVEYox4rmARBroXmwoaRpFhNVSuAloV0AFElgOQECgFarfADzolQRiBQLTAFQqzEEDcoqm881AGDSVYbBy8E4LyimAaA5jNmQYNLvMdJUJlx1r5LzHHswI4CxlXobggAMjD9EXlh855DzE65eCMDTLuSk3XfVf9Y22YP0zjnOdbsdN5C1n2y2fv9t24/+kUfWHQAsd7uQ+3brpSMP4E/4n9ssXYOH1s8jg39wvu4YAKQSw5sBAGGt4nEVTQCwDe+6ShhI7igZ3IxQJbxtALQEatQAytPw8pXyYhaeQUo2D4BmVzUBNwEAWskYL4s2AEC9TngewYqqUqFI2wAAsqBWPLCn1uRCoeDl6QTGeZgIMDcCcMOzD+WqE4yqhijSLAD2Iky3UDmkqswwH0+oOQG0CwDUh59KGwCkVUqe1oD2De31VM0AQEpqPCahpMQQTJqsAxCvX//cIuwOMdMAgEg8Sy3GYAqwBSMvkVV3EEY+fDAAWK/X21MIwNhXCLJsnetG/OOOP51+6YLf9jyzver9dWH51KlT/7Kc/bNvCeYBZ3y2/rWFQ/DDnb/5FoDvjdvF8u9Z31OYM3Q7LV3+u9ajP8/cP3mkcymAGjRugkK8WgcADcARKp/x8aYASEIAVt1RwHLHv6FBuwDgBIYbAKQX4WvjVJRR9wdAmKhgoVUTAJCVGtgDQBrbIAgiQLYNAA4/Uhgm/Xgt33jzRMmbqr+rmREABIBXcqmpEjyZAmoWAGcCK+Mil4UHIcRwvOIJlU0DIMjZQEDWoEulrC6aAUAkqEw2h4rJWEYPiDsAJPOi5sbNAYDXAcCNvW/i8FecR6s/WnXHG5E/0BrAw3Hfz/CCOB4jiP+aY/b0S99h+Ojn1juzTtu6z+cbs4zfcN0zLtwxW3/vwA/GFgKjOwDYbiDG9sHLn7vu+OFX45aukX8uWAamRr/2dw4A7/Uk7JZY7A0AWRg7nFig9wVAGcN00DYAIJAQQDSUFBIBIOoMynIRksDMAyCIIBzcMFMDyMSiAKWjMRaQGQMA0ejfKHwIbb8GcDwKP3xFfwfATg3d5AgAAsCvEDhABfMpQFWXy6ACExpcQAUUVOVk+wDgWtBIAaQJGr4e3mKbiRMSMAMAKJfixkVOxwl4gCYAJLeEVkOmRwCSBN/TuyK9A+BNVzpgAIxd/686ht4C8EkdgLOXpq4PXTba8xtjdQCWbP0X6gAMTr6tAXyxA8CkccfftnQh/ku//Lpu853rHABocSFfPDE1T/uCsVhMSzsTDDNVwfcFAJm1F0wAQFPUVmwxKWbkkurxglRIyXPulGkAapUSix03A0Cfh0lmWbGGbTMn8mgfl2XLcay4yajt1wD6ZEXdoiBjtYSu68XV/QCwMwIgWU5lmcw+ioB8JU97MaakK+EMk1SDbRcBsW2VY40iYJ+nUsvDFAAIToU0B4AYqf/1IByJiLsACGe5vLJoegSQDFZqRTlGkh4FRr4m/p8AMLaMIP3PbD9efDAxC5//8tvXhxfmYAowOLyTApx5uheAJ9eNJOH0hKVr5tENpP/wqGWocwDAIRgT2+4D4S0YJj0AqhsVhgi3XwPQAhBdggT0pgjQ7UrYBACgmikSSQmIcT1jFNxpQtM42hwAEgHSrK4GSDMA4Cm2yKbwCKsnNQJHvWpGQANb+obU/giAlNiivlnGQWEDNjYV3hTMjwCq8G4Ix/sAmcvobLp9AJIpEMiBagkHdKmAplh9S0Pna3om0HYNQKmpBApW4wAtFOE1g0pHVzRgDoDdE4HiNEnAvG0VHl7MERIVNzsCiGpbukrAoVzJiHxSjG7yBwfA095Gm3kPgIHHnw5+5j70ZG3SdvnUYO9jx6vBP1q/6j7yrVEEvGb9y8PuL1x7Aehesj7vuvA3m6Wra33hZPfaI8u9DgHgzIn1GZ343sm4aFVpE4BdHVXWgBkAduaW4vU5q5FcKpXNo+YAMFJA/MPTilvZFRivh2QnMPX1JXui1AIAiYBAGof6wLRUXBBMANAcKBy0D8Cec2jEZ0+UWkwB3i25IclGAWf3n306sCswGpCqSS5lZgSQ8MLM5sPXHxfKBwCAa7rRfpp5DwDb3D2n7VkPcnF44K/3Htiu3Ee+Xlh/fG1swNaPzLpdL0d9tr0AIMvD1lt3HRCAniGbc2h0YX25MwCUTyQ+soFaNLbImgfgOCWaA2D3jcpgHJMGJgH4L8c9oG3BxXy+8rE1+GQlX+nbBwCtzwRst7UEwMZ7y4JEPYl2GgA2iOUJ0D4AIAIj/7GeSsLnqv9jAJ647W/aiyN35g7D++/WLwjSc/bWl91Hl76/MvHglfEZZkenJ6ZHPxlBes7dnLDfGbo7goy8nptwDn23NDszefs+8uKBsVvoC/clxP/Z5LHxR+6BbqT3H9PDzqfnOlQERHn+I8s58EjTFKy2ARDfX75mAgA0zPMi/vsFAI9Gwx/r93uf+50BEH4vbmh0t/md+L8AIs9HUTMAGNHFW4z8wS8HXruwtlMguHimMeF37f7OJqH+C+f/w975hkTV5XH8jsMdpmnWNjFCmqFpRu0O0x1DndGh+Z+hqJNmOztiIUqUI4E1KUI7GiWYuQqyvhFWWiWwHmWgHrIyHlhoaV+4EtSbB2HxhRs8sbAvFnm2500Le8515s5of7ydc6yd/H1fxBG8336/37nn473n3HPv+88AH14cQKeHI57ADwBq3Y7Pf0oYPgzyFQDwOYIPgyjOMjs+DMJUumrPQGTYLJiJ9wcAAAAAAICsBQA3IAphP383xgEAAAAAgJ0HAP2hMW/JSDfFbsDTf9tHrjMzqQFz+nekHj/mywA49QNxJP9JA+AfxCb/TAPg5I/ELr9PA+C/+6j1r/wUAP5O7PFdGgC/JY9kJhMAp88Q+/xhAwD+8td9jPTHDADMnKH3+2FHvBLMdCWfQpd2J21id2ZIPWYK5RccdPmIIwmOpUwayU1m7I6kiWuIOJ/89H6Mdl8+tS69TgLgRCW5R1dqbjjQSR7J2UMZ583kJWKfysyH1cYr81npkndWtn19id4v1LETAGCd1dFoloWNPIHRR2EiL3/oKUzkdy1Q5WNgkc97JWZSHCtNIJkTTUzqg2TQsVMfm/A+5AcCgUAgEAgEAoFAIBAIBAKBQCAQCAQCgUAgEAgEAoFAIBAIBAKBQCAQCJQFcgUCgcgspUkTMgmoKU0MBcjERLs7w4nz0VKaaCPIZQ+jCutykRn1555NyKSAtp8O4H5y0KfUh31cLL9gy7TiqOYBpnbfsPRmURQLI5QujWFRTNRTmhSFUChtRyhdvMikspXSpKwQubDaKfoImSXMtC51KKLOXEqT3oQohqfoU3Kj/hZtWobnYWsUFYnd3twVfEJ26GF8b83yOp7n/RO0J5aR54203acVkUm1k9LFgvLBH2SjUrkfhTLIqMQNHmRWR+tyDOUVLaU0GcP9tEifUgH+vry9ieF52CwwrDjHjeMABwEAAAAAAAAA9JEB5zaZTIcmUaU85ahlIrpl6nPhQ5/iE8uMW2T38A58ZASPutAEbhLdpuqk/x8DTZzCLaeB5JbIiQ99gcesXfKjue027MEOI8XIbFIyKyJx6ZYO9aG84o24pSa59+6L4UNv4X6qwq0Y6VyLVJ8yPL7aAqjlpr4PWK/4vCBXXEdlZ5DsenGA5zQ4QB2M9I9ooBN/vACNOl6oxC3LYQITTRs+NIpMjGHcqh4munGXvqNgxCwK4dYDktOzWcpHxPmocMtOMvHmtuNDVXigiFJQZRQVdliwQyU280vFGSFxOSHFUYzzCuKWjaSfAi340DgOJY5bLSuEOUn9jWnES/Wp7qWdCnSdwz6FcsWrK6jsmoakmqcDrICR/hG14wGXIR/J/FtpeKMJ30wSytCmUCwkf3enhE35FBCYRHwbTYyjFBV2hjblRTRP6t1k8ppkuWVc3GiSIB0Xm3z4egPlaahRbXIcpbJz+DfZzcNIBwAAAAAAIAAAAAAAAJL1Ai/minjUGP241UJyb1lwDB/qwSbFuJVoIAllEB8q4hNdkFpekjmA8nQ+HtzaTfIpFdNuKR85lEQzRYXVk9jCj29v182Izu36xMZ+sh8gAXWlnJfUT0HShR+pv6VBtp7SFO0cQOykXKT1M6iRyu5AaFOA5TDSPyY9kk5aBmzATTKUG/ChA9LsMm6RrbxY8ZGHpWXAGG5aifPRL0nLgJT5SMuAOeT5bDAbl5YBKYujl5YBV8iLI4UirdYskhcnFU1EWgZ0UNcnHVmjtAyoZ+Ao2bXiAHNm2QT4TQueA/jgxQQ8B/Dp6z54DgAAAAAAAAAAsh8Ag+FwOBigdGmMIxf6vQDIpI12B4cX50O7F6AiiFyqGJV4pRKZUZ/bbciEei/AKDKJv6BPyYR8wmz3AqgYVpzjhnGAVQAABXKWlpZGaLfgaSPIhXaXmVWDTNy0nabG+dA+/VWE81EzqrAuF5nRXthwMWSiOUjbT8gkwOAPtyGAU2K5G7Aol2HFUc0jTO1AIBAIBAKBQCAQCAQCgbJB1j5iZT52YqCwYRyLntzF8H+TzwdiovBgXBxW9aEM6ONZ0oT3Qb9vWOU55BqRS6SuIncZlJcI5ilikZ9fXzGTm9xLrV452slNzKZUKCM5DFSSXIapoPAYTQ26mJfc5FeZq7C3KPo7c2G5wJzDTKPpBasSFn69OwIAXVfzSLVqlv8omJZrSF3WCuWNOZZV4lielaRMRn8hNnn7OnWOmzrXiF2W5dcE2J7kUevVcnLdsv0ZsUe/PbVG2LBMHslyxqNG1rlpYp+5zCfwK0J5rNRvST98EJqm91vN2REA8N7fRaqXGQC4+ZDU5fo1GQCnbhPH8koGwNEnxCZ/3i0D4Mp1Ypfv0wCo2UWthWspAEwTe/SkAfALeSSdmQC4eYPUpvbZBgB8f3EXI/VkAGDuBr3fbQAAAAAAAABgqz6dZtgxSzO9YLUCAAAAAIDsBIDLnOB5wddL/jyq48UEAAAAAADISgAcWRKCHe32hPiU+M/4GOO3GAAAAAAAgC8FgDJhCL9bq9UTJf7ITYkfAAAAAABkJwDKjRbcFUX2zgjHqXttS/URPIacY0vm8vH2GDcwH7lnuZWra+yynOhGZXYctVkemPSctazeNJ9jHjnMNZ4Ulo5qAQAAAABAFgKgNB5+kDvLcXjguOuMwU5PvILjDkwag6FiFd/AXYlHxUIh5PUE/UKVjnMPCZXVnsJmTm8T7vh9HqPXUJUwxnMcAAAAAAAgCwGgu+cXomdLytQGbraq2KZx9kavuPtKBLPb/cCDAcDvnTANGj0D7sb4MZfeK3hNrnLRp9Xb+FCFuzGhKlB3eaaOGAAAAAAAQDauAlhHcqJG3ni2mXOpgk6O66sqLt9zJaThuO69EgDGOa7Z2MJx+tfxQ7GoD5+QXXwFugJ4auW67YlxmAMAAAAAshcAnFUbu1BSLURLCwTP5NDQkMpY/0h1F0dehQEgqDlu3NjF4dc8FVxI+PeiXynk7+ltxehWYdaWaM0iAFxMq5YcAApdPg2ADJOLxACoVWaiAABKnbYEgMLifBoASqPZCgAKi6wYALUKs/scAHxO5bdlEnBM+qqFq864OMyLQ5Lmx8N23Me3MAD8DgyAqnUANPvDk9KvHNXbPBMcd7DriwBg/8u3smquEwNg//3LKU0vEANg/6u0y3NiAKzJJpd/IgbAn9Imbx9TAaAn7VRzkQIA59MlvrxADIC8tMltGgC8VFZkxQBYmJb97j9kA4DajMqvfQ0ATArSpjXrCaFD4+lUNzU1FTQ4c4O/wTcDS+8BYCXc4sK/csH5RQFwIyzI8i8QA+D5nCqlYA8xADJcfD3EAHgjm6guEwOgplA2ebefCgDT6XDeHKcAwMObsk9hHjEAVtPRrNIA4KqyIisGwE8+2e/mc0ZXAP3pGK9+DQC087sf4WW/oeL5I52JCivXNBgt7z7nGeCsF+LvAcAR8l9AZLCHKzIAcMszst0AEPlwYVK+5zQAeNe/rsvPSQFQi1zeJF0orgDeqOaSJv09NAD4d9Kk5jElAIIppzWaK4CH11Q/p0q8QAGA5VRx8ugAoKjIygEQVK0m/e7vZwYAX6ryL78GAFwWQWzzWlTFdoehV4wvHr0rTMa4ibD/1GBceA8AXK8nujh/VzhWlAGAp8Kde47tBsDb8ynV0gCg5nhStTRXAGtJE4o5gDeqq48VhLIlAM6nQtlFCYA5RU4KAPBKSYm3AsDPiqLZGgBykS8yAsCC0iopB8BNxZXflklAtVfF88ZoBzrRiqZURl6YDKBqNteJYXMLX5oCQEcSANqpKPrtuytcBgA0Ib4ysN0AWFNScAUAUDJ3s+UVwEslsWwFgCfHFZhsCYDHbFYBEAAUOSkAwH1F02NbAOCdkuIoAYASn88CAOtVgH7VrxVXfntWAfQ6V0NMJ72gxaozDTtRzxxsbTis02nbPBpOp7NynKEIv9lkVodGl7XIHXDinw7in9b/1brVhh0DgF0AAADANwWAD6jof+zdT0gb2xoA8PGGSCNabTYiJrTVMR3pje/R1jbFGyedaanM7TxtuUGRIj4krymBEm9DISSRpKBW+qDgxoWYG7qwGALxkb6Z3E1AmE2RgtnN5i1CIdkmEswmi3cm/9SYVGfO+Ofem29jqeOXOWcyvzlzzpfJovkeormjf9yOnEOcKgDQtwDUgVsAEuoW4CQjv2MB8F68W4CTjLrP7BbAckq3ACQlGwDnfAtQe3HQpEWtSnS5/8IAwO6UIgoFQGMV4NhVgC0SDoCLuQrgkQmAP90qQJ2HhNyZ//UfPedz/tcGQKktRRoKANRQDNoBMwI4UZbjACgnMQRgACgnyXthAShnckICUG4XkYQAoNI5kABU8thlAqCcMCcjAJWevzgAnGvUBMBFF4MfgQLA7i+GzwszAgicJMtxAOQzpSwxGADKSaI2SAD4SiYKDoB0KY+/EwKAjvLecHAAnKiTxQCwVu4lRjYAgidq618dgHisFExjFaAxCSh+EpBqTAL+sQForAI0AGisApzGx4FnS3G/+qGA/YP3632/c/vs/jYbGsVd918IgMYIoAHAnwmAKUxZDONYNQ1PzfXKe15ipfOsZd68MjhNdDVGAA0AGgD8MQHQWnsK8ar6bO/bPhkA3T2L104fgHClFLjQXVTtRdNjAaikKfw5xVhsUkYAI4eyMBZGCgDOzoMNIi0WSQBEykmo4q6QkgGIHOrfOplOAEDAW+kckAXkIyUAkD2wN8x++0QDsHWgk8nCTxscAL79ivT6rRMHwEJVW+unPB0AtutVA50QAGSjpf/0AbDmylF4C8cSdosUABacxWB9xSxsRsoIoKOcRbgeUAF2zyYBAL6UxFl4iSSbYKQAsFVKEhC6wxJnR6QCYCjvTlxoTKedVUsEoNzFTr+QhWUT9oxNNAB4OcmeyrYLkrDpXUYKAHQ5Dxi0ZQp5wjEoAPKlfJ6YiknXa504AIhKz6vIYkoHeS4AXH27udS2+Grq9Z3txc1ZAEDo38OrPRPC8wIUb3snb94Dx91969Xk/NgQAEDT/GJ1+/OAeeXS180fkYl3S1/W158o+pHWB8OTT9+8eXUJufR++0rv7+39sgCwH4VCgLDWlJQCwOFCoB0UTZASRgCHCoE4Hq25KHziQiDhNPKyKOqQAkClEEj4rY838IzUOoBDhUAO2pAlpQFwoBAolguy7FaQT1JiAThQCESmCSfr7CDsNgkAHKy22iXyCTaLOyMwABwoBPLyfIJNBb9TDCK6EIgJ0gnQYXTyXAB4PWBEQzTm6tEPmbW9AABM7/rFZHrUj7Ss69BvIdOHDWS5SfeSMBkBAA9eYvSQUV+aA3howk1DJqznBnKZxh7jRpdSoXlitK66sJlBeAA6w4H9EK7ZJE+Yat3efR8Ay9pIOXaF64AtlcvTEbEjAO/OoSzUCM/iatEAJCtJRvzCSUU7g2lSLADR/SRJ4cQPo+majBwPgP9AJvD+t9nxBO2XAACz38UjHAAgzahsDtxOigPAsZ8kAwAIgmPEZbMR0QAc7uRdHvSNpUa7TgwAt1vJp/YCAFjhgR6EBwqAzP4+OgAATpAyQyTOdA5guxDvAADab9cVL5Shvysuu/SDfdva5+Pdt1HDg9YP2MDd7v/+Glq5Ool9Hbz/UItpWuexd+2KJ1gZACX6YHDUYBrf+Fn3dfDuO61S4cb1XTdmp63/gwfgaM/qA1smTiwA1cc9ysfV6I5YAKpxcqY43mkTC0D1xgZfolZ1qahnAjK5lG8uzUgBoJpKns3wAZt4AKr2XgBA5afDNnEAHIoiAJFUyiIagMNRAID0LEQlA1D1NhEAUPmIMBQAh4+gAIAqigfOEoBSXAEAmN73I1PY6mvEPWPs6ts2vulH3G3YZ8WqdQls+la5+ZP1Obig/xjCNFcNLvAvxccKAE/AFtvKlWbrajeC3GhSKq7ixt/uI13Lg/IDQKb1mSQWhwTAFuYzXIeTgQPAh4dVLBGBA4DJ5Rm1YQ0SAD8+YmE7OBkAcKBJSz4bgQdgy+FY28pzKigAeLUj6QkmVbAA0PGMIx4MM7IBkM+A1mU5OQHIgpTOXPQsAZgZLMQzAIALXK1HjeBV3E+N433brstgg/e6NjCqJ4aGhlBl7239TWFK4DmmUWi/CZOAPWUAMGEVcVj5rwnTQ6EqYB3cAjwKYforn65r5AcgZp2zxfAaZ50oADrzKYa01zgwogAIEJzKYRiBAyBDj1AWOkVBAUDawcUtWWtIIxYAJpGNqfZQBzwAxNxc0LCghhsBoMG5II7HO2EBMPBzczSa4GQDwFBoXZKUEYBCyrk15nzmAKwPBAB6KgD8BP53TPfiFqpfnBFic8w0LBy4aUwzq1wUPjQ0XAZAeEYwGAcsvzcWABhQKpCWiXmrUkufwi2AWmvnOKdRDQeAzxCmwC3c0XVrMQBYclsWiss5LTAAkIE5H0UlDDEoAGJZJ0N5iRQ8ANEFD0lxRBoeAGfGn1HncD/cCCDp9zsSRNwGOwII+/2ZOJqyyTYCAPnWsnxGzhEASKnO1plYPGsAQiugXz9hX2c/4uNut/vu7euXXeuvwW9xTLPhIjaEP6kGYDQkbLFBKxUto7fcfUsDyleyA0DmtDqjUaf9DwkFgKfJQBCGpg4SBoAkjhIEgfJ+GABiHYUkNWpoxACQRIUsTQQHDcBIMVMwJsscgMp/9AOPoucAVMxcjpFhDkBFJVBOzjkAVQavN1kidQ5A5aft5EUAAHu3gbRPhqb6enUfEKT1b/pPfbge9PmyDtO4p3UTCHLZXA1A92PTFKK5gykV7SH8BuhO5bzsAETNC2tqtZrQxWAA6MxlwyCcaBQCANLOB0ASOxqHAcCBpkGS+NyWFwaABB0HWdLEHjQAeV7omQS+Jg8AUTkAoBY6ZAFA5ZEZAJ/8AET5swQg9LwYj55VA6DFfvj0UQuG9M1mbc/v0zq6Gblndm2+MGsxDbJEhB5+acKqAUAmzMb1VSMAwD2Dfdx8QpjuyQ2AzV4c/O9iaQoCADW9U9LWJh2AWJAV9sG2ECSlA0CxfOEtHsAzEABwfGEdMZZNReAAoLhCaYKKW2AtsAAkIt5IlDU44ADgvJ1cuEYhgFgAkt7OmAPvoGQDwOuN+Nj6j4WRAMAWSBlN4MkzuwVYNupK0fbsqWEcQVZCYMzunjd39f1gvRnS6YeFrw2/N6nDdJOjrcjGZytmXP9m0iCtY6hO98tA09LgzFAz0qYX3mNtuhXk6ti0mf4nrexDunrBHxG/9ckNgAV3FeaDOCsRkw4AA95Xws9IKhuTDkCy9NybAOqXDkAsWKwC9DcFIADYodVFHo/ecYsEIFyEyMbSHCwAfCqVX8A9DBQAhnwqn8NTHOwkIJFLpbL0gkO2SUAatG6OqFmTKhUAIeUC4bGcGQDfj75r5c8IDjaXHvzZ0lVe19u4VqPK79KbKXBonzWZhK27x9slPC30OAAiW8VBLrnHRqUDYInvFa8EySPfESACALW9eFpydodkAKiovXjKUva4dADItXixwVFPBg4AZjdcfCm/xwcHgDcufD/AXo1HjIgBQC0kCdcqjxUHgN8OEsV3jp5bEgFgAvVaJxkAWzGlz6a6IACIj0u4a/SG4gv2s/TPBzS+HFTaHED9aHw56PcF/j975/PSSNLG8TZNh5DJmN3TEJJgJmrSwV/DaCcrGmMk4sFEoygTRCRDYCZe5I1ZAi/JOwfBUZmA4MXDsPjiwUERXl/WgZ37XmTw8N7m4mn/gr3sXt+qnvSP+Dpjz1P1uqPzfA9LsdLfeeqprk+6qrqrvu3DQTlrOy53HEu/MXwhjABAACAAbi0AOmNd+cJJgOEDwZF/PoDqXyYAvPgF6vLrBx0A7X+AY/mbAYC/g01+MQHgV7DLnwYA/vOAWb8faQD4B9jjJwMAL+GRHJkB8OJHqM3P/24CwJ8/P+Ckn8wA+JHd749vY0uwk8x3UH3Y1MFTWnwJdXk5NqjD6AM4lsy2ZpKFm7xo12ZcBvMv4aHoL2M9hYdixLTQAMB4AuxxNKLtIpVchUfyvek5M7JxBPZJ9Jtuv2eJ73jpKGVss7VwxO734f43AQDFDpfx4UFtGO4ScGkuIkMsXn1eJAA3iWqPNLVBhvoc8sitrlKDsmEGD1EjdRtDO5XMe9Gx5Me8UuUM2LlJjPAIz+QnoFAoFAqFQqFQKBQKhUKhUCgUCoVCoVAoFAqFQqFQKBQKhUKhUCgUCoVC3SY56BEGLkaTGjVxMppEwsSkzHosaicNpcajPp2cMuzyEjMHq0uZmIRZ26mN1uuQvUo+6uPgeYIt14xzt7vDctlCoVD1CaNLticUiqdZu+4YMXndzegyQupTXGI06S+SUFKcUvyWmq2zumyQemVaGU3W4qHQ6B57lUqjpEpBnh1saZ5jxgVhiwaY8mH/vh6Vk5Ikje4yulT8kuR/zvos0kNMJqKMLu2kPj1ZRpOBURLKOqcUT8eJ2SSryxCpV/Exo8k2bacz9ip56KHSvbMc78OczDHjgrBDA1x3Yf9GACAAEACoK+UUiaIbFAAxWvRCxnQuhV56Tm+sFVoSQcNvB71ymPa6oSQtgkbMbeq/X6AAWKMlBfIE6FPrU6d9th1eH21Ww0sdctRsQzU7BLeTmCH1qi7R0iy8nc5oO/WpyYHWS/WZpv3rIECLrDM/jYxXZC4ZJzlX7bI0wPduWmzDnv4Jvbq3uLg4RX52JTlDzzFZLwNMAgV66Ri9sarqaSigH6k+euWqTFm0TItpyE3QP0cvPab1SdCSDfIwMWijlybo7Tiv1meHIcNKkDpkaHJ6VDPQk8mpeimhiBQaoqUU5Je3NU8v7aChFGlpMgmsUy+9eoL4SMerpDQ3wDoVGG1XM+7XM842gVM+0HIuqXZz/djTP6GnH08xp6xUCwnI/Nvj+WYTKQcJZUM3UV2kdshP5Z58qT6QfZSfJJpN/HWWeztzKTmgedKR5uT4f4NsY7XT0xzKcQxYp+PLVWKdabO3XHKsM9kpo5fsTrGnfxIAUpOAAGg2gQKg2QQIgEv1gQGg2YQVAM1uQAA0mwAB0GwCBsAlHx4AuOTICoBLdggABAACAAGA+h+Nj9IDDOnd5VePMlz0QsaWY/RS2vX8Mi3FQWPmmVBTKPEVyBzAq9HmUPaHIXMa+6oJDeVjfbIMGRbn1HoZyQHdjCfNyQkdKACTi2JzvYrTwDoltCo1muqcdQ5geOLSHZRlA8APml2jAQewp39Ch/Rtqe68ugpAi7DZZfqam7JHp3BGFPBLeOrriIPqKsAT8Dtc6mtuiroKUKGlMGgVgL6OqKypqwAK60uFkVnqEKNmeQX8Ep5TvZSuAhR3wG/guVSTNG2nTVryQuul5mdXXQUocXmv8GPGX8lcMk5XXqhHTn2OFGkRVwE+K3wP4CrhewCfF74HgABAACAAEAB3AAAHFABbHAAgsZ6y5KBL+MusALBRAOQYTbJ0JinIKcW71OyA1WWZAiDJaDJO2klOs1fJrr5nU+Z4H8ZCHDMuCM9ogEEEgAV1J5NJD+tMrrOVuIQZTSIBYlJiDUUhJq2s4z6nh7gonDLcZidmzEdQRYlJgPUrx05i8sTBXiUfbe9unrch14wLwiFfOxQKhUKhUCgUCoVCoVAoFAqFQqFQKBQKhUKhUCgUCoVCoVAoFAqFQqFQqJtUpzsKldv4dMQnRuE2+iYWDgYTfQvqGrxCUX33c5/CEIqLQ24NKY2YnDwaysXQTqL5WyyG/HSbP8eqdUd5yV3mEt5Vt/dd1vk9uMb1ruvuhbssuDWXM4ZY1jSTpQ24SUrb+cwdhJts6Iepjd/joJlGf6kweJxpnwp6CgyBmLdRszG0t3mHibcL97gpbZBlhoffu28CACMndqjeBPXfhMDcLtRla1HfoHNmGxxLX59mUp8Bm+zc01gUWLwAu2zou813pe3Myk01nm2epsAej3q1rjH9GzySfdOWI5GpAajNw4J5J75Y3mPnpHq78ZO9OsDut237NgCw7YCqbgLAwjDUpbSgA8C2Bo4lrQPgURBs8tAAwFwU7DJpAGDPwaytOQ0AJ2CPrAGA1/BIpswAmNuB2sy2NwGgEHZwUs4EgMVn7H4VBAACAAGAAOApl+rsdF69e0vksJP0KB8ZWLc5XAgABAAC4K4BIKYeRyDPB6/cadN70PJEEMcfC66u+DQCAAGAALh7AKgWCgf5oj8TuOKvs7YJj5A+jgmu59UtBAACAAFw9wCg7m4cfe1/etUJDk5HREiNxugQwIcAQAAgAO4oACJ7sq0z4j5db08nyWC/FhsprFREwflqWxxYlmfWHLGzkhAR68H3Z8maIOy885yvrz8qIwAQAAiAOwGAcTl46Fn0/zAUmh+IROpydarq7y3TOYCVY38xqNA5APuCv2MiNF+hDwVD8cSo3FVGACAAEAC3GwDv3e5oKZYI7c3OhFbsg6fzQx5loRhzJ18XdygAxGC8LtYIADrXQ12ewUo1kRRSUnVpOFc8bkUAIAAQALcbAD2ZTKYqy73l4eMhkQz2R/yn4uL8uehL5tzqKgCdA6CrAO75MXI31u7L25GURE9s6pV2EQAIAATALQfAxMTERrCiCDv+Gfp/BuSU70083lGoD0eaALDlL9C/Z0MrrpScpT1Vena7AFBWdHnLYAB4DZdZMADChkkYDABzfdgAMGvR6VoAhK0l5/MAmLWWnOsBYK2prAPAamDWAFA2hVf+i+cAqPqlLvX/yEGhVm+v+qX9t00A2JXUf38pbqulQjFSSt0YALbGDWUVMACm+55rejMIBcBwWjfZvAADYE83eT4ABsDSfd3kXTcTAHJGOOdhBgC4jeTcnwYDoGJEU2cBwPCZ0VRveQDAFNgjdgC0nuhuJ61fAQBa5XyNJPJc3uz02MvR2IacEs0AGJa/p38/lVOumwZAOmQoXwID4HSsRdPGQygAHu/rJpksGAAF3aSlDwyA8Q7dZLLEBIBNI5xehQEAgUXdp6MCBkCXEU2QBQBvTU2V4wEAq4FZA0D/hO62v/MVACCaqJIxvZgPDSSX50r01OmgWwVAPKsCQMkc7wiCchBai9w0AM4keSzTUDDKAoCu1Ee9i0IBQO6qQsOkL8kAgKmRhksWDIDtjhbNZFxkBEBCc9oLswEg3/B5vgUGQLBlQoumwgKAC2tN9SUAGNICW+MAgKGW9obbiecrAIDrdLTl3dqCvOL1vve/3qsvj1bUIUDavzpOlwF9az3FdCUvBxXh5gFQnHY3JJZZAPDQwkj3WgDsWRm5XQcAW7eFsfK1AIjymQPYbNmPWh3cXgOAMyvJuQ4Ak24ecwAX1prqSwCQd/ObAyAAyFqYkvo/A8BvHHNe3iYD//i6IggPF2TJP58uhwsdrYKnQ0q0rsR3Bcde0S+FbN2CBgD/9I0BoOUxh0lAAoAA8yoAAcAp+yoAAYBiweRaAIh8VgE2W1YtOVkAQHqWeRWAAMBKcqwAwMpQ/YsAoDis63oA5P7yVQBXp2k7JJ/TfiGq3wW2uXftTp8QOST/ERwB0ddGCz5nYMtN/15z0m8Da50+BAACAAFwmwFwO/YD4AcAj4VHQutDgFkWAHQzLwMSAAxqoXAbAngZAXAmMi8D/pe9O2hJJdoDAD41KEOl0lJUmtR0orRHOqVMNipFC52riaKIiBfhmptAB9zkKhANg8CNCxcXXCiBCxe26COE9BFc9UXeGXXKyu5t5syz7sP/IgLH4/HMzM+Zc878z8QtgEaqW4CQRACoPttMnwLA/uXDgP8KAB3/5SgKOSgA4EcB7r7PKMDN/+kogF6iUYDBNx8FoL7BKMC/AoBMphuFqQYFgMM7ipQcAgC+ECoGAYBxXIi3AgMAX5UWNAB8dfqQAPDfi4YB4LlxtiAB4NvnRCIAnismDQB89X7MAfgLACYqP4pVDxQA9uQo6iQEAJVxITUnBAAt27gUJQwA++NCrkhIAE74kgYsHACH43KSBxAAxPnaeCAB+MyuEgLA6qcq9mkA2uPSAqo5AH8ZBtznU6OG5p2A807AeSfgvBNwDsAcgDkA0LGUG4dz6c0rm6Rz7YM3qXIv29xq0b3iHIA5AHMA/kkAkoRsFK7gWxpOI54P3pQlxufZUilSJ1Ne9RyAOQBzAP5NAHRn+mGcvz3bN/qfA0CjL+Mz6AOoPa+QBvZPCE8m181iAPDwy8END9PoNS4GgOPnlfswsv6oODreDQkHwHDAr3CHYdu29tFhTykCADn/fUZdd2jQrhIHQBN/qQ4a7HFR2BMDQNfJLyE4bEvbzbTnFP4GQOqlNubksDK9S1wMABO7Ctver2x1bSQUAOWJZjIrj48ekigMAFV+1b/hZw4eh5Vj8cuMf5+cFQD9D175LADI7dLmDEYB8qlxtHYx1pYlTAuFkAgA0qejUOQwFr07z9fEABAfF6KvY54m3Vg2ThvG/RsAJ/1xKdcYVqWZVMq4qhQOgHVcSGV03t8wKbU4ALx8dboaLJc2MhRFxT1iAEgYRuUsJsGhPDh1hFNWp2AAGL429xh5RHHhzSbFADCxq0gF7dMv048kDADPFWuDo7BxYl1m2iQEAOVxaVugqffu+vRwH6uttC9MH5OzBmDtsnulKJ8nWXu/3M0BADrXhw197Ba8hF4a4lv7YL8Xd8/jpSAFANDKM41+zxqpY4/dbSRWuXpotfzoJrLiOYyfFgrnGIJV+6uGe9WmBAD4ZS/hSmK5sDdWK9N1EQBMTARSZVbppigAJiYCqbsBHL+k9KhgACYnAsVuPAcH916/cABeTQTSBLO0WAAmJwLl0taBWq2Wk2IAmJwIhKfCBfkBHhIMwMR8GzYHqqLeLT0NxAAwMRHIdpLBt6OGhgcGgImK7aWWA071IhOAAGByIlB61TsC4IG+znlK4fqsAWCtLmOHIS70FiqiMwAACMtF2O0+3kSWWibjU8d9c4vUFkxZr9sFAPBkCYZyWcZ9AEdu2k25Cb0ZUTLECe26kKFav+uscUGkSQkACHRfwo9jNt0Dhu1bHkICARg8PqcVKagwMlndEnEFkOs9F/J7fLLh6RYuEIDqS4YT/petdqIQCED9pZAqJ1AyfNgXB0DypSR7CACwuCeuD2D7+qUgcKI90lUxfQCxl0L4m+QapdAIBuD1rqoy9yGMzSSuxAPwqmKDhZ9cy9E7YgGQv1SvB46fYNUwBECVSoNLphj9yM6oD6A/jAoAQPe0jmZknf+gygsLudHXJaIam9HhWbkhrHuau3ynvhYnHknnkY7QrpSIigr1EzwAMqOHDDjc0Vuf6ZHcq+hkaJG2qM251NlAAgDenls6oCMZSZMCAXgX9yIAmBKexqJKIABvQ6P+pc8GBALwOlhPw5DbEgfAmwZIp21XUVQEAG84SPts+/Zpc3CE5gQkS1ml8E7A16GMZ6v4ru99PgmROQGTC/dcqcsGViQA7yIzBCCaOAIHtpxRhGYDwDhWAQDu6iaSJBosUky71Bt9V2GTywHaQxtn3LJAl7LuwVkC/KBvdwjtmuMC/Ic2nwHwgy36srr8rKFBEPOCDF2jXb+diLpGSg9Axs1dxnmf0G8CgN/xXmuBAMh9dP4SgwIAP/XtmiUBwJl20PRyBocFQF6mssYF+hyHBiBGdTWwALD2PJ1u+XYxiQCQe/uAfXX54+3FAVBv/gTfVUXpzbMBID1+WgoAcAF+rQMu8CnFU1d0o3/BZf2umhTgqt5LUZRRZrBZtrgugQShRXVPXCegngeAzxIacx9xswJa4BbguENYVtvrWukBOOxEwV/qx/cAgLxkTlEMEgBnOxNfvoIBgNwKD1hpAECr9v2C3qs3QwJwlfAFceUOvUNCApDrl6MYLADb/h8/S9mTICkRAOQO3apk4o4SJi0AtWwX1AClSpqZ9wGceTgA9M8AHCBcEuDMrtFSTnPRDboPuR2XIrQ5WZnLCHDIAzBOEVSruoYAWGUoshQrncl0zP/gFuDn8ArA0fgetwCFbNqDwQIAYj9/SEIAsJ4tK3YUTyeHdmgARiRZf+CQAAzCGfCN9sI+FSQAVUebhQWALVBtMxlLZWsSAYDljltP6SNaITEAg3wFnPpO6tz81QB06qBd28RjrklHi8Xinm1dedFiwas0ob298N5yb3kLQKDDbXHLyNClwG5x48oqO5cegJ4O7ERV5/Q7AIA+UotTP0kwAPJWPwcBwIGe68o5YVrH0gBAHjXvIAE4SJW4SvveV10YAHg5NbWJBQFAZriudrZn/C0VAMMs4QVjVWIA8KctcEE5YLqhrwaAqNwiqngnuWEw3SDIyo6lvUFbQJvXTIS2mDLFEEQZeQuA5sSdRLR2QoaqOrT5Vc5B6QAImMChF3T72a8HgPQzW9OfLBYAAItzV3u/mnqYK4AQlzDRqU8NUGgAtrkc485UwgkJALmYB4bgeQMKBUDokZmeq1wQAOYuxaVv6DkKkgHAna2+ck5iAEjDkxxjfzPVGQ0DdhKjOA69BUBHLLabOnBJL4/o9PcpEyNH9iMX3UxER2iRK2/n6GGBeAsAEou4Wg0XAKCYJppdv5dbWlhqALYNnXaPoTwYFADcMCD1UI1CAVCgs10/iAEMAOh52V84ztM2qE7A4VEuSR+A/aRyeZ32HsN2AmJ1KlWoWrNBFgqAaCI9/bsL6wP4lV8u1HoJq1oqADy9qj+xHGMlAsBeNXiv7eDXJMgYCu1mH58JADUXn29fETp1RBGk3gHX7MVSRL2xeLbVMVkOt7lGiZsIUzywgtz2zghX68mtRVaCRpMpbF24ItOUHFFYuGNMYaoja8FUhPnJyDYQtQG8yft7Q3oA2Luy2+0NhuAA2CvRXPSgALikhoVMyX4hAADyepX20vHrEDQAofP0OjwA9RaoTvajzOkCACALYYZJFEi4UYCbj3KLCAMgZC9TCcrwPt+BWAD2mzR1mvzD1sIAyHPHEWUHvwc3WZr21dmZAPDn2MD5ZwRJOTo6t5bU/LjeLT5llh9WSIJdG1pwc1troioRaUM/szQYm1ufNk3tn10bMJS7m/Zsw5etDej0KD+c4SpobUBSrZ6WPe9r1gbUqK9yU57aEAsAGY3++dEg0WsDksqD/7Z39yoNQ2EYx6FBKVIindtB/KCVFrfi4sfiZqgfCIUgEhDs3sHF7hUidOzQQeigi4NDHbyEIr0Kb8SlOTlCEXnfAyX0/7uAh5cmfRoazjnzniuysB9A/vjhrOJNqqfy9QEcDrr4Avjz5R6Hg/5TFg4Hda5/WN1+W/lWrBCmACgACiCzBbCzX4tu3zcUCwSPHn2pwCqA6ztpyqBjCuDiVTxL2xTA05Y45CYtgOZAnDJMC2Diq513kgI4EWeM0wK4kk8ysgugGUtjWi+/CqDX8h0ZWwUwivV5n8uxJdhzUSycmuJZ/5LH9OpJSqCY5d68aYnkKe3KLKRek4dEfjLKNCzq5WZf3liRESS7SK3uKQY5sG6bTXnOcGDlfPSKroRBuqdWzkVgfykKYM2TS185lBuKGFMjeUWI2WOtqwi5TEYplxQpBfN85jlQ2tVfKPPTWFBcp4b9F7Pm87H3tOt67uTdjDfn9gYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIvxA2vpz35XAE/FAAAAAElFTkSuQmCC\n", + "text/plain": [ + "<IPython.core.display.Image object>" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from IPython.display import Image\n", + "Image(filename='./samples/BERT-embeddings-2.png') " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## summary\n", + "\n", + "- bert input embedding：一种查表操作（lookup table）\n", + " - 查表\n", + " - token embeddings：30522*768\n", + " - segment embeddings：2*768\n", + " - position embeddings: 512*768\n", + " - 后处理\n", + " - layer norm\n", + " - dropout" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-21T11:59:36.568959Z", + "start_time": "2022-08-21T11:59:30.810157Z" + } + }, + "outputs": [], + "source": [ + "from transformers import BertTokenizer, BertModel\n", + "import torch \n", + "from torch import nn" + ] + }, + { + "cell_type": "code", + "execution_count": 58, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-21T12:22:27.942035Z", + "start_time": "2022-08-21T12:22:17.702981Z" + } + }, + "outputs": [ + { + "ename": "TypeError", + "evalue": "__init__() got an unexpected keyword argument 'output_attention'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m<ipython-input-58-126c1025e2fc>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0mmodel_name\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'bert-base-uncased'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0mtokenizer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mBertTokenizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_pretrained\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mBertModel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_pretrained\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutput_attention\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/transformers/modeling_utils.py\u001b[0m in \u001b[0;36mfrom_pretrained\u001b[0;34m(cls, pretrained_model_name_or_path, *model_args, **kwargs)\u001b[0m\n\u001b[1;32m 1383\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1384\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mno_init_weights\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_enable\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0m_fast_init\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1385\u001b[0;31m \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcls\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mmodel_args\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mmodel_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1386\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1387\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mfrom_pt\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mTypeError\u001b[0m: __init__() got an unexpected keyword argument 'output_attention'" + ] + } + ], + "source": [ + "model_name = 'bert-base-uncased'\n", + "tokenizer = BertTokenizer.from_pretrained(model_name)\n", + "model = BertModel.from_pretrained(model_name, output_attention=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-21T11:59:47.139396Z", + "start_time": "2022-08-21T11:59:47.136812Z" + } + }, + "outputs": [], + "source": [ + "test_sent = 'this is a test sentence'" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-21T11:59:47.144154Z", + "start_time": "2022-08-21T11:59:47.141516Z" + } + }, + "outputs": [], + "source": [ + "input = tokenizer(test_sent, return_tensors='pt')" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "ExecuteTime": { + "end_time": "2022-07-03T04:30:53.366612Z", + "start_time": "2022-07-03T04:30:53.361672Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'input_ids': tensor([[ 101, 2023, 2003, 1037, 3231, 6251, 102]]), 'token_type_ids': tensor([[0, 0, 0, 0, 0, 0, 0]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1]])}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "input" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-21T11:59:50.692924Z", + "start_time": "2022-08-21T11:59:50.690305Z" + } + }, + "outputs": [], + "source": [ + "input_ids = input['input_ids']\n", + "token_type_ids = input['token_type_ids']" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-21T11:59:52.000673Z", + "start_time": "2022-08-21T11:59:51.989681Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "torch.Size([1, 7])" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "input_ids.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-21T11:59:54.049576Z", + "start_time": "2022-08-21T11:59:54.047164Z" + } + }, + "outputs": [], + "source": [ + "pos_ids = torch.arange(input_ids.shape[1])" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-21T11:59:55.089418Z", + "start_time": "2022-08-21T11:59:55.085220Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([0, 1, 2, 3, 4, 5, 6])" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pos_ids" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. token embedding" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-21T12:00:18.355440Z", + "start_time": "2022-08-21T12:00:18.348069Z" + } + }, + "outputs": [], + "source": [ + "token_embed = model.embeddings.word_embeddings(input_ids)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-21T12:00:19.563400Z", + "start_time": "2022-08-21T12:00:19.559701Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "torch.Size([1, 7, 768])" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "token_embed.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-21T12:00:20.702998Z", + "start_time": "2022-08-21T12:00:20.668551Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[[ 1.3630e-02, -2.6490e-02, -2.3503e-02, ..., 8.6805e-03,\n", + " 7.1340e-03, 1.5147e-02],\n", + " [-5.7095e-02, 1.5283e-02, -4.6868e-03, ..., -3.2484e-03,\n", + " 9.7317e-05, 9.4175e-03],\n", + " [-3.6044e-02, -2.4606e-02, -2.5735e-02, ..., 3.3691e-03,\n", + " -1.8300e-03, 2.6855e-02],\n", + " ...,\n", + " [ 2.3670e-02, -6.1351e-02, -2.9575e-02, ..., -1.0239e-02,\n", + " -7.2316e-03, -1.1745e-01],\n", + " [ 3.2079e-02, 6.3135e-03, -6.4352e-03, ..., -1.1689e-03,\n", + " -1.0810e-01, -8.9524e-02],\n", + " [-1.4521e-02, -9.9615e-03, 6.0263e-03, ..., -2.5035e-02,\n", + " 4.6379e-03, -1.5378e-03]]], grad_fn=<EmbeddingBackward0>)" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "token_embed" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. segment type embedding" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-21T12:00:22.656519Z", + "start_time": "2022-08-21T12:00:22.654023Z" + } + }, + "outputs": [], + "source": [ + "seg_embed = model.embeddings.token_type_embeddings(token_type_ids)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-21T12:00:23.860761Z", + "start_time": "2022-08-21T12:00:23.857076Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "torch.Size([1, 7, 768])" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "seg_embed.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-21T12:00:26.633429Z", + "start_time": "2022-08-21T12:00:26.627699Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[[ 0.0004, 0.0110, 0.0037, ..., -0.0066, -0.0034, -0.0086],\n", + " [ 0.0004, 0.0110, 0.0037, ..., -0.0066, -0.0034, -0.0086],\n", + " [ 0.0004, 0.0110, 0.0037, ..., -0.0066, -0.0034, -0.0086],\n", + " ...,\n", + " [ 0.0004, 0.0110, 0.0037, ..., -0.0066, -0.0034, -0.0086],\n", + " [ 0.0004, 0.0110, 0.0037, ..., -0.0066, -0.0034, -0.0086],\n", + " [ 0.0004, 0.0110, 0.0037, ..., -0.0066, -0.0034, -0.0086]]],\n", + " grad_fn=<EmbeddingBackward0>)" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "seg_embed" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. pos embedding" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-21T12:00:30.521422Z", + "start_time": "2022-08-21T12:00:30.518935Z" + } + }, + "outputs": [], + "source": [ + "pos_embed = model.embeddings.position_embeddings(pos_ids)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-21T12:00:31.705824Z", + "start_time": "2022-08-21T12:00:31.702182Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "torch.Size([7, 768])" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pos_embed.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-21T12:00:32.905482Z", + "start_time": "2022-08-21T12:00:32.900641Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[ 1.7505e-02, -2.5631e-02, -3.6642e-02, ..., 3.3437e-05,\n", + " 6.8312e-04, 1.5441e-02],\n", + " [ 7.7580e-03, 2.2613e-03, -1.9444e-02, ..., 2.8910e-02,\n", + " 2.9753e-02, -5.3247e-03],\n", + " [-1.1287e-02, -1.9644e-03, -1.1573e-02, ..., 1.4908e-02,\n", + " 1.8741e-02, -7.3140e-03],\n", + " ...,\n", + " [-5.6087e-03, -1.0445e-02, -7.2288e-03, ..., 2.0837e-02,\n", + " 3.5402e-03, 4.7708e-03],\n", + " [-3.0871e-03, -1.8956e-02, -1.8930e-02, ..., 7.4045e-03,\n", + " 2.0183e-02, 3.4077e-03],\n", + " [ 6.4257e-03, -1.7664e-02, -2.2067e-02, ..., 6.7531e-04,\n", + " 1.1108e-02, 3.7521e-03]], grad_fn=<EmbeddingBackward0>)" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pos_embed" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4. input embedding" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-21T12:00:35.559752Z", + "start_time": "2022-08-21T12:00:35.554267Z" + } + }, + "outputs": [], + "source": [ + "input_embed = token_embed + seg_embed + pos_embed.unsqueeze(0)" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-21T12:00:37.337329Z", + "start_time": "2022-08-21T12:00:37.333062Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[[ 0.0316, -0.0411, -0.0564, ..., 0.0021, 0.0044, 0.0219],\n", + " [-0.0489, 0.0285, -0.0204, ..., 0.0190, 0.0265, -0.0045],\n", + " [-0.0469, -0.0156, -0.0336, ..., 0.0117, 0.0135, 0.0109],\n", + " ...,\n", + " [ 0.0185, -0.0608, -0.0331, ..., 0.0040, -0.0071, -0.1213],\n", + " [ 0.0294, -0.0017, -0.0217, ..., -0.0004, -0.0913, -0.0948],\n", + " [-0.0077, -0.0166, -0.0123, ..., -0.0310, 0.0124, -0.0064]]],\n", + " grad_fn=<AddBackward0>)" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "input_embed" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 5. 后处理" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-21T12:00:39.276354Z", + "start_time": "2022-08-21T12:00:39.270563Z" + } + }, + "outputs": [], + "source": [ + "embed = model.embeddings.LayerNorm(input_embed)" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-21T12:00:40.638877Z", + "start_time": "2022-08-21T12:00:40.635231Z" + } + }, + "outputs": [], + "source": [ + "embed = model.embeddings.dropout(embed)" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-21T12:00:42.332106Z", + "start_time": "2022-08-21T12:00:42.327600Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[[ 0.1686, -0.2858, -0.3261, ..., -0.0276, 0.0383, 0.1640],\n", + " [-0.6485, 0.6739, -0.0932, ..., 0.4475, 0.6696, 0.1820],\n", + " [-0.6270, -0.0633, -0.3143, ..., 0.3427, 0.4636, 0.4594],\n", + " ...,\n", + " [ 0.6010, -0.6970, -0.2001, ..., 0.2960, 0.2060, -1.7181],\n", + " [ 0.8323, 0.2878, 0.0021, ..., 0.2628, -1.1310, -1.2708],\n", + " [-0.1481, -0.2948, -0.1690, ..., -0.5009, 0.2544, -0.0700]]],\n", + " grad_fn=<NativeLayerNormBackward0>)" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "embed" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-21T12:02:06.326430Z", + "start_time": "2022-08-21T12:02:06.322627Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "torch.Size([1, 7, 768])" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "embed.shape" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 6. encoder" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-21T12:01:30.742279Z", + "start_time": "2022-08-21T12:01:30.738949Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "768" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model.config.hidden_size" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-21T12:01:35.621980Z", + "start_time": "2022-08-21T12:01:35.619037Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "12" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model.config.num_attention_heads" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-21T12:01:48.148730Z", + "start_time": "2022-08-21T12:01:48.145902Z" + } + }, + "outputs": [], + "source": [ + "att_head_size = int(model.config.hidden_size/model.config.num_attention_heads)" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-21T12:01:52.352083Z", + "start_time": "2022-08-21T12:01:52.349187Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "64" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "att_head_size" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-21T12:04:59.341811Z", + "start_time": "2022-08-21T12:04:59.338000Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + ")" + ] + }, + "execution_count": 39, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model.encoder.layer[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-21T12:04:42.151147Z", + "start_time": "2022-08-21T12:04:42.146734Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "torch.Size([768, 768])" + ] + }, + "execution_count": 37, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model.encoder.layer[0].attention.self.query.weight.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-21T12:04:50.814261Z", + "start_time": "2022-08-21T12:04:50.809823Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "torch.Size([768])" + ] + }, + "execution_count": 38, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model.encoder.layer[0].attention.self.query.bias.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 47, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-21T12:17:29.996361Z", + "start_time": "2022-08-21T12:17:29.993102Z" + } + }, + "outputs": [], + "source": [ + "Q_first_head = embed[0] @ model.encoder.layer[0].attention.self.query.weight.T[:, :att_head_size] + \\\n", + " model.encoder.layer[0].attention.self.query.bias[:att_head_size]" + ] + }, + { + "cell_type": "code", + "execution_count": 48, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-21T12:18:24.075806Z", + "start_time": "2022-08-21T12:18:24.072197Z" + } + }, + "outputs": [], + "source": [ + "K_first_head = embed[0] @ model.encoder.layer[0].attention.self.key.weight.T[:, :att_head_size] + \\\n", + " model.encoder.layer[0].attention.self.key.bias[:att_head_size]" + ] + }, + { + "cell_type": "code", + "execution_count": 50, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-21T12:20:09.874443Z", + "start_time": "2022-08-21T12:20:09.865950Z" + } + }, + "outputs": [], + "source": [ + "mod_attention = (1.0 - input['attention_mask'][[0]]) * -10000.0" + ] + }, + { + "cell_type": "code", + "execution_count": 51, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-21T12:20:13.470571Z", + "start_time": "2022-08-21T12:20:13.464648Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[-0., -0., -0., -0., -0., -0., -0.]])" + ] + }, + "execution_count": 51, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "mod_attention" + ] + }, + { + "cell_type": "code", + "execution_count": 53, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-21T12:20:54.416991Z", + "start_time": "2022-08-21T12:20:54.411761Z" + } + }, + "outputs": [], + "source": [ + "import math" + ] + }, + { + "cell_type": "code", + "execution_count": 54, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-21T12:20:57.050346Z", + "start_time": "2022-08-21T12:20:57.044051Z" + } + }, + "outputs": [], + "source": [ + "attention_scores = torch.nn.Softmax(dim=-1)((Q_first_head @ K_first_head.T)/ math.sqrt(att_head_size) + mod_attention)" + ] + }, + { + "cell_type": "code", + "execution_count": 56, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-21T12:21:07.873752Z", + "start_time": "2022-08-21T12:21:07.869076Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "torch.Size([7, 7])" + ] + }, + "execution_count": 56, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "attention_scores.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 59, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-21T12:23:24.115022Z", + "start_time": "2022-08-21T12:23:24.109896Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "BertConfig {\n", + " \"_name_or_path\": \"bert-base-uncased\",\n", + " \"architectures\": [\n", + " \"BertForMaskedLM\"\n", + " ],\n", + " \"attention_probs_dropout_prob\": 0.1,\n", + " \"classifier_dropout\": null,\n", + " \"gradient_checkpointing\": false,\n", + " \"hidden_act\": \"gelu\",\n", + " \"hidden_dropout_prob\": 0.1,\n", + " \"hidden_size\": 768,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 3072,\n", + " \"layer_norm_eps\": 1e-12,\n", + " \"max_position_embeddings\": 512,\n", + " \"model_type\": \"bert\",\n", + " \"num_attention_heads\": 12,\n", + " \"num_hidden_layers\": 12,\n", + " \"pad_token_id\": 0,\n", + " \"position_embedding_type\": \"absolute\",\n", + " \"transformers_version\": \"4.11.2\",\n", + " \"type_vocab_size\": 2,\n", + " \"use_cache\": true,\n", + " \"vocab_size\": 30522\n", + "}" + ] + }, + "execution_count": 59, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model.config" + ] + }, + { + "cell_type": "code", + "execution_count": 60, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-21T12:24:01.459481Z", + "start_time": "2022-08-21T12:24:01.410899Z" + } + }, + "outputs": [], + "source": [ + "from bertviz.transformers_neuron_view import BertModel, BertConfig" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.8" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/fine_tune/bert/tutorials/08_bert_head_pooler_output.ipynb b/fine_tune/bert/tutorials/08_bert_head_pooler_output.ipynb new file mode 100644 index 0000000..5eab70d --- /dev/null +++ b/fine_tune/bert/tutorials/08_bert_head_pooler_output.ipynb @@ -0,0 +1,93 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "ExecuteTime": { + "end_time": "2022-10-23T03:44:14.249044Z", + "start_time": "2022-10-23T03:44:08.843953Z" + } + }, + "outputs": [], + "source": [ + "from transformers.models.bert import BertTokenizer, BertModel\n", + "import torch" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. load model and tokenize" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "ExecuteTime": { + "end_time": "2022-10-23T03:44:59.645019Z", + "start_time": "2022-10-23T03:44:59.641141Z" + } + }, + "outputs": [], + "source": [ + "model_type = 'bert-base-uncased'" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "ExecuteTime": { + "end_time": "2022-10-23T03:45:17.042619Z", + "start_time": "2022-10-23T03:45:12.116350Z" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertModel: ['cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.weight', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.bias', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.bias']\n", + "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", + "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n" + ] + } + ], + "source": [ + "bert = BertModel.from_pretrained(model_type)\n", + "tokenizer = BertTokenizer.from" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.8" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/fine_tune/bert/tutorials/08_bert_head_pooler_output.py b/fine_tune/bert/tutorials/08_bert_head_pooler_output.py new file mode 100644 index 0000000..7858bd1 --- /dev/null +++ b/fine_tune/bert/tutorials/08_bert_head_pooler_output.py @@ -0,0 +1,12 @@ + +from transformers.models.bert import BertTokenizer, BertModel, BertForMaskedLM +import torch + +model_type = 'bert-base-uncased' +text = 'This is a text sentence.' + +bert = BertModel.from_pretrained(model_type) +tokenizer = BertTokenizer.from_pretrained(model_type) + +inputs = tokenizer(text, return_tensors='pt') + |