1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
|
{
"cells": [
{
"cell_type": "code",
"execution_count": 10,
"metadata": {
"ExecuteTime": {
"end_time": "2022-07-03T04:20:09.832759Z",
"start_time": "2022-07-03T04:20:09.828238Z"
},
"scrolled": true
},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAABAAAAAFUCAMAAACN5L73AAADAFBMVEX///8iIiJdXV3p6en/8swAAADZ2dn0zM3z8/O216j29vY+Pj78/Px4eHjU1NTm5uY4PTZAQEB9k3O3rZI7ODLWy6tRUE6nxprhvL3X19dCNzfv7+/9/f0BAQGIcnP5+fnc3NzY2NhkZGRZWVkkJCP19fUJCQnw8PAFBQX+/v6IcnJoaGjo6OgSEhJVVVV6enqdnZ3yysvNzc26urqrq6vt7e3k5OS2trYmJibi4uL77skwMDDe3t4MDAxwcG8qKikPDw7Q0NAoKCey0qRzc3OTk5PvyMgzMjFyhGqiiIm11qdMTEyIiIgtLSw6OjrDw8OwsLBbW1vx8fE8OjU9PT0YGBg/PTjT09Ozs7P+8ctBQUGpoYg7Ozv468a9vb1lV1dJSUmAgH/MwaNRUVHkv7+sy5/CwsJ2dnYeHh40NTRBOjqz1KXCo6M6NTVsbGwxLixea1jg4OA4OTjy8vKFnXvrxMWkpKTZtrZXVEtKU0bBt5wbGxr6+vpbaFYzOTHLqquqyJ2RrIahoaG6sZXeurthYWE2OzRGRkZ+fX3IyMhEQTioqKienp7j17bSsLGQkJBqZVhOS0PAwMDw5MDq3rs9NzdPRUXGxsbt4b5fWk6fl4FDQ0OYgIB6j3CQqIaYmJhiclu7np43NjZWS0vLwKQ2NDFPT0+MjIyYkX34+PgyNTB4Z2cVFRWdhYVxYWFsfmSCgoI8RTeCb2+mxJmbm5vMzMywz6JOWUmXsYyPj4+qkJCPh3SwlJVqW1yiv5WTfX19a2tFUEBFQz3nwsKcuJHTyKmQenq1mZmIgm+FhYXPxadXV1fGp6iWlpbm27mAemt3jG5neV/f1LNeUlL06MN0ZGTHvaFKRkIuMS1aTk5IQEBxbF5gWlrLy8tFPj5VUUW0q5JSXk5aV00/Pz/St65TSEiBlnhETEF8dme+tJp4c2Ofmo6upo2VjnqLd3fVy6xXZFKLo4F1iG1jX1M9RDt1cGFugmY5PjaZi386MTGSi3jaz69BSD6Ff22GgG+CfWy2kVifAABW00lEQVR42uycX2gT2RrAM1vsSNc/7Mtl2mtjrXG11u3u0W2bYm006JSh62VZFEUuThcD1w0zLauM82Bh4BKposwFodMH8zKWtkg2QcL1YcMMDgQapLlPIUgCNy+XvvY9b3cmTdo0SdfMmaz/+v0opaX06znfnPP7vjmdxOUCAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAOAToW+w0w6DO4TZN2onyujhxlHabI3leOMge9vtBBnqcxqhPKV9DYYy2InFyKFG8zp+xE6MtvoAHT1Yo+k5VhfpoP0oR9oaXqzBI50Oab/bIOyhIbxgg7tPAD2TfFfzFCd7GkaZeey1EaVrermhAS58ZSdIcXWkUZCO3mkbQfjxp/UR+u1EKON5PFOf24liFw7Tww1sMjJgJ1jxxoG6ab304Y1mrKP2Oj23P6/is18bXKwhzAxVD+9BfSE4NDyOFas4277b6v/IRDZENU/ImGhkgJknPGcjCsUF/9NRP5aeG3E7QUL6wMEGJeUfOclOEKZwpqYHGLyYkynbcMUntbt2ZDISonBI8PUGOLiqi3bm5Z69VtPWvHkrYY1GWnv407ZI8zfC9uclGn85UG/I1axIOURm+2uXwb6nPIMVK2TsUOA+W0YnDYG0gxBpYICZe0XVVhRSCi7XFcxrN9btjSUQO1rXA7zoVwL2hhLl57cXyimFIjFQizU9QM+EkSSxQKYBau4ChgZi9oLRJ2e3bbiONwUZbzRkSBmrntmFKysIZ1LZ2dqrdWpSp0nHJBdqeoC+YT6B8GIJxuRu6gH6RpciSfspqjXAzHc8ZzfhXKGmB+hrfxa3uxoE/dapuuot2gxCJ64MV0cYsx2hvMA5/kn1PhmajQi4axrV9gBDR2M2vYZo97OqHmDvy7cywh2OvPZwa4vNfxXG27VCfGl7D9A+EEmSLSC0vQew6j/2VGljdhf1AL9OYqxRIbLUU1P/edV+wqW55W1F7oDd+l/uAYacV+/w4qOtCGNKCHfXqnxVD4Bf/xv0AKfs1v/SharqATp+wK7/pS2WH6v46PT/VnCrNspOVp8DtE20ov6XlsFCb1UPcOZ5AuHHEoyJ3dID9PUsYRm4RpI49d8iXbi0dbzc1/nsJM5qEPSlturqncGp3nTib8NO639tDzAyG3G2uhPF4U2bjKzarf8b83JXTgKPPcyLyMlo5IX9f92o/1+G8eclxCe2mpL2W0aSbBEU21vpAWaeLjKOpkobz3ZJDzA0idmjJo2qHgCv/pd6AH55M8qRG268sVT3APjVO8w/qkQQHaydrR5gdMLp6kYaP1w+nWwb0PGCmT1AZ+lgZDkoOdxiYv7ivtL/aVaceI3e6gHallpV/0sGWOgvH1SeGU8gZ7GSu+UkcL+Ou0ZpfWzz5Hx0UcVNOHd5tBKlF7teCrk3lSCP89jdOzPZ5ixCpQf4ZWPtXHS+uhHzvHxK8kMugF3NHlhCusmLyOlwxKDZJd1dXXE2LzpWOU/8lhXIFhJQ/r0R93nU8VTp2NSuEMDYCnaKor2bN6g9Qew9Q3V1VqJ8wWCPxf1tJch3LP7y/vKgwwjlKQWPlIbSH3a+ptNzZQH8HMeOoQ1YD1xcV1qwxRZ+dLmOT3AOozC95SdApl6TLSX2+0ZcnnMei/kCBAACAAGAAEAAIAAQAAigNWfuIAAQAAhg1wpgpP9i27sU0dYJAgABgAA+SwF0+n8Zeddjub4xEAAIAASwWwXw9AQIAAQAAviMBdDz6KfT3/9oPZ114dXg03vXj1uPwM/vdbn2dc7fPbh8YmD+OAgABAAC+FwFsH98ynP1G9/jGdfEnaXbV38b6HEdnvKZQtjbP37h+6vd55+/AgFsIatCSwSABAF9bgKg02kaWwAh1bxmtEA3m+JmBICktPCnCODdF+/TEUD/nnP77/Wf8w257uy5/a+bY92rMy+OfnPBFMDR2/8d/fnE7PVBEEDVNSW4lgggrbuTrRNASP2Dl9XtJIBQStNE64sk1XjfipqmhWwIgJrLB7AFECaiJB1mUyQZIdQWCUDI8NKfIQA6posOBIAkTVOtq492yrysaSnqvQlg6rBr75XuA6473ZdmXMf+/vW1LQF84DMAs0xaBheoAG19U17jKBkwvzR/Rn/CAlDnWKp1AtB4t30BaHxeSZBIZQuFvK7RZFgpkUKGoqzpmkAmlPx46r0KwFhkPjYBJNHmwjMbFOvDjJufkxwIgM7yCiuTdKqU+RSNcqXE66KcUZSFuIRIRimMc+9LAN3XrcW+57Trznnrn4K/n7j30QiAiq3FzTxlugpZkZTZjaQn19e6cgwts2vMJyyAJCfTLRSA7w/et2AnASQKadOh6uW38ZVIYSFAGr6IYZJGuTnDWPCumys/Pae9TwEgmQt9ZAIQYrJolMs9ikbJqPVYP0o3dWuxowB01rqH0HyKmflgLomm81biV6g0n4tneT5tVjeOV9+XAL623oCuf88r1x3CenXlve6XZQGsfnABiL6chpjbXQZL5EX1fqka0ZH7ejz/20ooejvbtAAQx8iqOxsNoXQ4FudoUmRSJa2rjNisAFCIyWajmZIAQolINrzRdQc0Q19JMbJNAVAJlSaRyMSz6ynKlgBQQFuPrHBJIZWwtpvIpEk5681FGS1p7iBmc2BNCMDcHUgPmjfutJgSSGORok0QymUoWsxYOZTtCkDSjCxDbeQlHlvnaHsCSFuXY0MAEmP2IGaazCjWyQIKJNxZtya+WwC0lpATRsRKTkkASArHY5GomRQqoZUab4mRmhYAl5fVYPlmLZBh0FrUiqAlzF4goYmbsxXUuB5PMWqzArCCZApSKfM0mo7Qpcyn58JIUMcjVqLfnwDObQrgqnW3f6n7n6YATpuJvfXhBTAdJiUiZy4E9Xy4LACqmDGTV+iiyWLzAqCzfsVHEAS7Pm5+XuRI6WzQWksC729aANKah/Cav81Z70bj8fuIguUQivV6/ISHeG33DIBnBVJWPF4f4d2hfd9BAHLG6/ES/jC1ELSskyIMMkpY5GVSLXgWxz15zoYA2Ll0+XtjcWOhmwIIkMhtFU+7AjireAjCy5rzETP+ad57NkrbEsBJX6IsACnojQrWXM0o42bVFVmP57LHw75bAIH8dN4ahS5sCEBgCcLv9QTTpKictXaVwFp/pUkBvGaF17lK5udkKmj9MVqZDplpLc+Wtl7heJnwezwEa0MAdK7yemhTAKic3bD1ZmeZDyWA7psu18zkiWsvHpwbNn/qKwlg6gMLYP2+lT2UTpYFkFR8mixQIWRPAJ5FN5dSPJd1TYoRutnZ+a1eLuHRUZMCEDLemCSqb00BJFl/VhTD0wWRFAzvQjqU1rEEgGIetyxy+g4vZ20sgGRpHKncSpUAAq/9Men/7J3Pi9voGcdHXVqFkrBXbdKaHMRmMnShb3bTaJkGDaaREa1IKO0trCgIFoxd0RShwy4IFh2yFFQoSLkYijCmFNWm6LbCxgKDRVnvSZhBhvqmq/+Gvu8r2zPjGU8leZKZ0vdLDkM8fubVo1ef53mf94cMS7CO7YAxW1Ks5AYAHfDNUJxAd9Ihl7ahXAQAoCezEhkA67SMfo2dQP/yvsm49ZpVDgCNY7Ul0NqAr0Ardk2nF2zPMCdeLw8AWKdrBDYbZAAQul3LtEKIJTBkUZcxqrNRPgBYnlefo38w7IOxJ/M9+XDg6ysAsMfwaptsgwYiN490q1IIAHSXt8NgwgAIgA5yfM9EABAi1b8uAHz/N8/f//N+7e+3/7n/yzdff/tjCICv91/95d51AgD0qGVvXgKAdtX9agqjQzEA8Oh5H7JoS7zOzWAvlNoafJKlKG8NIFJTZlkDcGttEx0mBbusZWOMi4UB0IAAEBwWxxOlSAYAG47aoeinALCqAQy5FkChhdfzA0DotqsU3+4DOuR7HlRWA0j5Ll0cADZqkUeJdKMjo4dsSImlAGDNWHSH3doAjbUXVEAPsCVg5gEAHviLrAxOagBwTH3M0Cavws7kb9ysSwAQDvipV5VDDIDQdsL5LKysAYCNh7BhShv3I6MYALRWCj2fRACo8yl0/ALXAHpNfFvfDQD4UwB4tPdqnzs6+vDlG3QW3v7To8OXEACP+f2jJ9cKAJ/CNx2sAQAU17v7UVxwCKCiL/fZEPWQmg2DC3r4XDVW8gJgyOFzZxEARNYH+NGb0i6HUwix3BBgStmVyNCKFAHBmBtmvLgAADIro0e4frZQeTkAaKAxjUqzqZ+qAahOMnWFEgDARcAWdIbIHqOWxJRfBgBjR8WHfHVZB1lJqAU95mpeZIxAHgDM0d03OqmSAUARZVuVWJiv0T22Bax53coJACBETW0yswSApwDSLhOL0EErAMwRibvUmLbqqVkcANjzBx2bOakBNO12XMEHpbwTADz45MWtvTdfoW725qeP914dPf/uF8/uoy2C9//6+XeP/v2n9/b2Hn3zqx9c6xBAzHrztLsEgO5BX4+8D80SAIgyADQhAGCHmwKP6uadBRD8qriaBRizuKNMqJgOKI8unQHQVsJRnN2zCgBACKvLZl0AAIdt1rGKAAB3xQU1OVUDSPRsUAJKAQA6A4wpKWvJuAwAahS+NFCh1MzKkNZ7LMV25EluAFi2Y2IAjAZVqT0NmwgAkZQwYnXzpKdLagDDGLRWxwTpTsPK2HEaACIEQKODA0lRAGAX+5R1qgYwXE05vhMAbOjV0eOHt5c7hB/+aLVV+OHD660BWBRMvYXgaLgCwD9CDcbBp8yOANDrdnTY1PICAFS4Lr3OAA7woIDq0S67SwZAC0a3V2flAjUAsOBaawAYGwCIOVfIYkluAIz8CI5ClR4cZoc1czULoKzLXuUAAGO3f0FL8gKgnrAopIIWt1hbEaxg6rBzLXcGUEs0DIA+m+oC0B0EACXmA0ea5JwGtOR2p5k0O21/RAuL9kxKHS5tG+cBYJTJAJQwgmM/ZdBBGYC2mgU46R/XAoAbdh4AAgA4eDr3Bl/Udfe3M1mWx8zgI2caf5EIOwJA8Lh6lsrnGwJ0ORw4EAAiHg9wKzAzNdSZjvLyUhkAQH8duJI9KlADEKt4XEyDUdKZnAAANy5kF6DIOgCUH7dReEwkCJCQi6GD5cYJAEpnANBFsVJiHcASACLTptCZw/0TK9hVVke18gJAZHtZDcCnUJ8yMQBg2+ZcW8gJAKa7OPT8mt/qa7QQDdO05aTDlnkeAKZziNrgFgIA46gJ8vxCAKqNHB+a1wyA7718cAMBAF2f1B3IYEtOoXwNdJN62ipaBDwHAHoiUfYk/0Igy5ZEa7KoQgAwSXVhMv1ax6K1HpuKYsyVygCUeGgwZiA5WgEA6MfcwjKNcCxMKc8yujYCgKHWJ7olTGx1DD8L4gJFQH08SOIemtEMBlgNejjUds0AlBhV7S23F5SbBTBjCkKWSVgPWol6fVoODdNs1JpMDgB0AssKOryRzQK0qBRNteAaAK11qGo//0Ig19Hd5pIXSoJWAYCLhgD0kLK74pQvBABgYc+7gAZT7Hjf1MPoOgHw6f3bNw4AQwEHu7NPCBihlSHCjgAQ2tQFx+RuBQDwOWnerPKoJOEe8k5ak1DjLIdjuZpDtUpkAIpTradO7VAstA4gqFVnjs1VgMtx8zovIQBobbbuxLrQleBndV7NC4CsuqWMLm4AEIyyAKAndc525odcuVkANEJjfY1u2MiKygV0zNqO06wu8tQAqMPZXJIOhAwAVpOrz2pVDgMAhNT5Q9a3A2AoC+tVALptmLPsqcQAUE4BgJGrHMfDDpUXAMJ/8/x1AOAmaBMAXwy2JZLG3SMv/0rAIES5ozFFAUnzceYvtNlGkaXAgii3e5E41fH8UJz0sjs0ilqBPqb6BQGg+10BGBUYA7wJKLYU2PLleIAWoPUH7YE4wdekw/+rMPizRPbPLsDbCgBplvYvXbObzqpFAKAd4BcpNabIM0prEMvTYJQXAO60QYO+B3OyYIrvVejBBIUZQiseTMGX4VLIA4COP5Cn6GB4oeXrNDCm7SRsjBcKxjg7BLkBAMQ+La48oB+MrHE2IhHG4YjWKosRvloUWoRGV7TczTmPrQBQ08t3EwXQ8wQAyK3R1rXcIzeydtoL0Lgw9b5sL4B2sncLKMuf0QZWwMQbhaW85wEAbVsMuGwvAPyWsJxG0k6yoqUhTdncqroNAHoQBJe+m8uCv1BkN+CGtsW3YucBrK0IiiLk2QugzOa6cMYHArP+5qij6oX2AuQT0AQAhJAVcwEANIIgunQvqAE9zxAAFFNBAGjT6kWvayh6IEiQhK1hm9t4TR45EOSyxOLtHgiyLAJerBbl0W8BAPrMG7YG1WMmFwBKiQDgigFgzZ2Lwl9RAPQ7Ei/VNhM6AoBrBICz/WVsStx03woAahLPq8nm+ygJAG4wAJRoQl8BAAQrCqJzO3sJAK4PAKDR2Lo1Wms0tLcBAKC7Qf/8kk4CgBsMgG1PCzkT8H8cACWeLfJmIAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACgLes3zuVsmqfAoDklbUSUmsAfJuUNXIwOwFAs/QFeUsANA92UygtAZBWdlZPWgLg9ay0jTgDgHqwu1QMgMGOVpI1AJyDK9XdJQD4we6eT/4/APDZ53dK6+P10SX3/lbeyutbKysvdmjL85WRD16XN/IVftg++N2dHfVNdqrjx3euQM++zK7rzQ7OeYFI/fN/XUFrXv9hb+/LT3Y282TZcz67c7V6/Siz++wqjD3ZIyIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIlrq4Q9vbeqPZ/Tr03pwRvdO63FevZdbPzmt97frZ5fo0636D3vnE9NE2sfxIlNj2OpmA5qQhcACKqQiRGDBqQg0mB6AFcSiYx2CNXkph1eiJoa0nGwI+KfhzzSh1MMmpkVIOOxBD4QbSROosSZNMx6IKcGLEAwHwsUG952/nT8FOp0Z0BefjzE1rfPNM89MP/N7npnp3N+dS3uQL5GjQv6gyN6VE3twY3capHJBxJs3bz4yPGEY53OV4T8EYxydfCo4qkXc5NMqpFLEMz6PRDzm8ZyhvLz8NZ+/Gd5xXOEzzDBL84pPs4hFjn/43GEY4lHGwF6k283RyOeyiHs89CIKRAxwHDkyvg8CqBpqK+TxluN8AndFTCbQJKIrgcHBQfJvnBcct3j8RvKJ4heS6xS3efxF8oDjJZ+eOCYOQwItIuoSMHLUcBTxSOOw7YyDxMwwI6I0gXoRtQmcEVG8M7kCtID/ex7tgwD+7AH9CgBw5CalOBXOJKU2Keyx4NlhFUCyDt6rxwRHS+GxVHisNQtw8Ek8aseP6vEjffzwz9UEglpBUEoICw2i+ughyhK2VqFqF7aQoSobqsihyh2y8OHqILY24pdMg6KSSlBxCesxQbkmqOW4Oo8t/driHBEwwKcgAX0C9wRcTkBQHHcLYWvpsjhDAtg6XFCcLwppTkBQ8r+anR2+kox3ifzN43Ui5STP+TxO5FEiz4g/j54lULkzF/dBAP3ZxGiQHRgKRo3MKJMdiLKDVWosy45seWNedozMjqPZoTY7GI+P2Inx/A7j/aoD4feknE3KtaQ83IFjSjidlAwVmE5KXlL6VUCXlJyklOwBmPgDAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAODj+HDilgEk9E3NOQUxTOtuY2Sb5IZVsyNW78ptS9pAJSb8nP+TuH2xTXjedUsyknnmGRmWX/M6Z1TEtyj8ivyUF93n7zT35qzb5hJfTMHlKLZpm++OxehV6vundzyCABv96pny+9rI7lt8iN8PSG38yUOGa7JZsck8GGpUd4qWfDESuz0vZ65OZVRHfDZcyFfPFzz4abEJ2xvb5PNaOWfJbYrrA7TYl/jnZOU8reftfddZypkpsF8afMKMxfVWet/ZTPBmooQOF5GNtZ78wT91yMzxPuWcDWmS3ZI4TwIjsEBcngOuY7JRRTgBeSDGBDlYAC7IznJwA5uW3pJ0vgA6r3BhkQiCAKAyphJMngHar8jwLEAAQABAAEICKnI3/WkQD11bNxQsPgQCAAIAADr8Abp5hb8st5f3kyOXiMSAAIAAggMMvAPLBtd3a2uPHj//zKxAAEAAQwE84B9Cf6xC9AwQABAAE8NMJQHej8bapsPo0LYCM8sLmjJz87t9MbZ0ZmpzHA1cbX/5y5z4QABAAEMDhFEBDXbHhU/3MlRxSALrXpcYxTXZPseFFvfmxRqcvNs68LM0d0AEBAAEAARxGAZweLH6VkffRVptNCKD67zM9N0ryzhcPZ+Q12GbSdfrcW0fzGtJsDUAAQABAAIdRADdsD84RL0PaK4QABmZaxomdzmYgL4pt1LYSFcC7Es2xtzOdQABAAEAAh1EArfUF5IatyNVrLmu1WtNRjaaztJj81dx6bbNOf4b46k/rgQCAAIAADqkASvXTlAAGCAH0dOVeztFUl9roX4qu1OlriYpg+h4QABAAEMDhFMBHR1cV8TKsHSKGABW/GmsrcxrMn8i3TrTeBwIAAgACONwCqHpQezNPd85QXEGeBch5Xf/g0sMHZyrz+qtu1Xd+JwGgbhZYsQDCdga3fAFYmQgrqlgAHrY5YRkCgIPs0ogCAaSYIkEAGM4EBmQJANlgG4QqFICLDQqpIIAYk4VjagkAk7TxD1gAmpulpfoyQ21jP30dwIBW3189U1/Q3ZM72P+dBLAef4BQQLEAvrAP/FmWL4BRJmI+plgAqzVM1jckdQFgK+zKwAoE4O5jQupglQQQzGISNxE5AoBH2NUKKxSAlw2yqCCANSar166WAHzsYyXXfggB2Hqo1+nW22Zz0eJZjWYx7YlGk20wPZmuvu4wOxrzNbrjxjcaTd6QYfxABWDR2lZoXCoIYGWKIqxEAN+oCItHBQHM083xQXIEULdGLexFFAmg5XMKKZIE0D5HJToheQIoWqI7BVMsgD46KKaKAOhWeV3qCeAp3Tz7jyAAjpz0+8dEb5VUXbr2/S4FtmhTHT7vJYApxXMAo2nSa7akApCyajsLAMFW/D7lcwDuvvagunMAwaw+j7Svxi4CMK6qMgeAeNPm1JsDWEteNKYsgIkfaA7gB/49ACCA3SoAIIAkFQAQABCAWACffRQeJQJYpiICERUEsEI3xyVLACYvvbSyIYDfQoVY1RsCfFilEmMyBVAzRa+WwklAogLYooPcqgjgK5UVxNQTwCjdvDAQQBIB1ExQTKEqCKDOROFUIgADFRFVYxLQSDfnq6xJwKJeemllk4BMSrt6k4A1dLOW5E4CttDLu5QJgKgAmI29rIoA6FZ14OoJgOmnbSCAJALQ5lJMRFQQQPQ9yb8nlQhgi8r44lZBAB1U1PtVeZOAI9TC28oqACZlQb0KwL9GJVpkCqBok95GHsUVwCgdFFBFAHSrFmLqCaCd3vjLQABJBDDhovAgKgjgC0yBKBFATGKGlDkACVG7zwHg9NIK5wB8KaRImwMISUrcdQ7AKbl/k80BSA+SMAeQSqukzQHAMnoezAGASUBwFgCcBVCVqgqGznTxOcEbY7v9NOg4u0PpToyd7b9w9dhPKgAUPixnARA0gu56fBN9Jk0AMKpMAHTf7p2SylmAPZK+qwBS6fn9EEBnLnOpXW2r6JPTbebdbv7vKc6h/5ExYB6v6jJmH4AAIlbySkwksByAIZiZj4Vjy3gEggO467sIAJlzqiKAgBNWqwKAI/IEEBkZWQpBSNgytb6BQCeXCTJd6Opypg+F0KWRrVjKAghuKRPAexzB7UhwDVGpAlhdUF0A7syQcgF4iJ53QzDu9TqJl1Wy51fRCNHzQaLnR0a2wvsugJZXFMP5YgG8lSaAh0MFF/dPAI6nNBarw2+HnL11WY6oO5SWSR0Ho0VZxl67Z9OxJkkAHX0UuBIBjFIRS5RxkL4pJQIw0c2xIJYtOOUKoIZuyIro+2E3pCQAI50ygWAmCx5BvO3Rzah/HVowraysTNhdWaMr8x82YBw34VIF0BulEhcQp0nWJOAHulNCUIcFmVuAM/2IzAqgnQ6KX2rnHVUggHkqazNANDEUcZNNCkVC0MaST7YAeunmTUEh0zqOYiMdE5sfDCjaMU/0/BIWexklej4M46um4L4L4O0un0gVgKY/r2QfzwIwLFhb7Gi4aCLg8jq2ww5Kyetp6yFrVhRx921JEkD8XgA35iaKBgwPwMT2JLcmJlkA8XsBkJD9ZJQQgAcPUO3dWHWF0JQEwGQtIC7ioBsJ4u5UKgBm4SIYQlx4GKLWA3MjFmM4jJJhmCQBxO8FwP67QazK/PsQGnJaoW9RDwHq8js9vtHPxP/0SxZA/F4ApwkN+MjOjfh8mPTTgPF7ATosUCwGkQJAXRgEb1C39USCgqs49qoAGNYht8eDx2gBYEH7SRiKhIhvOMrbWKncC+CZwskzHMiczwtFAhjijoTsYcoI9pMhTLoAmMjPUCiLOMYvdyxHIrEpGG33Eh2PITGT3YN/2CZW2H+AAshoPlJ51/aiPL3ZYCtsIARQutjl+DRMTgTkl5kct66cJe8XGDT/9aqOEIBubNJm6G4yj18rG7ioed7W2mY0Fhwt0fRXDzpMd5oHf9ekLxrMJv2JHHVuB7b2BqE5P3nktawzAvAWrUegoBdCJ6QIgMdI1v+ou4PQtrE0AMDq5rmUbLqhtDmUJmQ2pK5DJilMp51UHsdu6OLDbtppvFWqqoJsCtP0MA1toQjbpzElU++KkWPD2p7DQIg7DuTQQ+dQdBMYMi6VQQjpYIqM5yIbGR+MLjFx9sluWidtd2zFG3beKbFjWf719OX9f9574WR3UZbdm1Emi4JoItciAE19Ne4Oyc4kmvMonEKDiO6hMHu6DQCaaaJAmgspK4GWRwBNDf7yULAaT8ET3YyJ8nBISc/LcghrXuH328uB6wDwWKZxe9UUHjYIgBegah5tHYDmzj+hyCcoEfdyITmYag2A5gYBULMGAGFdqdIxTvHE8VVPSPFIrYwAmlo2wcn2JAkBELMyhSnhqr0A8KQcbhmAJljK8WxmWwApIqZu0OXQajTGcfJiDqCEWwk5N1sFoKnVASCwXP1NIQAw8BEIQBqg2Qp6IACMD9abH3k4bvUdumkdm1u/OTEwN3L65YDDOe6zfteDzKxY77485Dh7BPlq3TV3xWWFAFydsh77weV4UwO4bPX5lny28fPIkwXX3N8dY5au/qNW+zWn7eanHQMAjVGNa0M3AIhgC7KaRk0AQEkCgx0XWErUuD5AhOi2AShzSVoIJSMhlk9RG4BIzEdynHkANkJV/kNrQ38TAJzgCny6EGkAQGpTNB2pZKsCkxXbBgBV7XlVi+KgNm1kF146USBTcgaYAQAPOHOCFpSiVEYox4rmARBroXmwoaRpFhNVSuAloV0AFElgOQECgFarfADzolQRiBQLTAFQqzEEDcoqm881AGDSVYbBy8E4LyimAaA5jNmQYNLvMdJUJlx1r5LzHHswI4CxlXobggAMjD9EXlh855DzE65eCMDTLuSk3XfVf9Y22YP0zjnOdbsdN5C1n2y2fv9t24/+kUfWHQAsd7uQ+3brpSMP4E/4n9ssXYOH1s8jg39wvu4YAKQSw5sBAGGt4nEVTQCwDe+6ShhI7igZ3IxQJbxtALQEatQAytPw8pXyYhaeQUo2D4BmVzUBNwEAWskYL4s2AEC9TngewYqqUqFI2wAAsqBWPLCn1uRCoeDl6QTGeZgIMDcCcMOzD+WqE4yqhijSLAD2Iky3UDmkqswwH0+oOQG0CwDUh59KGwCkVUqe1oD2De31VM0AQEpqPCahpMQQTJqsAxCvX//cIuwOMdMAgEg8Sy3GYAqwBSMvkVV3EEY+fDAAWK/X21MIwNhXCLJsnetG/OOOP51+6YLf9jyzver9dWH51KlT/7Kc/bNvCeYBZ3y2/rWFQ/DDnb/5FoDvjdvF8u9Z31OYM3Q7LV3+u9ajP8/cP3mkcymAGjRugkK8WgcADcARKp/x8aYASEIAVt1RwHLHv6FBuwDgBIYbAKQX4WvjVJRR9wdAmKhgoVUTAJCVGtgDQBrbIAgiQLYNAA4/Uhgm/Xgt33jzRMmbqr+rmREABIBXcqmpEjyZAmoWAGcCK+Mil4UHIcRwvOIJlU0DIMjZQEDWoEulrC6aAUAkqEw2h4rJWEYPiDsAJPOi5sbNAYDXAcCNvW/i8FecR6s/WnXHG5E/0BrAw3Hfz/CCOB4jiP+aY/b0S99h+Ojn1juzTtu6z+cbs4zfcN0zLtwxW3/vwA/GFgKjOwDYbiDG9sHLn7vu+OFX45aukX8uWAamRr/2dw4A7/Uk7JZY7A0AWRg7nFig9wVAGcN00DYAIJAQQDSUFBIBIOoMynIRksDMAyCIIBzcMFMDyMSiAKWjMRaQGQMA0ejfKHwIbb8GcDwKP3xFfwfATg3d5AgAAsCvEDhABfMpQFWXy6ACExpcQAUUVOVk+wDgWtBIAaQJGr4e3mKbiRMSMAMAKJfixkVOxwl4gCYAJLeEVkOmRwCSBN/TuyK9A+BNVzpgAIxd/686ht4C8EkdgLOXpq4PXTba8xtjdQCWbP0X6gAMTr6tAXyxA8CkccfftnQh/ku//Lpu853rHABocSFfPDE1T/uCsVhMSzsTDDNVwfcFAJm1F0wAQFPUVmwxKWbkkurxglRIyXPulGkAapUSix03A0Cfh0lmWbGGbTMn8mgfl2XLcay4yajt1wD6ZEXdoiBjtYSu68XV/QCwMwIgWU5lmcw+ioB8JU97MaakK+EMk1SDbRcBsW2VY40iYJ+nUsvDFAAIToU0B4AYqf/1IByJiLsACGe5vLJoegSQDFZqRTlGkh4FRr4m/p8AMLaMIP3PbD9efDAxC5//8tvXhxfmYAowOLyTApx5uheAJ9eNJOH0hKVr5tENpP/wqGWocwDAIRgT2+4D4S0YJj0AqhsVhgi3XwPQAhBdggT0pgjQ7UrYBACgmikSSQmIcT1jFNxpQtM42hwAEgHSrK4GSDMA4Cm2yKbwCKsnNQJHvWpGQANb+obU/giAlNiivlnGQWEDNjYV3hTMjwCq8G4Ix/sAmcvobLp9AJIpEMiBagkHdKmAplh9S0Pna3om0HYNQKmpBApW4wAtFOE1g0pHVzRgDoDdE4HiNEnAvG0VHl7MERIVNzsCiGpbukrAoVzJiHxSjG7yBwfA095Gm3kPgIHHnw5+5j70ZG3SdvnUYO9jx6vBP1q/6j7yrVEEvGb9y8PuL1x7Aehesj7vuvA3m6Wra33hZPfaI8u9DgHgzIn1GZ343sm4aFVpE4BdHVXWgBkAduaW4vU5q5FcKpXNo+YAMFJA/MPTilvZFRivh2QnMPX1JXui1AIAiYBAGof6wLRUXBBMANAcKBy0D8Cec2jEZ0+UWkwB3i25IclGAWf3n306sCswGpCqSS5lZgSQ8MLM5sPXHxfKBwCAa7rRfpp5DwDb3D2n7VkPcnF44K/3Htiu3Ee+Xlh/fG1swNaPzLpdL0d9tr0AIMvD1lt3HRCAniGbc2h0YX25MwCUTyQ+soFaNLbImgfgOCWaA2D3jcpgHJMGJgH4L8c9oG3BxXy+8rE1+GQlX+nbBwCtzwRst7UEwMZ7y4JEPYl2GgA2iOUJ0D4AIAIj/7GeSsLnqv9jAJ647W/aiyN35g7D++/WLwjSc/bWl91Hl76/MvHglfEZZkenJ6ZHPxlBes7dnLDfGbo7goy8nptwDn23NDszefs+8uKBsVvoC/clxP/Z5LHxR+6BbqT3H9PDzqfnOlQERHn+I8s58EjTFKy2ARDfX75mAgA0zPMi/vsFAI9Gwx/r93uf+50BEH4vbmh0t/md+L8AIs9HUTMAGNHFW4z8wS8HXruwtlMguHimMeF37f7OJqH+C+f/w975hkTV5XH8jsMdpmnWNjFCmqFpRu0O0x1DndGh+Z+hqJNmOztiIUqUI4E1KUI7GiWYuQqyvhFWWiWwHmWgHrIyHlhoaV+4EtSbB2HxhRs8sbAvFnm2500Le8515s5of7ydc6yd/H1fxBG8336/37nn473n3HPv+88AH14cQKeHI57ADwBq3Y7Pf0oYPgzyFQDwOYIPgyjOMjs+DMJUumrPQGTYLJiJ9wcAAAAAAICsBQA3IAphP383xgEAAAAAgJ0HAP2hMW/JSDfFbsDTf9tHrjMzqQFz+nekHj/mywA49QNxJP9JA+AfxCb/TAPg5I/ELr9PA+C/+6j1r/wUAP5O7PFdGgC/JY9kJhMAp88Q+/xhAwD+8td9jPTHDADMnKH3+2FHvBLMdCWfQpd2J21id2ZIPWYK5RccdPmIIwmOpUwayU1m7I6kiWuIOJ/89H6Mdl8+tS69TgLgRCW5R1dqbjjQSR7J2UMZ583kJWKfysyH1cYr81npkndWtn19id4v1LETAGCd1dFoloWNPIHRR2EiL3/oKUzkdy1Q5WNgkc97JWZSHCtNIJkTTUzqg2TQsVMfm/A+5AcCgUAgEAgEAoFAIBAIBAKBQCAQCAQCgUAgEAgEAoFAIBAIBAKBQCAQCJQFcgUCgcgspUkTMgmoKU0MBcjERLs7w4nz0VKaaCPIZQ+jCutykRn1555NyKSAtp8O4H5y0KfUh31cLL9gy7TiqOYBpnbfsPRmURQLI5QujWFRTNRTmhSFUChtRyhdvMikspXSpKwQubDaKfoImSXMtC51KKLOXEqT3oQohqfoU3Kj/hZtWobnYWsUFYnd3twVfEJ26GF8b83yOp7n/RO0J5aR54203acVkUm1k9LFgvLBH2SjUrkfhTLIqMQNHmRWR+tyDOUVLaU0GcP9tEifUgH+vry9ieF52CwwrDjHjeMABwEAAAAAAAAA9JEB5zaZTIcmUaU85ahlIrpl6nPhQ5/iE8uMW2T38A58ZASPutAEbhLdpuqk/x8DTZzCLaeB5JbIiQ99gcesXfKjue027MEOI8XIbFIyKyJx6ZYO9aG84o24pSa59+6L4UNv4X6qwq0Y6VyLVJ8yPL7aAqjlpr4PWK/4vCBXXEdlZ5DsenGA5zQ4QB2M9I9ooBN/vACNOl6oxC3LYQITTRs+NIpMjGHcqh4munGXvqNgxCwK4dYDktOzWcpHxPmocMtOMvHmtuNDVXigiFJQZRQVdliwQyU280vFGSFxOSHFUYzzCuKWjaSfAi340DgOJY5bLSuEOUn9jWnES/Wp7qWdCnSdwz6FcsWrK6jsmoakmqcDrICR/hG14wGXIR/J/FtpeKMJ30wSytCmUCwkf3enhE35FBCYRHwbTYyjFBV2hjblRTRP6t1k8ppkuWVc3GiSIB0Xm3z4egPlaahRbXIcpbJz+DfZzcNIBwAAAAAAIAAAAAAAAJL1Ai/minjUGP241UJyb1lwDB/qwSbFuJVoIAllEB8q4hNdkFpekjmA8nQ+HtzaTfIpFdNuKR85lEQzRYXVk9jCj29v182Izu36xMZ+sh8gAXWlnJfUT0HShR+pv6VBtp7SFO0cQOykXKT1M6iRyu5AaFOA5TDSPyY9kk5aBmzATTKUG/ChA9LsMm6RrbxY8ZGHpWXAGG5aifPRL0nLgJT5SMuAOeT5bDAbl5YBKYujl5YBV8iLI4UirdYskhcnFU1EWgZ0UNcnHVmjtAyoZ+Ao2bXiAHNm2QT4TQueA/jgxQQ8B/Dp6z54DgAAAAAAAAAAsh8Ag+FwOBigdGmMIxf6vQDIpI12B4cX50O7F6AiiFyqGJV4pRKZUZ/bbciEei/AKDKJv6BPyYR8wmz3AqgYVpzjhnGAVQAABXKWlpZGaLfgaSPIhXaXmVWDTNy0nabG+dA+/VWE81EzqrAuF5nRXthwMWSiOUjbT8gkwOAPtyGAU2K5G7Aol2HFUc0jTO1AIBAIBAKBQCAQCAQCgbJB1j5iZT52YqCwYRyLntzF8H+TzwdiovBgXBxW9aEM6ONZ0oT3Qb9vWOU55BqRS6SuIncZlJcI5ilikZ9fXzGTm9xLrV452slNzKZUKCM5DFSSXIapoPAYTQ26mJfc5FeZq7C3KPo7c2G5wJzDTKPpBasSFn69OwIAXVfzSLVqlv8omJZrSF3WCuWNOZZV4lielaRMRn8hNnn7OnWOmzrXiF2W5dcE2J7kUevVcnLdsv0ZsUe/PbVG2LBMHslyxqNG1rlpYp+5zCfwK0J5rNRvST98EJqm91vN2REA8N7fRaqXGQC4+ZDU5fo1GQCnbhPH8koGwNEnxCZ/3i0D4Mp1Ypfv0wCo2UWthWspAEwTe/SkAfALeSSdmQC4eYPUpvbZBgB8f3EXI/VkAGDuBr3fbQAAAAAAAABgqz6dZtgxSzO9YLUCAAAAAIDsBIDLnOB5wddL/jyq48UEAAAAAADISgAcWRKCHe32hPiU+M/4GOO3GAAAAAAAgC8FgDJhCL9bq9UTJf7ITYkfAAAAAABkJwDKjRbcFUX2zgjHqXttS/URPIacY0vm8vH2GDcwH7lnuZWra+yynOhGZXYctVkemPSctazeNJ9jHjnMNZ4Ulo5qAQAAAABAFgKgNB5+kDvLcXjguOuMwU5PvILjDkwag6FiFd/AXYlHxUIh5PUE/UKVjnMPCZXVnsJmTm8T7vh9HqPXUJUwxnMcAAAAAAAgCwGgu+cXomdLytQGbraq2KZx9kavuPtKBLPb/cCDAcDvnTANGj0D7sb4MZfeK3hNrnLRp9Xb+FCFuzGhKlB3eaaOGAAAAAAAQDauAlhHcqJG3ni2mXOpgk6O66sqLt9zJaThuO69EgDGOa7Z2MJx+tfxQ7GoD5+QXXwFugJ4auW67YlxmAMAAAAAshcAnFUbu1BSLURLCwTP5NDQkMpY/0h1F0dehQEgqDlu3NjF4dc8FVxI+PeiXynk7+ltxehWYdaWaM0iAFxMq5YcAApdPg2ADJOLxACoVWaiAABKnbYEgMLifBoASqPZCgAKi6wYALUKs/scAHxO5bdlEnBM+qqFq864OMyLQ5Lmx8N23Me3MAD8DgyAqnUANPvDk9KvHNXbPBMcd7DriwBg/8u3smquEwNg//3LKU0vEANg/6u0y3NiAKzJJpd/IgbAn9Imbx9TAaAn7VRzkQIA59MlvrxADIC8tMltGgC8VFZkxQBYmJb97j9kA4DajMqvfQ0ATArSpjXrCaFD4+lUNzU1FTQ4c4O/wTcDS+8BYCXc4sK/csH5RQFwIyzI8i8QA+D5nCqlYA8xADJcfD3EAHgjm6guEwOgplA2ebefCgDT6XDeHKcAwMObsk9hHjEAVtPRrNIA4KqyIisGwE8+2e/mc0ZXAP3pGK9+DQC087sf4WW/oeL5I52JCivXNBgt7z7nGeCsF+LvAcAR8l9AZLCHKzIAcMszst0AEPlwYVK+5zQAeNe/rsvPSQFQi1zeJF0orgDeqOaSJv09NAD4d9Kk5jElAIIppzWaK4CH11Q/p0q8QAGA5VRx8ugAoKjIygEQVK0m/e7vZwYAX6ryL78GAFwWQWzzWlTFdoehV4wvHr0rTMa4ibD/1GBceA8AXK8nujh/VzhWlAGAp8Kde47tBsDb8ynV0gCg5nhStTRXAGtJE4o5gDeqq48VhLIlAM6nQtlFCYA5RU4KAPBKSYm3AsDPiqLZGgBykS8yAsCC0iopB8BNxZXflklAtVfF88ZoBzrRiqZURl6YDKBqNteJYXMLX5oCQEcSANqpKPrtuytcBgA0Ib4ysN0AWFNScAUAUDJ3s+UVwEslsWwFgCfHFZhsCYDHbFYBEAAUOSkAwH1F02NbAOCdkuIoAYASn88CAOtVgH7VrxVXfntWAfQ6V0NMJ72gxaozDTtRzxxsbTis02nbPBpOp7NynKEIv9lkVodGl7XIHXDinw7in9b/1brVhh0DgF0AAADANwWAD6jof+zdT0gb2xoA8PGGSCNabTYiJrTVMR3pje/R1jbFGyedaanM7TxtuUGRIj4krymBEm9DISSRpKBW+qDgxoWYG7qwGALxkb6Z3E1AmE2RgtnN5i1CIdkmEswmi3cm/9SYVGfO+Ofem29jqeOXOWcyvzlzzpfJovkeormjf9yOnEOcKgDQtwDUgVsAEuoW4CQjv2MB8F68W4CTjLrP7BbAckq3ACQlGwDnfAtQe3HQpEWtSnS5/8IAwO6UIgoFQGMV4NhVgC0SDoCLuQrgkQmAP90qQJ2HhNyZ//UfPedz/tcGQKktRRoKANRQDNoBMwI4UZbjACgnMQRgACgnyXthAShnckICUG4XkYQAoNI5kABU8thlAqCcMCcjAJWevzgAnGvUBMBFF4MfgQLA7i+GzwszAgicJMtxAOQzpSwxGADKSaI2SAD4SiYKDoB0KY+/EwKAjvLecHAAnKiTxQCwVu4lRjYAgidq618dgHisFExjFaAxCSh+EpBqTAL+sQForAI0AGisApzGx4FnS3G/+qGA/YP3632/c/vs/jYbGsVd918IgMYIoAHAnwmAKUxZDONYNQ1PzfXKe15ipfOsZd68MjhNdDVGAA0AGgD8MQHQWnsK8ar6bO/bPhkA3T2L104fgHClFLjQXVTtRdNjAaikKfw5xVhsUkYAI4eyMBZGCgDOzoMNIi0WSQBEykmo4q6QkgGIHOrfOplOAEDAW+kckAXkIyUAkD2wN8x++0QDsHWgk8nCTxscAL79ivT6rRMHwEJVW+unPB0AtutVA50QAGSjpf/0AbDmylF4C8cSdosUABacxWB9xSxsRsoIoKOcRbgeUAF2zyYBAL6UxFl4iSSbYKQAsFVKEhC6wxJnR6QCYCjvTlxoTKedVUsEoNzFTr+QhWUT9oxNNAB4OcmeyrYLkrDpXUYKAHQ5Dxi0ZQp5wjEoAPKlfJ6YiknXa504AIhKz6vIYkoHeS4AXH27udS2+Grq9Z3txc1ZAEDo38OrPRPC8wIUb3snb94Dx91969Xk/NgQAEDT/GJ1+/OAeeXS180fkYl3S1/W158o+pHWB8OTT9+8eXUJufR++0rv7+39sgCwH4VCgLDWlJQCwOFCoB0UTZASRgCHCoE4Hq25KHziQiDhNPKyKOqQAkClEEj4rY838IzUOoBDhUAO2pAlpQFwoBAolguy7FaQT1JiAThQCESmCSfr7CDsNgkAHKy22iXyCTaLOyMwABwoBPLyfIJNBb9TDCK6EIgJ0gnQYXTyXAB4PWBEQzTm6tEPmbW9AABM7/rFZHrUj7Ss69BvIdOHDWS5SfeSMBkBAA9eYvSQUV+aA3howk1DJqznBnKZxh7jRpdSoXlitK66sJlBeAA6w4H9EK7ZJE+Yat3efR8Ay9pIOXaF64AtlcvTEbEjAO/OoSzUCM/iatEAJCtJRvzCSUU7g2lSLADR/SRJ4cQPo+majBwPgP9AJvD+t9nxBO2XAACz38UjHAAgzahsDtxOigPAsZ8kAwAIgmPEZbMR0QAc7uRdHvSNpUa7TgwAt1vJp/YCAFjhgR6EBwqAzP4+OgAATpAyQyTOdA5guxDvAADab9cVL5Shvysuu/SDfdva5+Pdt1HDg9YP2MDd7v/+Glq5Ool9Hbz/UItpWuexd+2KJ1gZACX6YHDUYBrf+Fn3dfDuO61S4cb1XTdmp63/gwfgaM/qA1smTiwA1cc9ysfV6I5YAKpxcqY43mkTC0D1xgZfolZ1qahnAjK5lG8uzUgBoJpKns3wAZt4AKr2XgBA5afDNnEAHIoiAJFUyiIagMNRAID0LEQlA1D1NhEAUPmIMBQAh4+gAIAqigfOEoBSXAEAmN73I1PY6mvEPWPs6ts2vulH3G3YZ8WqdQls+la5+ZP1Obig/xjCNFcNLvAvxccKAE/AFtvKlWbrajeC3GhSKq7ixt/uI13Lg/IDQKb1mSQWhwTAFuYzXIeTgQPAh4dVLBGBA4DJ5Rm1YQ0SAD8+YmE7OBkAcKBJSz4bgQdgy+FY28pzKigAeLUj6QkmVbAA0PGMIx4MM7IBkM+A1mU5OQHIgpTOXPQsAZgZLMQzAIALXK1HjeBV3E+N433brstgg/e6NjCqJ4aGhlBl7239TWFK4DmmUWi/CZOAPWUAMGEVcVj5rwnTQ6EqYB3cAjwKYforn65r5AcgZp2zxfAaZ50oADrzKYa01zgwogAIEJzKYRiBAyBDj1AWOkVBAUDawcUtWWtIIxYAJpGNqfZQBzwAxNxc0LCghhsBoMG5II7HO2EBMPBzczSa4GQDwFBoXZKUEYBCyrk15nzmAKwPBAB6KgD8BP53TPfiFqpfnBFic8w0LBy4aUwzq1wUPjQ0XAZAeEYwGAcsvzcWABhQKpCWiXmrUkufwi2AWmvnOKdRDQeAzxCmwC3c0XVrMQBYclsWiss5LTAAkIE5H0UlDDEoAGJZJ0N5iRQ8ANEFD0lxRBoeAGfGn1HncD/cCCDp9zsSRNwGOwII+/2ZOJqyyTYCAPnWsnxGzhEASKnO1plYPGsAQiugXz9hX2c/4uNut/vu7euXXeuvwW9xTLPhIjaEP6kGYDQkbLFBKxUto7fcfUsDyleyA0DmtDqjUaf9DwkFgKfJQBCGpg4SBoAkjhIEgfJ+GABiHYUkNWpoxACQRIUsTQQHDcBIMVMwJsscgMp/9AOPoucAVMxcjpFhDkBFJVBOzjkAVQavN1kidQ5A5aft5EUAAHu3gbRPhqb6enUfEKT1b/pPfbge9PmyDtO4p3UTCHLZXA1A92PTFKK5gykV7SH8BuhO5bzsAETNC2tqtZrQxWAA6MxlwyCcaBQCANLOB0ASOxqHAcCBpkGS+NyWFwaABB0HWdLEHjQAeV7omQS+Jg8AUTkAoBY6ZAFA5ZEZAJ/8AET5swQg9LwYj55VA6DFfvj0UQuG9M1mbc/v0zq6Gblndm2+MGsxDbJEhB5+acKqAUAmzMb1VSMAwD2Dfdx8QpjuyQ2AzV4c/O9iaQoCADW9U9LWJh2AWJAV9sG2ECSlA0CxfOEtHsAzEABwfGEdMZZNReAAoLhCaYKKW2AtsAAkIt5IlDU44ADgvJ1cuEYhgFgAkt7OmAPvoGQDwOuN+Nj6j4WRAMAWSBlN4MkzuwVYNupK0fbsqWEcQVZCYMzunjd39f1gvRnS6YeFrw2/N6nDdJOjrcjGZytmXP9m0iCtY6hO98tA09LgzFAz0qYX3mNtuhXk6ti0mf4nrexDunrBHxG/9ckNgAV3FeaDOCsRkw4AA95Xws9IKhuTDkCy9NybAOqXDkAsWKwC9DcFIADYodVFHo/ecYsEIFyEyMbSHCwAfCqVX8A9DBQAhnwqn8NTHOwkIJFLpbL0gkO2SUAatG6OqFmTKhUAIeUC4bGcGQDfj75r5c8IDjaXHvzZ0lVe19u4VqPK79KbKXBonzWZhK27x9slPC30OAAiW8VBLrnHRqUDYInvFa8EySPfESACALW9eFpydodkAKiovXjKUva4dADItXixwVFPBg4AZjdcfCm/xwcHgDcufD/AXo1HjIgBQC0kCdcqjxUHgN8OEsV3jp5bEgFgAvVaJxkAWzGlz6a6IACIj0u4a/SG4gv2s/TPBzS+HFTaHED9aHw56PcF/j975/PSSNLG8TZNh5DJmN3TEJJgJmrSwV/DaCcrGmMk4sFEoygTRCRDYCZe5I1ZAi/JOwfBUZmA4MXDsPjiwUERXl/WgZ37XmTw8N7m4mn/gr3sXt+qnvSP+Dpjz1P1uqPzfA9LsdLfeeqprk+6qrqrvu3DQTlrOy53HEu/MXwhjABAACAAbi0AOmNd+cJJgOEDwZF/PoDqXyYAvPgF6vLrBx0A7X+AY/mbAYC/g01+MQHgV7DLnwYA/vOAWb8faQD4B9jjJwMAL+GRHJkB8OJHqM3P/24CwJ8/P+Ckn8wA+JHd749vY0uwk8x3UH3Y1MFTWnwJdXk5NqjD6AM4lsy2ZpKFm7xo12ZcBvMv4aHoL2M9hYdixLTQAMB4AuxxNKLtIpVchUfyvek5M7JxBPZJ9Jtuv2eJ73jpKGVss7VwxO734f43AQDFDpfx4UFtGO4ScGkuIkMsXn1eJAA3iWqPNLVBhvoc8sitrlKDsmEGD1EjdRtDO5XMe9Gx5Me8UuUM2LlJjPAIz+QnoFAoFAqFQqFQKBQKhUKhUCgUCoVCoVAoFAqFQqFQKBQKhUKhUCgUCoVC3SY56BEGLkaTGjVxMppEwsSkzHosaicNpcajPp2cMuzyEjMHq0uZmIRZ26mN1uuQvUo+6uPgeYIt14xzt7vDctlCoVD1CaNLticUiqdZu+4YMXndzegyQupTXGI06S+SUFKcUvyWmq2zumyQemVaGU3W4qHQ6B57lUqjpEpBnh1saZ5jxgVhiwaY8mH/vh6Vk5Ikje4yulT8kuR/zvos0kNMJqKMLu2kPj1ZRpOBURLKOqcUT8eJ2SSryxCpV/Exo8k2bacz9ip56KHSvbMc78OczDHjgrBDA1x3Yf9GACAAEACoK+UUiaIbFAAxWvRCxnQuhV56Tm+sFVoSQcNvB71ymPa6oSQtgkbMbeq/X6AAWKMlBfIE6FPrU6d9th1eH21Ww0sdctRsQzU7BLeTmCH1qi7R0iy8nc5oO/WpyYHWS/WZpv3rIECLrDM/jYxXZC4ZJzlX7bI0wPduWmzDnv4Jvbq3uLg4RX52JTlDzzFZLwNMAgV66Ri9sarqaSigH6k+euWqTFm0TItpyE3QP0cvPab1SdCSDfIwMWijlybo7Tiv1meHIcNKkDpkaHJ6VDPQk8mpeimhiBQaoqUU5Je3NU8v7aChFGlpMgmsUy+9eoL4SMerpDQ3wDoVGG1XM+7XM842gVM+0HIuqXZz/djTP6GnH08xp6xUCwnI/Nvj+WYTKQcJZUM3UV2kdshP5Z58qT6QfZSfJJpN/HWWeztzKTmgedKR5uT4f4NsY7XT0xzKcQxYp+PLVWKdabO3XHKsM9kpo5fsTrGnfxIAUpOAAGg2gQKg2QQIgEv1gQGg2YQVAM1uQAA0mwAB0GwCBsAlHx4AuOTICoBLdggABAACAAGA+h+Nj9IDDOnd5VePMlz0QsaWY/RS2vX8Mi3FQWPmmVBTKPEVyBzAq9HmUPaHIXMa+6oJDeVjfbIMGRbn1HoZyQHdjCfNyQkdKACTi2JzvYrTwDoltCo1muqcdQ5geOLSHZRlA8APml2jAQewp39Ch/Rtqe68ugpAi7DZZfqam7JHp3BGFPBLeOrriIPqKsAT8Dtc6mtuiroKUKGlMGgVgL6OqKypqwAK60uFkVnqEKNmeQX8Ep5TvZSuAhR3wG/guVSTNG2nTVryQuul5mdXXQUocXmv8GPGX8lcMk5XXqhHTn2OFGkRVwE+K3wP4CrhewCfF74HgABAACAAEAB3AAAHFABbHAAgsZ6y5KBL+MusALBRAOQYTbJ0JinIKcW71OyA1WWZAiDJaDJO2klOs1fJrr5nU+Z4H8ZCHDMuCM9ogEEEgAV1J5NJD+tMrrOVuIQZTSIBYlJiDUUhJq2s4z6nh7gonDLcZidmzEdQRYlJgPUrx05i8sTBXiUfbe9unrch14wLwiFfOxQKhUKhUCgUCoVCoVAoFAqFQqFQKBQKhUKhUCgUCoVCoVAoFAqFQqFQqJtUpzsKldv4dMQnRuE2+iYWDgYTfQvqGrxCUX33c5/CEIqLQ24NKY2YnDwaysXQTqL5WyyG/HSbP8eqdUd5yV3mEt5Vt/dd1vk9uMb1ruvuhbssuDWXM4ZY1jSTpQ24SUrb+cwdhJts6Iepjd/joJlGf6kweJxpnwp6CgyBmLdRszG0t3mHibcL97gpbZBlhoffu28CACMndqjeBPXfhMDcLtRla1HfoHNmGxxLX59mUp8Bm+zc01gUWLwAu2zou813pe3Myk01nm2epsAej3q1rjH9GzySfdOWI5GpAajNw4J5J75Y3mPnpHq78ZO9OsDut237NgCw7YCqbgLAwjDUpbSgA8C2Bo4lrQPgURBs8tAAwFwU7DJpAGDPwaytOQ0AJ2CPrAGA1/BIpswAmNuB2sy2NwGgEHZwUs4EgMVn7H4VBAACAAGAAOApl+rsdF69e0vksJP0KB8ZWLc5XAgABAAC4K4BIKYeRyDPB6/cadN70PJEEMcfC66u+DQCAAGAALh7AKgWCgf5oj8TuOKvs7YJj5A+jgmu59UtBAACAAFw9wCg7m4cfe1/etUJDk5HREiNxugQwIcAQAAgAO4oACJ7sq0z4j5db08nyWC/FhsprFREwflqWxxYlmfWHLGzkhAR68H3Z8maIOy885yvrz8qIwAQAAiAOwGAcTl46Fn0/zAUmh+IROpydarq7y3TOYCVY38xqNA5APuCv2MiNF+hDwVD8cSo3FVGACAAEAC3GwDv3e5oKZYI7c3OhFbsg6fzQx5loRhzJ18XdygAxGC8LtYIADrXQ12ewUo1kRRSUnVpOFc8bkUAIAAQALcbAD2ZTKYqy73l4eMhkQz2R/yn4uL8uehL5tzqKgCdA6CrAO75MXI31u7L25GURE9s6pV2EQAIAATALQfAxMTERrCiCDv+Gfp/BuSU70083lGoD0eaALDlL9C/Z0MrrpScpT1Vena7AFBWdHnLYAB4DZdZMADChkkYDABzfdgAMGvR6VoAhK0l5/MAmLWWnOsBYK2prAPAamDWAFA2hVf+i+cAqPqlLvX/yEGhVm+v+qX9t00A2JXUf38pbqulQjFSSt0YALbGDWUVMACm+55rejMIBcBwWjfZvAADYE83eT4ABsDSfd3kXTcTAHJGOOdhBgC4jeTcnwYDoGJEU2cBwPCZ0VRveQDAFNgjdgC0nuhuJ61fAQBa5XyNJPJc3uz02MvR2IacEs0AGJa/p38/lVOumwZAOmQoXwID4HSsRdPGQygAHu/rJpksGAAF3aSlDwyA8Q7dZLLEBIBNI5xehQEAgUXdp6MCBkCXEU2QBQBvTU2V4wEAq4FZA0D/hO62v/MVACCaqJIxvZgPDSSX50r01OmgWwVAPKsCQMkc7wiCchBai9w0AM4keSzTUDDKAoCu1Ee9i0IBQO6qQsOkL8kAgKmRhksWDIDtjhbNZFxkBEBCc9oLswEg3/B5vgUGQLBlQoumwgKAC2tN9SUAGNICW+MAgKGW9obbiecrAIDrdLTl3dqCvOL1vve/3qsvj1bUIUDavzpOlwF9az3FdCUvBxXh5gFQnHY3JJZZAPDQwkj3WgDsWRm5XQcAW7eFsfK1AIjymQPYbNmPWh3cXgOAMyvJuQ4Ak24ecwAX1prqSwCQd/ObAyAAyFqYkvo/A8BvHHNe3iYD//i6IggPF2TJP58uhwsdrYKnQ0q0rsR3Bcde0S+FbN2CBgD/9I0BoOUxh0lAAoAA8yoAAcAp+yoAAYBiweRaAIh8VgE2W1YtOVkAQHqWeRWAAMBKcqwAwMpQ/YsAoDis63oA5P7yVQBXp2k7JJ/TfiGq3wW2uXftTp8QOST/ERwB0ddGCz5nYMtN/15z0m8Da50+BAACAAFwmwFwO/YD4AcAj4VHQutDgFkWAHQzLwMSAAxqoXAbAngZAXAmMi8D/pe9O2hJJdoDAD41KEOl0lJUmtR0orRHOqVMNipFC52riaKIiBfhmptAB9zkKhANg8CNCxcXXCiBCxe26COE9BFc9UXeGXXKyu5t5syz7sP/IgLH4/HMzM+Zc878z8QtgEaqW4CQRACoPttMnwLA/uXDgP8KAB3/5SgKOSgA4EcB7r7PKMDN/+kogF6iUYDBNx8FoL7BKMC/AoBMphuFqQYFgMM7ipQcAgC+ECoGAYBxXIi3AgMAX5UWNAB8dfqQAPDfi4YB4LlxtiAB4NvnRCIAnismDQB89X7MAfgLACYqP4pVDxQA9uQo6iQEAJVxITUnBAAt27gUJQwA++NCrkhIAE74kgYsHACH43KSBxAAxPnaeCAB+MyuEgLA6qcq9mkA2uPSAqo5AH8ZBtznU6OG5p2A807AeSfgvBNwDsAcgDkA0LGUG4dz6c0rm6Rz7YM3qXIv29xq0b3iHIA5AHMA/kkAkoRsFK7gWxpOI54P3pQlxufZUilSJ1Ne9RyAOQBzAP5NAHRn+mGcvz3bN/qfA0CjL+Mz6AOoPa+QBvZPCE8m181iAPDwy8END9PoNS4GgOPnlfswsv6oODreDQkHwHDAr3CHYdu29tFhTykCADn/fUZdd2jQrhIHQBN/qQ4a7HFR2BMDQNfJLyE4bEvbzbTnFP4GQOqlNubksDK9S1wMABO7Ctver2x1bSQUAOWJZjIrj48ekigMAFV+1b/hZw4eh5Vj8cuMf5+cFQD9D175LADI7dLmDEYB8qlxtHYx1pYlTAuFkAgA0qejUOQwFr07z9fEABAfF6KvY54m3Vg2ThvG/RsAJ/1xKdcYVqWZVMq4qhQOgHVcSGV03t8wKbU4ALx8dboaLJc2MhRFxT1iAEgYRuUsJsGhPDh1hFNWp2AAGL429xh5RHHhzSbFADCxq0gF7dMv048kDADPFWuDo7BxYl1m2iQEAOVxaVugqffu+vRwH6uttC9MH5OzBmDtsnulKJ8nWXu/3M0BADrXhw197Ba8hF4a4lv7YL8Xd8/jpSAFANDKM41+zxqpY4/dbSRWuXpotfzoJrLiOYyfFgrnGIJV+6uGe9WmBAD4ZS/hSmK5sDdWK9N1EQBMTARSZVbppigAJiYCqbsBHL+k9KhgACYnAsVuPAcH916/cABeTQTSBLO0WAAmJwLl0taBWq2Wk2IAmJwIhKfCBfkBHhIMwMR8GzYHqqLeLT0NxAAwMRHIdpLBt6OGhgcGgImK7aWWA071IhOAAGByIlB61TsC4IG+znlK4fqsAWCtLmOHIS70FiqiMwAACMtF2O0+3kSWWibjU8d9c4vUFkxZr9sFAPBkCYZyWcZ9AEdu2k25Cb0ZUTLECe26kKFav+uscUGkSQkACHRfwo9jNt0Dhu1bHkICARg8PqcVKagwMlndEnEFkOs9F/J7fLLh6RYuEIDqS4YT/petdqIQCED9pZAqJ1AyfNgXB0DypSR7CACwuCeuD2D7+qUgcKI90lUxfQCxl0L4m+QapdAIBuD1rqoy9yGMzSSuxAPwqmKDhZ9cy9E7YgGQv1SvB46fYNUwBECVSoNLphj9yM6oD6A/jAoAQPe0jmZknf+gygsLudHXJaIam9HhWbkhrHuau3ynvhYnHknnkY7QrpSIigr1EzwAMqOHDDjc0Vuf6ZHcq+hkaJG2qM251NlAAgDenls6oCMZSZMCAXgX9yIAmBKexqJKIABvQ6P+pc8GBALwOlhPw5DbEgfAmwZIp21XUVQEAG84SPts+/Zpc3CE5gQkS1ml8E7A16GMZ6v4ru99PgmROQGTC/dcqcsGViQA7yIzBCCaOAIHtpxRhGYDwDhWAQDu6iaSJBosUky71Bt9V2GTywHaQxtn3LJAl7LuwVkC/KBvdwjtmuMC/Ic2nwHwgy36srr8rKFBEPOCDF2jXb+diLpGSg9Axs1dxnmf0G8CgN/xXmuBAMh9dP4SgwIAP/XtmiUBwJl20PRyBocFQF6mssYF+hyHBiBGdTWwALD2PJ1u+XYxiQCQe/uAfXX54+3FAVBv/gTfVUXpzbMBID1+WgoAcAF+rQMu8CnFU1d0o3/BZf2umhTgqt5LUZRRZrBZtrgugQShRXVPXCegngeAzxIacx9xswJa4BbguENYVtvrWukBOOxEwV/qx/cAgLxkTlEMEgBnOxNfvoIBgNwKD1hpAECr9v2C3qs3QwJwlfAFceUOvUNCApDrl6MYLADb/h8/S9mTICkRAOQO3apk4o4SJi0AtWwX1AClSpqZ9wGceTgA9M8AHCBcEuDMrtFSTnPRDboPuR2XIrQ5WZnLCHDIAzBOEVSruoYAWGUoshQrncl0zP/gFuDn8ArA0fgetwCFbNqDwQIAYj9/SEIAsJ4tK3YUTyeHdmgARiRZf+CQAAzCGfCN9sI+FSQAVUebhQWALVBtMxlLZWsSAYDljltP6SNaITEAg3wFnPpO6tz81QB06qBd28RjrklHi8Xinm1dedFiwas0ob298N5yb3kLQKDDbXHLyNClwG5x48oqO5cegJ4O7ERV5/Q7AIA+UotTP0kwAPJWPwcBwIGe68o5YVrH0gBAHjXvIAE4SJW4SvveV10YAHg5NbWJBQFAZriudrZn/C0VAMMs4QVjVWIA8KctcEE5YLqhrwaAqNwiqngnuWEw3SDIyo6lvUFbQJvXTIS2mDLFEEQZeQuA5sSdRLR2QoaqOrT5Vc5B6QAImMChF3T72a8HgPQzW9OfLBYAAItzV3u/mnqYK4AQlzDRqU8NUGgAtrkc485UwgkJALmYB4bgeQMKBUDokZmeq1wQAOYuxaVv6DkKkgHAna2+ck5iAEjDkxxjfzPVGQ0DdhKjOA69BUBHLLabOnBJL4/o9PcpEyNH9iMX3UxER2iRK2/n6GGBeAsAEou4Wg0XAKCYJppdv5dbWlhqALYNnXaPoTwYFADcMCD1UI1CAVCgs10/iAEMAOh52V84ztM2qE7A4VEuSR+A/aRyeZ32HsN2AmJ1KlWoWrNBFgqAaCI9/bsL6wP4lV8u1HoJq1oqADy9qj+xHGMlAsBeNXiv7eDXJMgYCu1mH58JADUXn29fETp1RBGk3gHX7MVSRL2xeLbVMVkOt7lGiZsIUzywgtz2zghX68mtRVaCRpMpbF24ItOUHFFYuGNMYaoja8FUhPnJyDYQtQG8yft7Q3oA2Luy2+0NhuAA2CvRXPSgALikhoVMyX4hAADyepX20vHrEDQAofP0OjwA9RaoTvajzOkCACALYYZJFEi4UYCbj3KLCAMgZC9TCcrwPt+BWAD2mzR1mvzD1sIAyHPHEWUHvwc3WZr21dmZAPDn2MD5ZwRJOTo6t5bU/LjeLT5llh9WSIJdG1pwc1troioRaUM/szQYm1ufNk3tn10bMJS7m/Zsw5etDej0KD+c4SpobUBSrZ6WPe9r1gbUqK9yU57aEAsAGY3++dEg0WsDksqD/7Z39yoNQ2EYx6FBKVIindtB/KCVFrfi4sfiZqgfCIUgEhDs3sHF7hUidOzQQeigi4NDHbyEIr0Kb8SlOTlCEXnfAyX0/7uAh5cmfRoazjnzniuysB9A/vjhrOJNqqfy9QEcDrr4Avjz5R6Hg/5TFg4Hda5/WN1+W/lWrBCmACgACiCzBbCzX4tu3zcUCwSPHn2pwCqA6ztpyqBjCuDiVTxL2xTA05Y45CYtgOZAnDJMC2Diq513kgI4EWeM0wK4kk8ysgugGUtjWi+/CqDX8h0ZWwUwivV5n8uxJdhzUSycmuJZ/5LH9OpJSqCY5d68aYnkKe3KLKRek4dEfjLKNCzq5WZf3liRESS7SK3uKQY5sG6bTXnOcGDlfPSKroRBuqdWzkVgfykKYM2TS185lBuKGFMjeUWI2WOtqwi5TEYplxQpBfN85jlQ2tVfKPPTWFBcp4b9F7Pm87H3tOt67uTdjDfn9gYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIvxA2vpz35XAE/FAAAAAElFTkSuQmCC\n",
"text/plain": [
"<IPython.core.display.Image object>"
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"from IPython.display import Image\n",
"Image(filename='./samples/BERT-embeddings-2.png') "
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## summary\n",
"\n",
"- bert input embedding：一种查表操作（lookup table）\n",
" - 查表\n",
" - token embeddings：30522*768\n",
" - segment embeddings：2*768\n",
" - position embeddings: 512*768\n",
" - 后处理\n",
" - layer norm\n",
" - dropout"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {
"ExecuteTime": {
"end_time": "2022-07-03T04:30:13.318068Z",
"start_time": "2022-07-03T04:30:13.314599Z"
}
},
"outputs": [],
"source": [
"from transformers import BertTokenizer, BertModel\n",
"import torch \n",
"from torch import nn"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {
"ExecuteTime": {
"end_time": "2022-07-03T04:30:27.736594Z",
"start_time": "2022-07-03T04:30:16.308462Z"
}
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertModel: ['cls.seq_relationship.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.bias', 'cls.seq_relationship.weight']\n",
"- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
"- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
]
}
],
"source": [
"model_name = 'bert-base-uncased'\n",
"tokenizer = BertTokenizer.from_pretrained(model_name)\n",
"model = BertModel.from_pretrained(model_name)"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {
"ExecuteTime": {
"end_time": "2022-07-03T04:30:27.741720Z",
"start_time": "2022-07-03T04:30:27.739010Z"
}
},
"outputs": [],
"source": [
"test_sent = 'this is a test sentence'"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {
"ExecuteTime": {
"end_time": "2022-07-03T04:30:32.350227Z",
"start_time": "2022-07-03T04:30:32.346899Z"
}
},
"outputs": [],
"source": [
"input = tokenizer(test_sent, return_tensors='pt')"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {
"ExecuteTime": {
"end_time": "2022-07-03T04:30:53.366612Z",
"start_time": "2022-07-03T04:30:53.361672Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"{'input_ids': tensor([[ 101, 2023, 2003, 1037, 3231, 6251, 102]]), 'token_type_ids': tensor([[0, 0, 0, 0, 0, 0, 0]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1]])}"
]
},
"execution_count": 18,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"input"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {
"ExecuteTime": {
"end_time": "2022-07-03T04:17:29.842351Z",
"start_time": "2022-07-03T04:17:29.839938Z"
}
},
"outputs": [],
"source": [
"input_ids = input['input_ids']\n",
"token_type_ids = input['token_type_ids']"
]
},
{
"cell_type": "code",
"execution_count": 22,
"metadata": {
"ExecuteTime": {
"end_time": "2022-07-03T04:33:05.156796Z",
"start_time": "2022-07-03T04:33:05.153169Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"torch.Size([1, 7])"
]
},
"execution_count": 22,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"input_ids.shape"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {
"ExecuteTime": {
"end_time": "2022-07-03T04:32:38.322884Z",
"start_time": "2022-07-03T04:32:38.320215Z"
}
},
"outputs": [],
"source": [
"pos_ids = torch.arange(input_ids.shape[1])"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {
"ExecuteTime": {
"end_time": "2022-07-03T04:32:41.882032Z",
"start_time": "2022-07-03T04:32:41.878152Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"tensor([0, 1, 2, 3, 4, 5, 6])"
]
},
"execution_count": 20,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"pos_ids"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 1. token embedding"
]
},
{
"cell_type": "code",
"execution_count": 24,
"metadata": {
"ExecuteTime": {
"end_time": "2022-07-03T04:34:02.215913Z",
"start_time": "2022-07-03T04:34:02.212945Z"
}
},
"outputs": [],
"source": [
"token_embed = model.embeddings.word_embeddings(input_ids)"
]
},
{
"cell_type": "code",
"execution_count": 31,
"metadata": {
"ExecuteTime": {
"end_time": "2022-07-03T04:36:27.730990Z",
"start_time": "2022-07-03T04:36:27.726189Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"torch.Size([1, 7, 768])"
]
},
"execution_count": 31,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"token_embed.shape"
]
},
{
"cell_type": "code",
"execution_count": 36,
"metadata": {
"ExecuteTime": {
"end_time": "2022-07-03T04:40:37.818118Z",
"start_time": "2022-07-03T04:40:37.812608Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"tensor([[[ 1.3630e-02, -2.6490e-02, -2.3503e-02, ..., 8.6805e-03,\n",
" 7.1340e-03, 1.5147e-02],\n",
" [-5.7095e-02, 1.5283e-02, -4.6868e-03, ..., -3.2484e-03,\n",
" 9.7317e-05, 9.4175e-03],\n",
" [-3.6044e-02, -2.4606e-02, -2.5735e-02, ..., 3.3691e-03,\n",
" -1.8300e-03, 2.6855e-02],\n",
" ...,\n",
" [ 2.3670e-02, -6.1351e-02, -2.9575e-02, ..., -1.0239e-02,\n",
" -7.2316e-03, -1.1745e-01],\n",
" [ 3.2079e-02, 6.3135e-03, -6.4352e-03, ..., -1.1689e-03,\n",
" -1.0810e-01, -8.9524e-02],\n",
" [-1.4521e-02, -9.9615e-03, 6.0263e-03, ..., -2.5035e-02,\n",
" 4.6379e-03, -1.5378e-03]]], grad_fn=<EmbeddingBackward0>)"
]
},
"execution_count": 36,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"token_embed"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 2. segment type embedding"
]
},
{
"cell_type": "code",
"execution_count": 26,
"metadata": {
"ExecuteTime": {
"end_time": "2022-07-03T04:35:22.076984Z",
"start_time": "2022-07-03T04:35:22.074266Z"
}
},
"outputs": [],
"source": [
"seg_embed = model.embeddings.token_type_embeddings(token_type_ids)"
]
},
{
"cell_type": "code",
"execution_count": 30,
"metadata": {
"ExecuteTime": {
"end_time": "2022-07-03T04:36:11.922415Z",
"start_time": "2022-07-03T04:36:11.917723Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"torch.Size([1, 7, 768])"
]
},
"execution_count": 30,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"seg_embed.shape"
]
},
{
"cell_type": "code",
"execution_count": 37,
"metadata": {
"ExecuteTime": {
"end_time": "2022-07-03T04:42:08.656111Z",
"start_time": "2022-07-03T04:42:08.651050Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"tensor([[[ 0.0004, 0.0110, 0.0037, ..., -0.0066, -0.0034, -0.0086],\n",
" [ 0.0004, 0.0110, 0.0037, ..., -0.0066, -0.0034, -0.0086],\n",
" [ 0.0004, 0.0110, 0.0037, ..., -0.0066, -0.0034, -0.0086],\n",
" ...,\n",
" [ 0.0004, 0.0110, 0.0037, ..., -0.0066, -0.0034, -0.0086],\n",
" [ 0.0004, 0.0110, 0.0037, ..., -0.0066, -0.0034, -0.0086],\n",
" [ 0.0004, 0.0110, 0.0037, ..., -0.0066, -0.0034, -0.0086]]],\n",
" grad_fn=<EmbeddingBackward0>)"
]
},
"execution_count": 37,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"seg_embed"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 3. pos embedding"
]
},
{
"cell_type": "code",
"execution_count": 27,
"metadata": {
"ExecuteTime": {
"end_time": "2022-07-03T04:35:46.864620Z",
"start_time": "2022-07-03T04:35:46.862083Z"
}
},
"outputs": [],
"source": [
"pos_embed = model.embeddings.position_embeddings(pos_ids)"
]
},
{
"cell_type": "code",
"execution_count": 29,
"metadata": {
"ExecuteTime": {
"end_time": "2022-07-03T04:36:02.611104Z",
"start_time": "2022-07-03T04:36:02.606687Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"torch.Size([7, 768])"
]
},
"execution_count": 29,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"pos_embed.shape"
]
},
{
"cell_type": "code",
"execution_count": 38,
"metadata": {
"ExecuteTime": {
"end_time": "2022-07-03T04:42:45.328938Z",
"start_time": "2022-07-03T04:42:45.323868Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"tensor([[ 1.7505e-02, -2.5631e-02, -3.6642e-02, ..., 3.3437e-05,\n",
" 6.8312e-04, 1.5441e-02],\n",
" [ 7.7580e-03, 2.2613e-03, -1.9444e-02, ..., 2.8910e-02,\n",
" 2.9753e-02, -5.3247e-03],\n",
" [-1.1287e-02, -1.9644e-03, -1.1573e-02, ..., 1.4908e-02,\n",
" 1.8741e-02, -7.3140e-03],\n",
" ...,\n",
" [-5.6087e-03, -1.0445e-02, -7.2288e-03, ..., 2.0837e-02,\n",
" 3.5402e-03, 4.7708e-03],\n",
" [-3.0871e-03, -1.8956e-02, -1.8930e-02, ..., 7.4045e-03,\n",
" 2.0183e-02, 3.4077e-03],\n",
" [ 6.4257e-03, -1.7664e-02, -2.2067e-02, ..., 6.7531e-04,\n",
" 1.1108e-02, 3.7521e-03]], grad_fn=<EmbeddingBackward0>)"
]
},
"execution_count": 38,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"pos_embed"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 4. input embedding"
]
},
{
"cell_type": "code",
"execution_count": 34,
"metadata": {
"ExecuteTime": {
"end_time": "2022-07-03T04:37:13.880241Z",
"start_time": "2022-07-03T04:37:13.877608Z"
}
},
"outputs": [],
"source": [
"input_embed = token_embed + seg_embed + pos_embed.unsqueeze(0)"
]
},
{
"cell_type": "code",
"execution_count": 35,
"metadata": {
"ExecuteTime": {
"end_time": "2022-07-03T04:37:15.108932Z",
"start_time": "2022-07-03T04:37:15.103221Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"tensor([[[ 0.0316, -0.0411, -0.0564, ..., 0.0021, 0.0044, 0.0219],\n",
" [-0.0489, 0.0285, -0.0204, ..., 0.0190, 0.0265, -0.0045],\n",
" [-0.0469, -0.0156, -0.0336, ..., 0.0117, 0.0135, 0.0109],\n",
" ...,\n",
" [ 0.0185, -0.0608, -0.0331, ..., 0.0040, -0.0071, -0.1213],\n",
" [ 0.0294, -0.0017, -0.0217, ..., -0.0004, -0.0913, -0.0948],\n",
" [-0.0077, -0.0166, -0.0123, ..., -0.0310, 0.0124, -0.0064]]],\n",
" grad_fn=<AddBackward0>)"
]
},
"execution_count": 35,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"input_embed"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 5. 后处理"
]
},
{
"cell_type": "code",
"execution_count": 39,
"metadata": {
"ExecuteTime": {
"end_time": "2022-07-03T04:43:51.050970Z",
"start_time": "2022-07-03T04:43:51.048456Z"
}
},
"outputs": [],
"source": [
"embed = model.embeddings.LayerNorm(input_embed)"
]
},
{
"cell_type": "code",
"execution_count": 40,
"metadata": {
"ExecuteTime": {
"end_time": "2022-07-03T04:44:01.356938Z",
"start_time": "2022-07-03T04:44:01.354326Z"
}
},
"outputs": [],
"source": [
"embed = model.embeddings.dropout(embed)"
]
},
{
"cell_type": "code",
"execution_count": 41,
"metadata": {
"ExecuteTime": {
"end_time": "2022-07-03T04:44:03.779315Z",
"start_time": "2022-07-03T04:44:03.774460Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"tensor([[[ 0.1686, -0.2858, -0.3261, ..., -0.0276, 0.0383, 0.1640],\n",
" [-0.6485, 0.6739, -0.0932, ..., 0.4475, 0.6696, 0.1820],\n",
" [-0.6270, -0.0633, -0.3143, ..., 0.3427, 0.4636, 0.4594],\n",
" ...,\n",
" [ 0.6010, -0.6970, -0.2001, ..., 0.2960, 0.2060, -1.7181],\n",
" [ 0.8323, 0.2878, 0.0021, ..., 0.2628, -1.1310, -1.2708],\n",
" [-0.1481, -0.2948, -0.1690, ..., -0.5009, 0.2544, -0.0700]]],\n",
" grad_fn=<NativeLayerNormBackward0>)"
]
},
"execution_count": 41,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"embed"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.8"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
|
