1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
|
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"ExecuteTime": {
"end_time": "2022-08-23T14:15:27.053371Z",
"start_time": "2022-08-23T14:15:27.036312Z"
}
},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAABBoAAAEICAIAAAB+mAA/AABqo0lEQVR42uy9Z1QUyff43UMOIhJUBAwgKoJZwIA5Z1ZcRTHHrwFzWFHMmDCguGBaZUVFRURRUUB0VUBUJIMoSlBAWEmS08A85799Tv/qqR6GnsgM3M+r6Zru6uo7PVV1q25Q4HA4BAAAAAAAAAAAAP/IgQgAAAAAAAAAAAB1AgAAAAAAAAAAUCcAAAAAAAAAAAB1AgAAAAAAAAAAUCcAAAAAAAAAAABAnQAAAAAAAAAAANQJAAAAAAAAAABAnQAAAAAAAAAAANQJAAAAAAAAAABAnQAAAAAAAAAAAAB1AgAAAAAAAAAAUCcAAAAAAAAAAAB1AgAAAAAAAAAAUCcAAAAAAAAAAAB1AgAAAAAAAAAAANQJAAAAAAAAAABAnQAAAAAAAAAAANQJAAAAAAAAAABAnQAAAAAAAAAAAAB1AgAAAAAAAAAAUCcAAAAAAAAAAAB1AgAAAAAAAAAAUCcAAAAAAAAAAAB1AgAAAAAAAAAAANQJAAAAAAAAAABAnQAAAAAAAAAAANQJAAAAAAAAAABAnQAAAAAAAAAAANQJAAAAAAAAAAAAUCcAAAAAAAAAAAB1AgAAAAAAAAAAUCcAAAAAAAAAAAB1AgAAAAAAAAAAANQJAAAAAAAAAABAnQAAAAAAAAAAANQJAAAAAAAAAABAnQAAAAAAAAAAANQJAAAAAAAAQNK8f//+1atXIAcp5+7du7m5uSAHUCcAAAAAAACkCH9//9GjR+fl5YEopJykpKRhw4ZlZGSAOgEAAAAAAABIBefOnbO1ta2oqABRyASpqanDhg1LTk4GdQIAAAAAAABoYnbs2LFhw4b6+noFBQUjIyMQiJTTrVs3giCys7NHjBgRHR0N6gQAAAAAAADQZGzfvv3EiRPkZw8Pj4EDB4JMpJz58+evX7+eIIj8/PzRo0eHhoaCOgEAAAAAAAA0ATt37jx58iT1eeXKlSATmcDV1XXatGkEQZSUlEycOPHp06ctUAgsDocDrwIAAAAAAEBT4eTkdPjwYfKznZ3drVu3WCwWiEVWKCsrGzFiRExMDEEQioqKN2/enD17NqgTAAAAAAAAgCQ4derUtm3byM/W1tYhISEqKiogFtnix48fgwYNysrKIghCTk7Oz8/PxsYG1AkAAAAAAABAvDx79mzy5Ml1dXUEQZiYmEREROjq6oJYZJG4uLhhw4aVlZURBKGhofHu3buePXuCOgEAAAAAAACIi7S0NEtLy8LCQoIgVFVVo6OjTU1NQSyyi5+f36xZs8jP3bp1e//+fZs2bVrCg4MrNgAAAAAAgKQpLy+fOXMmqUsQBOHi4gK6hKxja2u7aNEi8vOXL1/s7e3r6+tbwoPD7gQAAAAAAICksbOz8/HxIT9PmDAhMDAQ3K+bAcXFxX369Pn+/Tt5uHPnzqNHj4I6AQAAAAAAAIiS48eP79y5k/yspaWVmJior68PYmkevHjxYty4cdQE+86dO3PmzGnejwzGTgAAAAAAAJIjKCho165d1OH58+dBl2hOjBkzZsOGDdTh0qVL4+Pjm/cjw+4EAAAAAACAhEhNTbW0tCwqKiIP7e3tb968CWIRE9evX8/PzxemhoEDB44YMYLfqyorKwcMGPDp0yfy0MjIKDIyUkdHB9QJAAAAAAAAQHDKy8sHDx6cmJhIHhoaGiYkJLSQ4D+Sp6KiQltbu7q6WphKvL29582bJ8CFkZGRQ4cOZbPZ5OHYsWODg4Pl5JqnWRAYOwEAAAAAAEiCvXv3UroEQRCXL18GXUJ8vH79mtQltLW17ezsrly5Eh8f/+PHj+Li4vLyctTAzMvLq/o/cnJywsPDly1bJi8vTxAEi8UaO3asYHe3tLT8448/qMPnz5+7u7s3V1HD7gQAAAAAAIDYiYuLs7CwoJarJ02a9PTpUxCL+Ni6devp06cHDx786NEjLDlgcnKymZkZ+VlOTu7ff//FTiB95fv27RsbGytwA8rLy01MTHJzc8nD1q1bJycnN0s/GdidAAAAAAAAEC8cDmfNmjWULiEnJ+fi4gJiESvBwcHDhw9/8eIFPdH4s2fPqM8DBw6kn7Bp0yYFBYXx48cL0wB1dfX9+/dThyUlJZs2bWqWogZ1AgAAAAAAQLxcvnw5IiKCOly6dGnv3r1BLOIjJyfn48eP586dU1VV5appUJ8nTpxIP0FZWblz587jxo0TshkrVqzo2bMndXj37t3AwMDmJ20wdgIAAAAAABAjeXl5PXr0oKI5qampffnyBYLDipXr168/e/bMy8uL/lVNTY2Ojk5ZWRl5GBYWZm1tTT/NyMjo48ePXLURvnj48KGNjQ11aGxsnJiYKHy1UgXsTgAAAAAAAIiRbdu2UboEeQi6hLiZPn26m5sb168iIiIoXUJTU3Pw4MFcT3N1dRXJpH/GjBloqNm0tDRnZ+dmJm3YnQAAAAAAABAXr169GjVqFHWop6f35cuXVq1agWSaCicnp8OHD5OfZ82a5evrK+47vn//ftCgQdShoqJiXFwcagQl68DuBAAAAAAAgFiora1ds2YNWnLgwAHQJZoW1HFi0qRJErijlZXVnDlz0Ldi9erVzUmksDsBAAAAAAAgFo4ePbpr1y7q0MzMLD4+nsxpIG0UFRV9//499z/q6+v1/sPMzExZWbk5/SKFhYVt27atr68nD79//96xY0cJ3Dc1NbVnz561tbVUiaen55IlS0CdAAAAAAAAALiTkZFhZmZWWVlJldy8edPe3l56WlhXV/fmzZugoKDAwMDo6Gj6nFBdXX3ChAmzZ8+2s7MTJqNzdHT0iRMnmJ9vaWm5ZcsW6jAiIqIhRwgUc3NzJycn3ufcvXuX2igwMzNLSkqSmLRXrVp1+fJl6lBXVzclJUVLS6sZvOoK8G8HAAAAAAAQOYcPH0Z1iQ4dOsyePVtK2lZTU3Pt2jUXF5evX79SmoPJf+jr66ekpERHR+fl5ZWXl9//j6NHj548eXLChAmC3c7AwGDixIlZWVnh4eEhISFU/g0MRUXFGTNmWFhYWFlZoeWdOnUaM2ZMQkJCSEhIcnIy/UJdXd3JkycPGDCg0ZagGSe4hogVHxs2bEDVifz8fDc3t3379jWHd50DAAAAAAAAiJSsrCwlJSV0xnXgwAFpaBibzT579qyBgQHVsN69e3t5edXW1mJnpqamYnncnJychG9AdHS0hoYGfUY6duzY7Oxs3tdWVlbSU86tWrWqpKSE4d27dOlCXRgUFCRh4Y8ePRptORmvthm87eCKDQAAAAAAIGJcXV1ramqoQyUlJWnwvs3IyBg1atTGjRuzs7MJgtDS0vLz84uPj1+4cKGCAm6xYmxsHBgYeOjQIcrZw9nZedmyZUK2oX///mioK5KBAwc+fPiw0fi5nz9/zs/Ppw719PSCgoIuXrzIVT+h8+XLl4yMDPKzqqoqGr9VMqxfvx49LCgouHTpUjN420GdAAAAAAAAECVFRUUXL15ES+zs7Nq1a9e0rbpx40bfvn3DwsLIw27dur19+3bmzJm8polyck5OTqtWraJKPD09z5w5I2RLqqqqsBInJyc1NTXeV/369WvhwoXU4ZgxY2JjY/myv0ItnUaOHKmioiLhn2DGjBmdO3dGS06dOoWqnaBOAAAAAAAAAIS7uzuVKI1k48aNTduk7du3L1y4sKSkhDy0trZ++/Zt9+7dmVy7d+9edK6/Y8eOmJgYYRqTmJiIHhoYGEyfPp33JRUVFTNmzEhISCCVnL179z579qx9+/Z83RcNESthxwkSeXn5tWvXoiXZ2dnXr18HdQIAAAAAAAD4v4nv2bNn0ZKhQ4cOHDiwCZu0b9++kydPUoedOnV68OCBtrY2w8v19PRQK53a2to9e/YI3JikpKScnBy0ZOrUqbyD5+bm5o4aNSo0NJQgiHbt2gUGBh44cIDfSFNsNvuff/6hDiWTcYLOihUrsGTbLi4uVOBaUCcAAAAAAABaOleuXEHt+8mQPk3YnhMnThw8eJA6VFFRuXfvHt2hmTfjxo1DDwMCAuLj4wVrT1BQEFbC22ApMTFx0KBBkZGRBEGMGDEiJiYGcxBnyLt376jNmc6dO5uamjbJz6GtrT1//ny0JCUlxc/PD9QJAAAAAAAAgGCz2adOnUJLDAwMZs2a1VTtOX/+/I4dO9ASDw8PCwsLfusxMTHBSv7++2/BmhQYGIgeysvLjx07tqGTb926NXTo0O/fv7NYLEdHxxcvXjTqrt0QTRgiFgNzyCbTHYI6AQAAAAAAAPy/6e+3b9/QkjVr1tCDJkkGb2/vdevWoSUTJkxYunSpAFV17NgRi3sbEhIiQD0VFRWkzRKFpaVlmzZt6GdWVlauWrXK3t6+tLRUR0cnICDgyJEjwiQUR9WJprJ0IunTp8/IkSPRkujoaNSvA9QJAAAAAACAlgiHwzl+/DhawmKxhI+sKhjZ2dlr1qxBE13Ly8ujHhR8IS8vj6aqIG2QMJsuJrx69QoL68TV0ik5OXnQoEFk0rehQ4fGxMRMnjxZGGkUFxe/e/eO/KygoMBjP0Qy0KMGy/QGBagTAAAAAAAAIuDRo0dJSUloyZAhQzp06NAkjXFwcKBcBUiWLVvWu3dvgSvEauNwOFQOB+YwcZy4cOHCwIEDySBO27Zte/XqVceOHYWUxj///FNXV0f9KK1bt27aV2XatGnKyspoycuXL9++fQvqBAAAAAAAQMvFw8MDK7G1tW2Sljz4D7SkVatWqEM2v9TU1BQUFGCFeXl5QqoTrVu3HjRoEHWYn59vY2OzZs2ayspKLS0tf3//EydOiMRUDDUlalpLJ+rnoDuU//XXX6BOAAAAAAAAtFDy8vKeP3+OFfJOEicmSkpKHBwcsEIHBwc9PT2B68zNzaUXFhYW8lXJt2/fPn36hJaMGTOG0haCgoJ69+798OFDgiCsrKyio6NnzJghEoHU1dU9evSIOkQVmCaErmreu3dPRlPagToBAAAAAAAgLHfv3mWz2WhJv379jI2NJd8SJyen7OxstITFYqGZrUWlTmhoaPBVSUOWTtXV1Zs3b548eTJ5lw0bNoSFhXXp0kVUAvH19c3KyhK42WJixowZmGf5r1+/nj59CuoEAAAAAABAS+TWrVtYSZNYOpWUlNBtZiZMmGBkZCRMtZmZmfTCtm3bCq9OJCUlWVlZnTlzhsPhaGpq+vr6nj17VlFRUSTSqK2tvXHjxooVK9DCv//+WxrSxuno6GDxnbi+RaBOAAAAAAAANH++f/8eHh4uDerE3bt3KysrsUIhtyYIgnjz5o2Q6gSbzcaMwYyMjJ4+fWphYUFmxDM1NY2KihJVjo64uDgrKysdHZ2FCxeWlZWhX50/f15HR8fKymrNmjVN+9rQ35BHjx5hrZUJWGgEMQAAAAAAAIBfTpw4gWWL6969++fPnyXfkhEjRmCJHfT09DIzM4V0aB44cGB0dDRaoqysXFxcjIUn4kFYWNjw4cPREnl5eSraEkEQ+vr6KSkp6urqLee1yc7O7tixIzYVv3HjBpY2W/qB3QkAAAAAAAChkBJLp7S0NEyXIAhi3rx5QuoSxcXFsbGxWKGVlRVzXYKrpROWafvHjx+ynhyaXwwMDKysrBp9l0CdAAAAAAAAaM58+vQpJiZGGtQJLy8veuG4ceOErDYsLIzubDBs2DC+KsHUCUVFxcePH2Ne0adOncJyijd76O9JcHAwPSYvqBMAAAAAAADNFvpycseOHS0sLCTfkidPnmAlioqKI0aMELLaly9f0gtHjx7NvIb8/PyoqCi0ZMiQISYmJrt370YLq6qqMJuxFqhO1NbW+vr6gjoBAAAAAADQUrh9+zZW8ttvv7FYLMm3JD09HSsZNGhQq1athKmTw+HQZ7cGBgZjxoxhXsmzZ8+w/Q0yROymTZuwWLo+Pj5hYWEt5+UxMTGhpyqXOXsnUCcAAAAAAAAEJCoqKiUlBSsU3r5IAMrKyvLz80XekufPn2dkZGCFS5YswXIm8IbuODFx4kTSn/vEiRPYV5s2bZKGQK4Sg54e+/Xr12iiDFAnAAAAAAAAmi30hWQWi4WFMJIM9Em/AB4OdK5evUp/wGXLljGvgcPhYOqEjo7OgAEDyM+2trajRo3CNLRr1661nFdoyJAhdInduXMH1AkAAAAAAIDmj5+fH1bSp08fLS0tKVEnOnbsKEydRUVF9+/fxwrnzp3LV7bv+Ph4LKn2uHHj5OT+bwrq6uqKHhIEsWvXrtLS0hbyCg0ePJhe+OjRI1AnAAAAAAAAmjmZmZl0dwXhXZ8Fbgy90MDAQJg6vb29q6qq0BIlJaXDhw/zVQnXZNjoYb9+/bDtjtzc3CNHjrSQt8jwP7DCd+/e1dTUgDoBAAAAAADQnKEneSAIYuTIkU3SGG1tbaxEU1NTmKxwtbW1Z86cwQodHByMjIz4qicwMJC3OkEQhLOzc+vWrdESV1dXuqrWXKFvUFRVVb1//x7UCQAAAAAAgObM69ev6YVNtTtBn+Xr6+sLU+H58+e/fv2KlnTs2HHPnj18VVJWVhYeHo6W9OzZk74Y3759eyxobHV19fbt21vIi0R3n2hIWQV1AgAAAAAAoDmrE926dWvbtq2UqBNKSkoC11ZcXHzw4EG0RF5e3tvbu02bNnzV8/LlS8xoh4zpRGfjxo1du3ZFS+7du/fq1auW8CJxdZ/gqqyCOgEAzRkOhwNCAOA9AYCWQ35+/qdPn7BCKmCR5Gnbti2WYkIYY6EjR45guZn37NkjQJwoJpZOJC05aOyAAQPout+bN2/q6upAnQCAlkJOTs6xY8dADkCjbN68uUXFUweAZkxoaCh9gWDgwIFN2CRsg6KkpKSwsFCAetLS0tzc3NCS6dOnOzk5CVAV5oetrKzMw7dk5syZWLLt2NjYK1euNPt3SUVFpV+/flhhSUlJXFwcqBMA0CIoKCgYP358Tk4OiAJolIcPH65cuRL2KACgGcDVFqUJdycIgrCzs6MrBvxWUlFRYWtriwZ0GjVqlI+PD19566i7Y94X1tbWampqPC45ffo0FjTWycmppKSk2b9OXN0nZMXeCdQJABCKkpKSiRMn5ufnYz5kAMAVNzc3T0/PjRs3gigAQNbh6inbtOqEg4ODpqYmWhIWFsZvJUuXLkUXxS0tLR8+fKiioiJAexoNEUunX79+ixcvRkt+/vzp7Ozc7F8nru4TsuKNDeoEAAhOfX39nDlzYmJibty40b59exAI0CjTpk3bvHnzuXPnTp8+DdIAANmltLQ0NjYWKzQyMmqSBHYUmpqaDg4OaMmpU6f4Sl9w5MgRHx8f6nDq1KkhISEaGhqCtYfuODF+/PhGr3J2dsZ2MM6ePZuamtq83yiZDu4E6gQACI6zs3NQUNCmTZvGjRsH0gAYcuzYsd69e//xxx8CrBoCACAlhIeH091km3ZrgmTTpk1oromsrKxr164xubCurm7Hjh3oTvv27dsfPnyIpYNgTkVFRUhICFqirq5O9xCgo6+vv2jRIrSkpqZmw4YNzfuN6ty5s56eHlaYl5eXnJwM6gQANFtCQkIOHDigr6+/f/9+kAbAHEVFRXd3dzabbWdn9/PnTxAIAMgiXJeNe/Xq1eQN09XV9fb2Rm2TDhw48OXLF95XFRUVTZ06lQqspKure+vWLRcXF8yNgS8CAgIqKirQkrZt2zKscMaMGVjJkydPbt++3bxfKq66lkxsUIA6AQCCkJOTY29vX19ff/LkSYF3gYEWy/DhwxcuXPjjx4958+aBWzYAyCJcfWS7d+8uDW2bMWNGcHAwlSAiOzt78ODBDSVwKCsrc3Nz69u3L+nnwGKxli9f/vnz57lz5wrThqqqKnokqMzMzNzcXCaX0w3JCIJYvXp1REREM36puKYblwlvbBaMZAAgAHZ2dj4+PoMHD27eXRsgVo3U2Ni4qqrqzp07c+bMAYEAgAzBZrNbtWpVXV2NlX/48KFpA8WiJCQkTJo06cePH+ShkpLSggULhv1H27ZtU1NTv379GhkZeeXKlV+/fpHnjBw58vDhw9bW1gLftL6+PjU1NSIi4ujRo/SkHARBmJqabtmyZcSIEV26dFFWVsa+/fXrV0xMjL+//5UrV8rKyuiXq6mpLV68ePbs2T169OjQoQOLxWpO79WJEyd27NiBFXbr1i0lJQXUCQBobjx79oyMTeHv70/fkAUAhqxbt87Dw6Nr167JycmKioogEACQFb5+/dqtWzd6eUlJiVTtV5eUlNy5c8fT05P3ypeGhsb06dPXrl0rjCJB4uDg4O7uzuRMOTm58vJy1Chr6dKlf//9N/N7LVq0iKFbiKzg6+s7e/ZsrFBBQaGyslJBQQHUCQBoPlRXV/fu3fvLly+9evWKj49vZksjgCTJyMjo1q0bm80+d+4cFowFAABpJjAwcPLkyVhhhw4dqK0AaePz589nz549f/48QRCqqqqGhoYdOnQwNDTs06fPgAEDhg8fLlgcWEC0REVFWVhYcFVfu3btKs0tV4AfDwD44sSJE6RP244dO0CXAIShS5cudnZ2N2/ePHjw4OLFi8EJp0Xx69cvDofTtEFFAYHBUrORSInjBFfU1dXJUHKdO3cODQ3t2LEj/IhSCFffCYIgvnz5IuXqhCRcsT98+LB9+/aoqCh4UQBZp6Sk5OTJk2Rsb/qOpJgoKiqKi4sLCgq6du2ap6fn06dPY2Ji6Da7gCyyYsUKMhSgi4sLSKPZw+FwHjx4MG7cuLZt22ppaWn/x8KFCz9+/AjCkS24Bkrq0aOHdLY2Pj5+8ODBCQkJenp6ISEhQuoSlZWVa9asSUhIgNdA5Ghra3MNy8tVfZUqxL47UV9fv2rVqpiYmFatWkmPfxIgDFeuXAkJCenWrdu2bdsEDkcto7i7uxcXFxME8fvvv4t1a7iuru7NmzdBQUGBgYHR0dF0o0R1dfUJEybMnj3bzs5OmEB+QhIdHU0FFmSCpaXlli1bqMOIiAg3N7dGrzI3N6dHCBESX1/fe/fuMTmzf//+dN84bGKxd+9eHif06dPH0dGR61cjR47s1KnT9+/fT58+vXbt2g4dOkAP01xJS0uztbWNi4szNDTcv3+/hYVFdHT02rVrb9y4ce/evWfPnglvtg5IDK7TO2NjYylsamBg4Jw5c0pLS3V0dEJCQkxMTISp7devX9OmTfvx4wdESBcTXbp0iY+PF1idaKqhTezqxNWrV2NiYgiCSEpKgrekGbBv376DBw+Sn+/duxcVFdVyDC4rKyvPnDlDfl6wYIGY7lJTU3Pt2jUXFxeq+1BXVzf5D319/ZSUlOjo6Ly8vPLy8vv/cfTo0ZMnT5Ku4ZLHwMBg4sSJWVlZ4eHhISEhbDab62mKioozZsywsLCwsrJCyzt16jRmzJiEhISQkBCumXp0dXUnT54sjsxQpqamPXr0CAgI+PDhA9cTVFVVp02b1qdPn0GDBvGuSldXd8SIETExMX5+fgUFBehXcnJykyZN4hGKnsVi2dvbHzt2rKKiYv/+/RcvXoROplny5s2bGTNmFBQU9O3b9/nz5zo6OgRByMvLU33L6tWrYblX1tUJAwMDaWvnmTNntm3bRqbb09PT27t3r46OjqGhoYGBgYmJycCBA1u1asW8tqioqMWLF+fk5Lx586Z9+/bwGogDIyMjYdSJJhvaOOKkuLiYeuHMzc05gIxTX1+PdT3nz59vOY9/7tw58qkNDQ3r6+tFXj+bzT579iw6IPXu3dvLy6u2thY7MzU1dfz48egP4eTk1OTyiY6O5mr9P3bs2OzsbN7XVlZW6urqYheuWrWqpKRErG2urq6eOXMmvc2WlpZFRUX81lZUVNSzZ0+0Hnd390avoiaR8vLyycnJ0M80P9LS0sjXW11dPT09nSqnlmZIysvLQVYyAZvN5hqK7cWLF9LTyMrKyuXLl/OeL8rJyZmbmy9duvTy5cufP3/mUduPHz82bNggLy+voqISFhYG74D42LRpE/2X6tGjh5QPbeJVJ7Zv344uT9JnRYBsgamnZKTLlvP4VFjA5cuXi7zy9PT0YcOGUYLV0tLy8/PjcX5dXd2hQ4eo1U0yxF6Ti2j69OnYGzJw4EAmkyQsY5Genl5QUJBk2pyTk0OfGdy7d0+w2tDxe+PGjQyvMjQ0JC9ZuXIl9DPNjLq6uv79+5O/7+HDh9Gv0HiaPXv2BFnJCqmpqVxn57xn5JLky5cvXPMr80ZPT2/WrFk7d+68ePHi06dP/fz8Ll68uG/fvkGDBpFxR9q0afPkyRN4AcTK2bNn6T+NkpISm82W5qGNEOvbrKSkhD7Gx48f4UWRdbS1tfldfG0evHv3jnrqmzdvirby69evo14o3bp1YzgsrVmzBv05XF1dm1ZK2J4JQRD3799nsvLRu3dv6pIxY8bk5uZKstkjRozAmh0eHi5YVZSHmLGxcWVlJcOrFi1aRKmR1dXV0M80J7y8vMgfV0NDg4zmRFFVVbV8+XJVVdWePXvGxsaCrGQFMns0nbKyMil55cQRJq5///6pqanw64ubhw8fcpU/urEphUObGNUJGxsb7DHu3r0LL4qs8+eff6ILz1VVVS3kwdevX089uGgnu9u2bUP/JtbW1gUFBcyXH9TU1NA9QNJvu6nAPIkNDAwaXVApLy8fPnw4tfO+d+/euro6CTd7z549AmhBdNAUsI8fP2Z+IZqJifeuFCBb1NbWUuEd16xZw/WcmpoaEJRswTVNm6amZpM3rKioyM7OjiCIVq1azZ8/39nZ2dvbOyIiIjw83NfX99ChQ6NHjxbA3VFbW9vFxYX5+gggDA35UD179kyahzZxqRPPnj2jy+LAgQMiv9G///5bWlratDXIKAI/+MuXL/ft23flypWWo0vU1ta2a9dOHF5AWNiETp065eXl8VXDH3/8gdYwderUppJSYmIi3fmhUXXI0tKSPLldu3bBwcFN0vIHDx5gLXdxcRGgnlWrVpGX29jY8HVhVlYWdetZs2bBgNpsQH3rX7582eTjHSASNm/eTJ/hmJmZNXnDwsLCevXq9eeffxYXFzd0TmVl5fPnz/ft2zdq1ChVVVXeioS5ufmhQ4d41AaInNLSUq6/hYeHhzQPbWJRJ9hsNtdIJnPmzBHtjT5//qyjo3P27NkmrEFGabEPLhhPnz5lPkVmDpZqQEVFJTIyUnjVPS4urkmkdOrUKXrEOt5rMJ06dSLPHDFiRKPu2uIjIyMDa/mKFSv4rSQtLY00VFVTU8vIyOD3ckoUysrKmEkMIKNUVVVRXjHt27cXctsNOm3pYdq0afQZzvjx42XuQaqrq8PCwtzc3BwdHZcuXTp58uSxY8fOnz9/27Zt7u7uaWlp8Fs3CfTAJARBbNmyRZqHNrGoEx4eHqSmPnbsWEzNFeFdiouLTU1NCYKIiYlpqhpklBb74AKzbt066jU+d+6cCP8mWFRlAepJT0/H6tm8eXOTSAlznJCXl+cRQcLb25u07mWxWI6Ojvw6mYkczNR45MiR/NawePFi8tojR44I0ICpU6cKuR8NSBu3bt2iflN7e3votJsN5G+BMXfuXJAMIBK4plefMWOGNA9tolcnCgsLSb3q6dOnW7duxTzTRRXcqaqqasqUKaSBhGAhO4WvQUZpsQ8uDOhum/AWCxwO5+bNm2SgDIoJEyYIVhWbzcZiHvTu3VvyIiovL8dMcgcPHsz1zIqKipUrV5Ln6OjoSEmckMGDB6ON79ChA1+Xf/r0iYyy1aNHD8F8qVGjtU2bNsGfrhlga2tL/aZ//fUXdNrNBmVlZfpsb/Xq1SAZQCRQseBQ+vXrJ81Dm+iT6R48eDA/P3/Sf2AmTzU1NVzz0vNLWVnZ1KlTnzx5QhDEuHHjsGmZZGqQUVrsgwtDfn4+moSRR0oyhmRnZ5NOmehC/smTJwWrTV5eHsudlJiYmJ+fL2EpvXr1qqqqClOQ6KclJycPGjTo8uXLBEEMHTo0JiZm8uTJ0vArm5ubo4c5OTllZWXML3d0dCQTRbm7u2PaHUPQ2FavXr2C/10z6GxRI8mRI0dCp908qKmpqa6uppe3adMGhAOIBHV1dXphQz4VUjK0iVid+PTpk7u7u7y8PGlFjT0GGStWyFt8//599OjRz58/5zFlEXcNMkqLfXAhCQ0Npab+HTp0INPZCoODg0NJSQlasmzZMnQ2yS9YbVwtJsUNPXIi/QW7cOHCwIEDybAV27Zte/XqVceOHaXkVzYzM8NKmK99hISE3L9/n7R2wCw8mYOqqXFxcb9+/YK/nkzz+PHjyspK8rO2traJiQl02s1GUeRarqmpCcIBRAIasLHRF09KhjYRqxNbt26tra393//+Rz6AmZkZtpSCrvIKgJeXV58+fdDM4fQ49+KuoSFIQy90yVkwqBGIycSxqR68vr6e6/KMYLDZbNFOnkQrw5cvX1Kfe/ToIWTbHvwHWtKqVSssOS5f1NTU0NML5uXlNa060bp1azSBf35+vo2NzZo1ayorK7W0tPz9/U+cOKGgoCA93bfAfS6bzSaTmGpoaNCd0ZmDGsvW19eHhobCmMqDoqKimpoa+h8/OztbSnqku3fvUp+pkO1NO1rV1dXRhYb2JAx79fr6eoFHIt5UVFQ01HtXVlbyaDzDypk0u/Y/BFAnYHcCEBVcdyckrE7wO7SJcjh/+vTpkydPNDU1Dxw4QEnEyMgoLS2NOkew3YnKyso7d+64ubnFxMSg5bq6umFhYdRhr1696IITVQ1c+fXr1+3btx89epSamvrt27eqqiplZWVTU9Nt27bZ29vLyckx+cE+f/4cFxcXGxsbExMTHR29c+dONJs42btdu3YtODg4NDQ0NzdXV1d3xYoV+/btYxI9WlQPXlhYmJSUlJiYGB8fHxcXFx8f7+7uTjnoCEB0dLSnp2d0dPT3799zcnLq6uratWtnbm7ev3//bdu2YekLmlaG6HDOV8O4ajIODg5YoYODg56ensB15ubmcv29JNn3ffv2DY1LTaaio7SFoKCgJUuWkO20srK6c+dOly5dpK377tmzJ1aSkpLC5EIPDw9yleTAgQP6+voCN0BVVVVTU7O4uJg8jI2NpacYb+EUFhYePXr09evXKSkp5HTf2Nh45MiRo0ePtrKyqqio+OOPP0xNTd3c3CTfI3348AEd6bBwcIqKij4+Puj52tra48aNE/doVVVVlZCQEBMTQ/aN8fHxQUFBaPZ9slO6fPnys2fPQkNDq6ur+/fv7+7ubmVlxVWL8PX1PX36dGxsbHV1dZcuXU6fPj1z5kwhf9by8nJPT8+AgIDk5OTv378TBGFoaLhq1aotW7aoqakdOHDA398/MzMzPz///Pnzq1ev5rd7DAgICA4Ofvv2LVn57Nmzvb29ua5lhIaGOjk5RUZGqqmpLVmypCEDVFAnAHHDdXeioqKCw+Hwa+4ouaFNVI4jtbW1ZKyDkydPouUzZsxAb9erVy9+a37y5ImWlhaTh798+bKYaqATHx8/Z84c0h9rzJgxFy5cePTo0e7duykVwtHRsaFri4qKzp8/v3LlSgsLC/p0FkuHHBUVxXXMcHBwELfoUlNTbWxsLC0tuZr3ZGVlCeY37O7uTib/V1dX37Jli5+fX2xs7O3bt0lfQ3Jh+9y5c7yDKkpMhhwOp23bttQlAkRqQ0Fz4ZGwWCwhg/Gh6bop/P39Jek3hgbXp3oi0oV006ZNVPe3YcMGqc3YVV9fjy0ILVq0qNGr8vLyyDlEnz59hI8zgYaLWbhwIfgjoly5coXqzWbOnPn333/fv3//wIEDmC+Tm5tbk/RI9OyzvJk9e7aYRquMjIzTp08vXLiwV69e2KRZU1MTe0sDAgLoEwUdHR16Js20tDT6HguLxYqKihImcIujoyP5vG3btt21a5ePj8/Lly/PnDnTvn37iRMn1tTUtG/fnrod8/jLFRUV58+fHzVqFNdFvRs3btAv2bdvH+lySvH+/XvmXS65ogr/U0AkUHke6Kqs1A5tItudcHd3//TpU9euXbEJk7m5OZowPCUlhc1m82XkUFBQsGTJEmoKi+4gz58/n0ouRoauF1MNmIJ44MCB06dPs9lsDQ0NHx8fSmWaNm3a8+fP3759SxDEsWPH1q5dSwUdR/n586e/v39hYeHHjx8x71VTU1PU5sHX13fx4sUVFRUKCgrkYEZ9dfHiRVdXV96SFPLBtbS07OzsysrKUlJSLl++TK2bkj8r5v7LhIyMjPnz579584Y0xSPHDPKrvn37zpkzZ/PmzWfPni0pKVm/fn18fPylS5caqkpiMiwuLkYNh4TZnSgpKfnrr7+wwgkTJhgZGQnz18vMzKQXoiqQBODqOJGUlGRvbx8fH0/OY65cuTJr1iypXQ1isVg9e/ZEd6KYLOHs3r37169fLBbL3d1deNutDh06UJs8Iola0Wx4/fr1ypUr6+vryaUyKvnjb7/9tn379hUrVnh7e5Ml3bp1a5Ieafjw4ehs+8mTJ58/f6YO169fj70eDW1NCD9affnyJSwsrKCgICcnh81mN7RnSBCEq6vrtm3bSKlibXj8+PGiRYvQibK9vT3dAIzD4Vy4cIGHWHgQGhq6YMGC79+/k7nwt2/f3qpVK/KrkSNH2trampmZubi4/Pvvv9Qv27lzZ4aVl5aWhoaGzp8//8yZM23atHn48OGGDRtQTWz+/Pno+Xv37j106BBWSU5ODuxOANKzO0G+e1ztoKRiaBOJIpWfn08uMPj5+dEDYmJ3TE5OFvhGZ86coepRUVERIOW7kDVERUVRdhrGxsaJiYk8lp8fPHjAu7bq6mpsp3jHjh3Ut97e3nJycsbGxrdv3y4qKvr48SMWP5ivVFlCPrizszN6awEyG3h7e1Oeajt37uR6TllZGZq9JTAwsNFqxS3DyMjIRpe1GELXJQiCuHfvnpD/vi1btnCdUkhsHaW2thbzQTQyMjp37hy1a2Rqavr161fpXxBauHAhtkbL+/zo6Ghy7XPJkiUiaYC9vT3zuwtAampqd4mwcuVKETa7qqqKyvGnqqpKXyqrqamhumXmb76YeiQSVLXo2rVrk4x3dXV1WIzICxcuUN8ePnyY9Njx9PTMz88nDymcnZ2pMwMDA5WUlNTU1I4fP56ZmYkacREEMXr0aAGWS6mtABUVlYaSXa5ZswbdeSaj4QnMxIkTUduz8vJy6itPT09yv+jhw4dbtmwhlRZra+uGtlL9/f25zqYgJQggKnbv3s31HRNsJJXM0CYadWLt2rUNZceIi4vDxME7US5vUGPisWPHSriG4OBgavnE0NDw58+f9HPmzp1L1f/o0aNG68RymYWHh1MdloKCwuTJk0n3bnrjFRUV+bIbEVJ0aFB8fnd1Kysr0YWuhkZuEtL1hxIyaSzYhDKkVj1Jnj9/LvDbO3z4cOy/oKenJ7yFzIABA7BqlZWVq6qqJNbx0Z2GMZsBfX19AbZoJc/Ro0fpy7Q8zidt0LW0tLh2BcJrhuh7KxKSk5Mls7Q2ZcoUETb7/fv3VM19+vTheg45I1RQUGDyhxJ3j1RVVUUmkSVZsGBBU413aOMJgkhPTyfL//zzT4Igpk+fXlxcTJZgK/FXrlyh/t1qampmZmZJSUlUta1bt6bOnDp1Kr+6BGXIIS8v/+zZs4bOPHv2LNok+mIlX3h5eaG1UfcNDw9XVFRctWoV2sJv377xqIq+SEpCX14EAME4cuQI13csNjZWaoc2EUR2SkxMvHjxopycnKurK/3bHj16YBslAseKZbPZaDh2AQIyClPDrVu3pk6dSu5yKigo3Llzh25PUltb+88//6BWN41Wi5o0tGvXjlxMSkhIsLe3nz59+uPHj1E7WnTnulu3buiIJW7RodaiysrKfMVQX7ZsGdWVjx07FlsGw0DDKWZlZdHVUQnLEHWvFCYUYFpaGn3aPW/ePCEtZIqLi2NjY7FCKysrrlmWJGbphMXE/PHjB707k0L4ioBx69Yt0i/28OHDojItw94uer7zlgn6hjcUsszOzk5BQaFLly5M/lDi7pFiY2PR0EBDhw5tkvGO9CKjPnfv3p3cwwkJCdm4ceO6dev8/f0pxQCzmSR3VzIzM3/77TcTE5PQ0FDq35GTk4PGR6KHg+dtDbFu3TrKOOrEiRMNGX1hNkXy8vKjR48W5i3CBEhauJWVlS1cuHDEiBHnz59HjUOo3TC+jJ2Yj8gAILCxk9QObSJQJzZv3lxXV7d48WKuafyUlZWxuYXAsWIjIyPRXoxHNyTyGl6/fr148WJqhDh8+DDXEeLGjRuUoefQoUOZBBoPCQmhPk+bNk1OTq6goMDGxsbc3PzmzZuYGxk6NvDViQspuqqqKtIhhGTYsGGqqqoMr3V2dr516xY117916xbvgFdYP05a3jehDDFzYSaRoLiCrY0J/A5jhIWF0U2fscgtElYnFBUVHz9+jBmVnTp16tu3b1LegzOPgFFeXr5jxw6CICwsLP73v/+JqgHY2yWmWJwyBxr7NScnB7M/JFFVVe3VqxcTxwkJ9EjodgpBEEOGDGmS8Y4giKioKOozae2Tnp5uZ2c3a9asc+fOoSFiUNNqDQ2NXr16sdnsuXPntmvXLiQkRFtbG5uIC9YwR0dHauJOuqbwOBndMLGwsBDSM0FfXx/1qSN/o02bNpWUlHh5eTEJwwjqBCAxGnKQEEydkMzQJqzv4MOHD0NCQtTV1Xks8PTq1QsNIinw7gQ6a9TS0hIgkrdgNWRlZc2ePZvSJaytrbEgpNQ8FXX2opwFefDjxw8ypReJjY0NuXJWX1//6NEjbMpeUFCArpSjQf3FLbqwsDDU3Rk1QuXNvXv3UDkwUXaxoOONDt7iliH27xVYnSCT2mJjD7/RYOigOTEohFzG44v8/Hx0ykJOnkxMTHbv3r1z505UI92xY8edO3ekuQc3NjZWVlZGQ+83tIRz7NixrKwsOTk5Dw8PviYivMH2lAQbOXjQvXt3NKCC+BBtRhFjY2P08MiRI2RmJYxBgwY1mo9cAj0SqQlQn1u1aiVYhkrhx7vv37///PkT7bfZbPa8efP69u17/fp1LNxkREQE9XnMmDHy8vI7duxISUmJiorCRIT2OSoqKnQbTh59oIuLC6WxYLZMvNUJ4VdeyDU+ahYVGRnp7+9/5coVX19ffuM7l5eXgzoBiJWGdicaevekYWgTqtOvqanZunUraXvKI+JNr169fH190Wk3v8GdSKjMoOSESYBRXIAa6urqZs2ahXbK+/fvp8f9raio+P3336nhf/fu3Uzm3M+ePUNX18aPH3/+/PmAgIDQ0FA0ggfXFS/mc3rhRRccHIweMszMGhsbS8YjIw/79eu3bNmyRq9CRU3OVptWhqJSJ+iGK4MGDaJccQSDw+Gg/ywSAwODMWPGSKzXe/bsGbY9Qr4emzZtunTpEqq8+fj4rF+/XsI7J3whLy/fo0cPdL7IdQknPT2dDEi/atUqS0tLETYAe7tErk7IycmhVu+ywoQJExQUFCgzxQcPHvj7+5MLByjHjx+Xhh4J62esrKwwVyKJjXeohqCkpDRq1KiDBw9mZ2fHxcXR9S40o8WkSZOeP3/u6ur69OlTutkPatA7fPhwhl1iTk4O6dZJHu7Zs6fRZDvR0dHUZ5Fkmx06dOjff/9N/ayLFi2aMmWKAOHmGvpjNqrNioqqqqp169bBnFvy2NjYYMkPxIRodyckNLQJ4yxy4sQJgiA6duzI2zuNPuP59OkTv/cqLy9H/6tkVHsJ1IAF1B8yZAjX6B/oG7Z69er6+np+A7nMmDEjOTlZVVX1wIEDXE92cnKiTtbX15ek6Mig7CTt27dn+HTYetI///zD5Ko9e/agV23fvr1pZfjbb7+h7cnLyxPgn1JaWkr/6+3fv19Iby1UlaLYvXu3JD3G6KkMIyMjya/u3buHfTVw4EDewfubHDs7O7TBAwYMoJ9DRhJr27atyF2lqbkOCfM0OM0ebAjX0dHhK66dJHskMroidb6Tk5MAzyt8p01GRkI3HCIiIpSUlF6+fEk/E3Oc+PjxY5cuXQ4fPkw/E0ua6eLiwrAxkydPpq7q2rVrdXU17/NTU1PRdaJGz2dCYmIipr2npqYKHHuGjsg7hIaQzB4jQGffvn2S+YmxNVwKd3d3qR3aBN+d+PnzJxmn2cjIiLefJT14c1JSUo8ePfi6XWhoKJpgX4CtTwFqKCsr27dvH1pCN2FKSUmZM2cO6Z+npKR05swZtBPnrcih08Fp06YtW7Zs4MCBDQUIQ09muD8gEtH9/PkT9T4cP348k6SMkZGR6Gb9wIEDR40aJcAqPu/0yRKQoUh2JzIyMuiFwq/TX716FSthsVhMFlxFBYfDwRwndHR0qEhTtra2o0aNQk0joqKirl27tnTpUqkdLTCXNfqOcEhICGlp4+LiwjDdGHPEbewkuxw8ePDx48fUPlhBQYGtrW14eDjz/6NkeiTSCQFNbiOYH7bw4x1BEKgn9+jRo1esWLFjxw6uUTTQDQcrKysPDw99fX1HR0f6mZh1JcNNg3/++QcNL7t58+ZGF/LRH8va2lokC/9mZmZo4vk5c+ZgdnRC7k6AsRMgnbsTEhraBFaeVq5cKczYwO/tUHeFTp06CdBgAWrYv38/9nug3/78+XPTpk1UD2JjY8NXsH90J1dOTs7R0VFdXb2hJbfCwkJ0x/zWrVsSEx0WFM/Ly4vJVba2tuhVf/31F8PbYaM170DvEpAhZhksQKoTDofz6NEj+r8AS93NL4WFhfS51Lx58yS5ZkwPKmVnZ4eeEBMTgxlp6OnplZSUSO0qOJo1jFoNob6tra0l3fetra0Z7tEJc/fjx4/DvgTF8uXLsZ9m8eLFzC+XTI+ERXhksViCrfMJP979/PkTXfextbW1sLBgs9lcT6YS55HRY1VUVD5+/Mj1TDRZb7t27Rj+C1Bn9DZt2jAJGz1t2jTqkiNHjojqLUINXCdNmiRYJQsWLOA6sZFYzwa7E81+dwILeEBx9OhRqR3aBNydiI2NvXLlCovFWrBgARPD0AcPHqARcgTwxkbXKgRbquG3Bg6Hc/nyZbSEXPetrKwMCQnx9PR8/PhxbW2toqKinZ2dg4MDv4vN6LJu+/bt3dzcDhw40FDWzxcvXtTV1VHzZr4kIKToMIsaJstRnz59Qn0l27RpM2/ePCb3ysrKQhfyW7duzTsirQRkiC0SVFdXC7BBwTVxtQBpxVG8vb2xdOBKSkq8Q16KHK7JsNFD0jwdzd+Xm5t75MgRqY0bSw+ol5KSQhl5e3h4JCUlycvLe3h4MNmj4xfUVY7HAlXL5NSpU8+fP0f7h2vXrllZWTVkedIkPRLmOGFqairYOp/w4x1pykUdBgQEvH79uqHB+sWLF2iHv3PnTnooGPo+xrhx45j8C548eYJ6caxYsaLRF/vbt29o7AoRxpYYOnQo1WtxDRHGhIZ83srLy7GIdmJCTU2Nik4GSJJevXpJ5kYNuVwL7G8piaFNMC2E7FKZL4Vic9DevXvzdbv8/Hz0CW/evMlvgwWoAXVNI9mxY8f48ePJ2aSysvL48eNPnz7977//CiZDrIvs169fQ0tH2JqQhYWFJEWHznobyiGFsXr1avTRFi5cyPBeWBfZ6OqjBGT4+++/o7cQ7Oe+ffs29i5pamoKs3RRU1NDD0O8ZcsWCS8Y04f5zMxMurE15v6rrKyclpYmnUvg1dXVWJQIahk7Ly+PDFW5adMmMd39ypUr6K3//vtv2JTAVuywX0dZWTklJUV6eiSsw1y2bJkAjyl8p83hcDCTwqVLlzLxUiAIQldXF80YzcPFwtPTk0lLSINsivj4+EYvQbOmtmrVSvhcnxRofgnB3Dg5HM62bdu4zqZkIvc/IBM8ePCA6zvG8E/XJEObICEO79279+rVK0VFRdJ3gglYQreUlBRqnZgJL168oBZaWCyWAAl9BKgB2xtSVFQMCAjQ0dHZs2fP06dPCwsLg4ODN2/eTA8fxFD1DA8PR0uOHTvGY59HYMcJIUX38eNHNOg7w1ujS2t8rS1hMue9gigZGWKLAdiGAEOMjIywEn5DE2KcP3/+69evaEnHjh0xn1FxU1ZWhsm/Z8+ehoaG2Gnt27fHXFmqq6u5hlqWBpSUlDA9jYqA4eTk9OvXL319/YMHD4rp7tjbJWTgL67USwR0XVyEDBkyBDNAra6uXr9+vZT0SGTcarTDFMxxQvjxjjTKoj6ThqAMhbNx48aGglSiWxMMd6rLy8vRlnTq1KnRsLmVlZXofqa1tbWo4g7n5OSg0ThIHxUB6mlod0WwIJ4AQKeiokK0uxOSGNr41T8qKyvJudGaNWuYX0Wm9Bd4VQDNpsHvzobANfTt2xdtsKurqwhVz4CAALTy/v378zgZc5p59eqVxER35swZbEbe6CV0z3uGS9G5ubmoK5upqSmPrQaJydDBwQG9UDCHByzWJEEQffv2Ffjl+fXrl46ODlqbvLx8aGiohJdP6A4hDa1tVFVVde3aFTuZa4QZaQAzsp85cybppUM6gfDltsQvp06dQm8dFBQk2vqTk5MlMxZOmTJFTCKqq6ujG//wfvkl1iPR1xSTkpIkM1phYE5NNjY2DF94DQ2NoqIiJjsemCchQ4ttJtMGLPyJqBwn6uvrJ0yYICcnh3aeZOBgfiFjWtJ58+YNLKsDIgGztKd4+vSp1A5tfO9OnDp1Kj09XU1NjUmaNgp6HCe+3Cck7zhB5gDidyWGOVgUMDINIZNldQ0NDb5yrAopOrSdqqqqTPxDsHjG+vr69LV5rly7do3KFUjqb7zdciQjQ2wxALNuZ0jbtm2xeuhpKJhz5MiRgoICtGTPnj2Sz+eALjry3vNRVlamD8CbNm2i5/OWBjCrcfJ93rhxY319/dixY+fOnSu+W0tgd0LWkZOTu3r1KrZ8fu3aNWnokTDHCS0trYY8EMQ93mH/TTS/KgabzUYTXCxYsIBH8ml0d4LhgIj10o2mxElPT8fyhwif65PEzc0tODh4165d6F9YsN2Jhv6YDa0oAwC/iNx3QhJDG1/KR2ZmJrnNt2vXLr4uxKbmBEEcOnSI4bVYkM2AgAB+dSYBaqBH4xIspE9DoNZf2travINqo6kPZsyYIdYHR6mtrUXf3YkTJzK5CkszYmVlxdCwD43Zx2R1UzIyxBaM3717J9gvTt/iLygoEKCe1NRUzBd8+vTpjS6aigNs51RZWbkhk2sSuoXJpUuXpHBZCAtlpqKiQpYoKSklJyeL9daYJQZfkeKa2e7E2rVrf/z40dC3mJ1tjx49pKFHwlJbCBY4SPjxjnJuJOncuTOPUC3YfDoiIqKhM9GUlMwbhmkdZBRd3h01mhJXRUVFJBknEhISyATebDYby82flZXFb23Xr1/n+tr7+/vDsjogEtAAcSgxMTFSO7TxZ5K4Y8eO8vJybW1t3ivBdAwNDdXU1FDdPSkpieG16NqJoqKiAGsVAtRAt04RYcLLzMzMT58+UYezZ8/mUTmbzUbXhAROhi2A6CIjI1G1iqG/AbZCw9C3xNXVlRquVFVVXV1dpUSG2KSZSU5crtjZ2SUkJGAWF9ra2nxVUlFRYWtri65hjxo1ysfHR7C0u8KQlpaGOW9YW1s3ZHJNcvr06YEDB6I7Ek5OTnZ2dtKWpxmLgFFVVUUavG3duhXzARM5eXl56B+20RQH/KKgoCCk0w5DMGM8fqmtrb1w4cKIESOw1EsU69atc3Z2prYKs7KymrxHItfmPnz4QB0K5jgh/HhXVFSEOjXNmzePR6gWNDhb586dBw8ezGRrQklJqdEIVyTYSiI9xza2HfT8+fOtW7dSHjKWlpbCj7zV1dXz589XV1f39vaWl5fHRBoaGsrvuiyPyE6wrA5I7e6E2Ic25poHFUtOgKwRWGZlMt4WwwtR77fhw4ejXy1btiw2NlYcNdTU1GApaZjEDyHt2hMSEnifg/qZkVvbPE5GQ+xhC5ZJSUm8I4cIKbpjx46ht0afKy0traHsjFiDmcQ2ycrKov4kLBbL19e30UskJkPMKo95uHr6i6GpqYnNV/itZM6cOWgNlpaW/EY6F1VSag8PD6wnOXbsWKNX0RPYNZpjWKxPwZWKigosVwY50+K99yIS0NzP3bt3b7Erc6Sm6uDgwOMcNC8B790JyfRIHA7n8+fPzDsl8Y133t7eaDOio6N5nGxpaUmdyTuuy/z586kzR44ciX7l6Oj44MEDrldhXss89lETEhLU1NRu3ryJprT6448/0H+9j4+PACLdvHkzQRCPHj2iSrp3707dYu3atdj5jx8/5j2IY8HTKSCNPSAqNm3axPUdQ5NFSNvQxtR3oqamZt26deT+CMOszxiYL+anT58YmqFjga7R6UtWVlafPn3EUYOioiLa4xAEgWaGboiEhAQLC4uzZ8/yPg01J9XR0eGdnBVdrOrSpQu1WF5UVGRjY8NbVRVSdOgwbGBgQEVcrqiomDlzJma+35AGjOZ2bYgNGzZQ2yCHDx+eNWtWo5dITIbGxsboP5Du1skQTU1NzKv71KlTTIRDceTIER8fH+pw6tSpISEhzMOch4SEWFlZqamp6evr//7771h0SH6hO04wsaV2dnbGdjDOnj3LV0tE+xRcUVVVpW8LnDlzhvfei0hA365u3bq12JU50uAHixuGgfbPVCL2JuyRMMcJeXl5KysrAZ5d+PHu8ePH6LDbv3//hs7Mz8+PioqiDlFtlg6aDxsNNuXl5eXm5tZQemlsAxYNe4X94lOmTJk7d669vT2azBt1cnNwcDh16hS/3msBAQFnzpzZvHkzqn+iGxSvX7/GZibz5s0rKSmB3Qmgme1OiH1o49eod+7cuYIpLljkb4IgwsLC+F3voS559OhR27Zts7OzxVcDtgHar18/3tGv//77bzU1tenTp9fU1PBeWEV72EYDmaOWOVQYitra2nHjxllaWvKwKxVedKidz8qVK6n2z549e+jQoTzWmTp27EhdOHjwYN53QTOaMQzTLjEZkqD/QPpSFnPy8vKwtTqGzgNsNhuLrLp9+3a+VuijoqKwYIutWrUSOBhUeXk51gGpq6szbA+9H2BuZy/ap2C48k1qbpJZkUL/OOLLbiH9kHuPLBaLx2L8ihUrKFm5uLgwF6w4eiQS1OO5X79+Ajy48J02m81GE+dt27aNx8moD0CrVq14jFyY3k5Fxnv9+rWysvLt27cbuhBbZeC6ifHt2zdjY+O+ffuWl5djZsaUY8POnTuNjIz4TfuTnJzcunVrCwsLrJP38vJC70LF6yspKTEzM0O3RBraSGloxQeW1QGRgG4GUrBYLIFTVktgaGOkTrx+/ZoaxQWOl0qXzubNmxu96tKlS+glFRUVHA4nPDxcTU2N4b6nwDW8f/8e2xhav3491w43NTWVnLCOHTu2qqqKd3vQFSyCIPz8/Hifj1ohOzs7k5Pp+fPnt2vX7vv37+ITXV1dHWq06uHhQRYuX75cQ0ODd6RFNC6whobGr1+/GjrTz8+PEvLu3bsZ/lUkJkMS1GmEjK0mMP7+/qgjtYGBQaNGdIWFhag6pKurK0BAN647iqampoK5OaKbJNSeD8Nr0WS3FAyfSLRPwQNUeVNRUUlNTZXA+FFfX48aWF64cKHFDqVUohIeEU4pK39dXV0eTtuS6ZGwJjEc3cQx3qFL++QOD4+TUcMq3gEqsEScpaWlHA7n7du3bdq04W2viNlsDBgwAPu33rp1S0tLq3///vn5+RwOB932VFdXJ89xdHRs06bNx48feRtyODg4GBkZrV69+uXLl3V1denp6SYmJm3atKH/fzGPDtKsrry8fMKECb179260P8G80il2794N82BAJKDBY+j/COkc2hpXJyIiItq2bUu1wM3NTYDbFBYWorlCqeUQsgfhwfLlyzHTiOPHj6uqqu7du5fhrYWpYePGjfR9cy8vr7i4uNLS0oyMjCdPnsybN09ZWZnFYm3evLlRXQKLSaKsrFxWVsb7fGVlZep8CwsLHx+f0aNHa2ho8AjBIRLRFRYWopcvXbo0NDR03LhxZDq/RidGaNzShhZ7Ll26RPoQKysrX79+nfnrJDEZkqBx0C0tLYX8y71+/RoNxaitrd1QBobS0tKzZ89SC6ssFmv58uWCxYOaPn16Q7ZD/FZVWVmJ2QGSph0MbTq5BqzQ1NRkErJdhE/BG09PT6ryAwcOSGb8+PHjR0OuSi15Ze7Jkyf0E9BQqg1Z7UuyRyLd7dCVgsePHwvw4MKPd2jC5vbt2/PYM2Sz2ege77lz53hUi/nReXh47N+/X1VVdeXKlby3JdPS0rBIdObm5h4eHo8ePTpw4AD5o1hYWBQWFpLnY+H2N27cOGTIEHV19UbT1GAeI3p6eqqqqnJycg3F6cfMxqytrXV1dTU1NZmkCqFHaoEdRUC0cDUebt++vTQPbbzUiYyMjGPHjqmqqmIDv4eHR0ZGBpPYlFVVVSkpKZ6enpTZPd0w/cGDB7m5uQ3VMGXKFPpVfGXQE6aG0tJSJhlJ9fX1meR3Ixk+fDh1IZNIgnRjWXV1dSYGHsKLjp77U05Oztvbm8m1X758MTc3J69SUFA4ePAg+sKkpKRQtmSzZs36+vUrX2+txGRIQgUhIHVgIXcbORxOfHw8GmBHSUlp2bJlV69eTUlJKSoq+vDhw+3bt7du3YpqHSNHjmRiHNgQWARS1G6SYQ11dXUpKSnXrl1rKASEqanppUuXPn36xFWpLioqevHixcaNGxsy/VRTU1uzZs2LFy+ys7MbkrDwT8GQd+/ekTWbmJgwWSMQCah/p66urvCvmexibW1NiUJBQQGzCXzy5An591FQUNi/fz/DOsXaI5GWeGib+Y2RIKpOG/17Ll++nMeZYWFh6F14r/37+/vTG7Zz504mTeKdrd/GxgbdLLp//z52QuvWrZl0fVzTfp8+fbqh8w8fPkx3mGS4MNFQfgnKJBgARNgHoq5Q0jy0NahOoOl7uKKoqMi76szMTB7x6TA8PT25VoJlZdbQ0Dhz5gxfVuPC13D58uWGMvtYW1t7enoy94svKSlB7Rn+/PPPRi+5e/cuanNlZWXVaOhuUT34li1b0BqMjIz4yshYW1vr6upKhQHV09ObNGnSvHnz+vbtKy8vr6mpOW/ePN578U0uQ2o9Ht3fYJhSlzfFxcWXLl1qNJuehoaGvb29MIoEJTSuYSuZG/uSkRiYICcnhyVpWbJkCV8OXQ2lqhX+KZj/OmTNwqQg5Rc0FKmtrW1LHkoNDAz09fXXrl27cuVKMgWqrq7utGnTZs2aRWViGjt2bGJiIl/ViqlHIjl//jz18w0bNkywSoTstLHc/w8fPuRx8q5du6gzO3TowLvm8vJydBDs3bt3YGAgw1bV19efPn0a26Mg3UtevHiBnVxWVoZ2tgMGDOAdmYoiNjYWvVBeXp63uWB5eTnqPt62bVs09FOjcI3NbW9vD/NgQCRgoVBJ+vfvL81DG+v/qRRSDJlnJygoqLy83NzcfOrUqfzGTRe+BoIgSkpK4uLiEhISEhMTFRQUOnfu3KlTp759+9KtPsRBVFRUUFCQhoaGmZnZmDFjGCppIhHdtWvX3r59q6Wl1bt375kzZ2JbVQxFl5iYmJKS8vnz53///bddu3YdOnQwMzMbNWoUFopXCmWIRgKh8j09fPiwIasbAfj8+fPZs2fJuYiqqqqhoWGHDh0MDQ379OkzYMCA4cOH04dhgYmNjU1PT9fR0WnVqtXkyZN//vwp2meRDJJ5io4dOw4aNAjLgCZWli9ffvXqVcrKhUcm4+ZNfX396NGjL168SC20p6amBgYGJiUlVVZWdu3atWfPnmZmZoIlnBZfj4T+fG5ubuvXr2+S8U58lJWV3blzp7q6um/fvkOGDKFHnORNenr6y5cvY2NjFRQUzM3Ne/XqZWFhwbWStLS0+/fvl5SU9OvXz8bGhvmNfv78+fjx44iICH19/cWLFzcUbIqisLDQ3d09MzPTzMzM3t6eYUISanENSzhI7rQ8ePAAohIBwtO9e3dsdYB03fTz85PaoU3a1QkAkAb27t1LOWwcPnwYXdgTkqysrClTpiQkJHTu3Dk0NBQNQSNWDA0Ns7OzMzIyOnfuLLu/i/ieorS0VEVFRZIa76BBg6gYA/Hx8fQ06oA0Y2ZmRiYdl5OTy8rKQlM7A82PCRMm0LNPDBky5M2bNyAcQHi0tbWLioqwwu3bt7u4uEjt0KYAPxsANIqtrS2lTjDJQMKQ+Pj4KVOmZGdn6+nphYSECKlLVFZWbtmyZe3atY3ORJOSkrKzs6dMmSLTuoRYn4J5Qg+RwGazExMTyc89evQAXUK2yMnJIXUJcicTdIlmj4mJCV2daCjiE0AQRGZm5u+//y5trZo8eTKVgl16KC4upusS5FsnzUMbqBMA0Dj9+vUzNzdPSkoiY66LpM7AwMA5c+aUlpbq6OiEhIQI2VP8+vVr2rRpP378YNI5btmyhcViOTs7y/SP0jyeguT9+/eUfyfXiOOANINGmlq7di0IpCWoE/TCf//9t6KiQgL5LmWRq1evYhHepQHJ2KvzC92OjkTKc5vKwVsOAExYsGABNWZQC8kCc+bMmWnTppWWlpIuoXv37l21atXBgwevXLny6tUrKiMvQ6KiooYNG5acnPz06dP27dvzPnnv3r3BwcFr167lkS5X+mkeT0GBJm4HdULmoBaqO3fubGtrCwJpmeoEj4lgC4fM8wtyYEh6ejpfb52UALsTAMAIe3v7Xbt2kb5Gz58/byj2caNUVVU5ODhcuXKFKkn6j/+fli8n17NnTysrq6FDh44YMYLHCkpOTs6xY8fc3d3JKIdkGBweODs7Hzp0aM6cOW5ubrL7WzSPp+CqTgwZMqRRF1JAqqirq6N2JzZv3sw15g/QQtSJ9PR0MzMzkA9d3wZFS0h1QkVFxdDQENQJAJB5OnXqNHr0aDIHxfPnz+kpDpnw9evX2bNnx8bG8j6tvr6e1DHIpDN6enrW1tbdunUzMjLq1KlTZWVlXl7ejx8/AgMD379/z+Fw2rRp4+3tzTVSNUVJScnSpUv9/Pxmz559/fp1fgOzSAnN4ykwKioqIiIiyM+LFy+G/5oUUl9ff/jw4cDAwKlTpzo4OFDRZgmC8Pf3z8nJIZ1ewNKphWBsbCwnJ1dfX89kIghQy2f/+9//mMcclwAN5QCQQnWia9eu/EaklDQQ3xcAGBIeHk7+a9TU1ARIU+Xl5SUOL6j+/fszSZV/6NAhPT29u3fvyvRP0DyeAsPHx4f8Kdu3b4+l7ACkhODgYOofd/z4ca4Jp4KDg0FQLYdOnTrRe+PNmzeDZDAKCgqolCDJyckgkEaZNm0a/dWysbGR8maD7wQAMGXo0KEzZ84kl5P5Cv/869evuXPnktnZ5s+f7+zs7O3tHRERER4e7uvre+jQodGjRwuQX0JbW9vFxeXNmzdMzGPWr1+fnJwshbE1+KJ5PAXGjRs3yA9btmwRYZoRQIRQKh9BEC9fvqQ+h4aGkqsMW7ZsGT9+PAiq5cDV3gl2J+h4eXlVV1eTaX/RrO0AX7sTUu6HDcZOAMAfR48effToEZvNvn79OnO7FNJy6c8//1y4cCFqJkHh5ORUVVX15s2b169fv3r16t27d5WVlTwqNDc3nzt37oYNG7jWxhVNTc1mIP/m8RQoBQUFT58+Jbfd16xZA38x6QRdih43bhz5ITMz087OjiCIadOmnThxAqTU0tQJ0vYV1AneUOkdV6xYAdIQWJ2Qcj9sUCcAgD969OixfPnyixcv/vPPP9nZ2QYGBkyusra2TkhI4H2OiorKmP8gCKKmpiYyMjI6OjonJyf3P2pqavT09Dp06GBkZDR58mQjIyP4LZoHPj4+tbW1BEE4ODhIONkFwJwVK1Zcvnw5MzOTIIjc3NyCgoLg4OBDhw7l5OSsW7fO1dW1ebjxAHypEwwngi2Zd+/ekWOfhobG7NmzQSCN8vPnTypouGypE5AVGwD4Izc318TEpLy83NHR8ciRIyAQQGA4HI65uXlycnKHDh0+f/4M6oQ0k5qa6ujo6OvrSw2a48eP37Zt24QJE0A4LZD79+9zDQpcUFCgra0N8iFZtWrV5cuXCYJYuXLlpUuXQCBMFLDBgwfTy799+8bVXUd6gAUVAOAPPT29rVu3EgTh7u5eXFwMAgEExt/fn8ymfPr0adAlpJyuXbv6+PiUlZXF/UdZWVlwcDDoEi2WhmzZITc2RUVFxZ07d8jP4rZ0qq2tjY+P9/b2rqqqkmmhyWiUWDB2AgBB2LZt24ULF37+/Onu7r5r1y4QCCAYR48eJW3x586dC9KQCdTU1Pr06QNyAIyNjVksLvYd6enpFhYWIB+CIO7cuVNSUkI6+1lZWYmqWjabnZ6enp2dnZWVlZaW9vXr1/j4+OTk5JqaGnl5+alTp8p0QIuGHCek35wS1AkA4BsNDY29e/c6ODi4urquXbtWOmNXA1LOkydP3r9/r6Sk5O7uDtIAAJlTLHv37h0fH89kOtgyodJNiHZrIjY2dujQoaTLGcaAAQNkPVwH1/dn6NCh0t9yMHYCAEFYs2bN2LFj8/Pz9+zZA9IA+KW6uprMhHjq1CkeWc8BAJBahg8fTi/8+vUrSIYgiM+fP5MxlJWUlBYsWCDCmi0sLGpqasrLy9+/f49tBJGxTGQarsZyXN80UCcAoDkgJyfn7e1tYGBw/vz5RrNcAwDGyZMnv379amdn5+DgANIAAFlkxIgR9ML379+DZAiC+Ouvv8gPv/32m66ursjrV1NTs7S0nDdvHlo4evRomRYah8OJiYlh+KaBOgEAzYR27dr5+PjIycmtWrWK68YrAHDl8+fPR44c6dGjBxnzBAAAWYTrmnFiYmJZWVkLlwyZmon8vHz5cvHdqLy8nPqsqKg4bNgwmZZbSkpKYWEhVti5c2cpj+kE6gQACMvQoUNdXV0jIyMdHR1BGgATqqqq5syZo6io6OvrC9GcAEB26dChAz0bQF1dXWRkZAuXzMOHD//9918y/yOV9lEcREdHU5+trKzU1dVlWm5v375lqLWCOgEAzY1169YdPHjw9OnTAQEBIA2gUbZs2ZKamvrkyZNevXqBNABApuFqhRIREdHCxUI5YS9btkx8IYnq6+tfv35NHTYDxwmu6oRMWDqBOgEAImDPnj3bt2/38/MDUQCNEhAQ8ODBA5mI1AEAAG+4rhxznRS2HLKzs4OCgkgPwyVLlojvRlFRUahpUDNQJ7gqorKyOwGBYgFABBw/fjwsLAzkADTKzZs3Zd3AFwAAEq4rxy1cnfD09KyrqyMz6nTu3Fl8NwoODqY+q6ioDBkyRKblVlZWlpiYiBW2a9fO1NRUJtoPuxMAIBpgjgjAewIALQpjY2MDAwOsMC8vLzU1tWUKhMPheHp6kp/FnQmb3AMhGT58uLKyskyLLjIyklTDUGRlawLUCQAAAAAAAAHhOuFrse4TL168IDMn6Ojo2NjYiO9GJSUlqJAnTJgg66KTaT9sUCcAAAAAAAAEBOydUCgn7IULFyopKYnvRv/88w+bzW726oSs+GGDOgEAAAAAACAgsDtBUVRUdP/+ffKzWNNNYJZOenp6ffr0aX7qhKamZt++fUGdAAAAAAAAaM6Ym5vr6OhghfHx8RUVFS1NFDdu3KiqqiIIYtCgQaINhF1bW1tSUoKWoH7Y48ePl3XRpaWl/fz5Eyu0trYWX5hdUCcAAAAAAACkAhaLNWXKFKyQzWZ/+PChpYni6tWr5AeRbE2w2WwfHx9bW1tTU1M1NTVNTc22bdsePXqUIIjU/6DObK6WTjNnzpShR4BAsQAAAAAAAAIyb96869evY4Vv3ryRIcN34fnw4UNsbCxBEOrq6nPnzhWmqurq6jNnzvz5559ZWVkqKirDhg1bunSpiYmJr6/vrl27dHR0UK8JFovVDHYn6NZxioqKs2bNAnUCAAAAAACg+TN+/HhdXd38/Hy08PHjxzt37mw5QqCcsOfMmaOhoSFwPTExMYsWLSIzMAwZMuT27dudOnUiv5o1a1br1q0vXrzYsWNH6vy+ffu2b99e1qWHuoKQTJo0SUtLS4YeAYydAAAAAAAABERBQeH333/HCiMiInJzc1uIBCorK2/dukV+FsbS6fjx44MGDUpMTGSxWDt27Hj9+jWlS5A4OjrGxsY+e/aMKmkGlk6Jif8fe/ceFNV5+H/8sIAgohECRhoqlJDgNAlpuIQhMDRgIwmFuiJJK+mgEHNpLMNYAjHNYJN0WrAxqVZGtKIVlZsUUShFW5UCglbAqkM74CJIEajKRQii3HZ/8+uZ7+mZXV0FgV3g/frrPI9nDw+HM8757HOrU6lUWpWrV6+eXr8FcQIAAGD8IiMjtWrUavXRo0dnya+fl5fX29srCMLSpUv9/PzGcQWNRhMXF7dx48bh4WFBEH7xi19s3rzZzEx7BI2zs7OFhYV8mvsMiBPSclgSKyurH/zgB8QJAACA2cLf39/R0fGBr4kzlTTSaXxdE2q1+r333vvd734nFkNCQjZt2nS/M4eGhuSv3f7+/tP97h0+fFirJiwsbN68ecQJAACA2cLExER3/nFpaan4nb2x6ezsTEtLa21tnZCrqVSq8vJycfZwVFTUOK6wYcOG3bt3i8cuLi4HDx40MTG555k9PT2jo6NSMSAgwMLCYlo/OVevXhWnsMtNu5FOxAkAAIBHpfsKODQ0VFxcbGztrK2t9fT0/OCDD6TegEckdU2EhYUtWrRorB/Pzc2VWqJQKPLy8vRMQb5586a8OCNHOi1cuPD1118nTgAAAMwuHh4ezzzzzANfFg0rIyPD39//3//+tyAIBw4ckK+4Oj4jIyP79+8Xj8cx0qm+vn7dunVSce3atR4eHnrOv3z5srwYHBw88+JEeHj4nDlziBMAAACzjm4HRUlJibhRtMENDw/HxsauXbtWas/169f//Oc/P+Jli4uLOzo6BEF48sknx/pyr9Fo3nrrrf7+frE4d+7czz//XP9HqqqqpGNHR8dvf/vb0/qBuXHjRmVl5QOfIuIEAADALI0Tt2/flq9qaijXr18PCgpKTU3Vqpf2sR43aaTT2rVrTU1Nx/TZzMzM8+fPS8UNGzY8+eST+j8if/nWP9Lp7t271dXVaWlp69atM9oVe48ePapWq+U1TzzxRGBg4HR8+E00Gg3/BQAAADwiDw+Pf/zjH/Ka6OjoR39rfxRnz56NiIhoa2vT/Sdzc/O2tjZ7e/vxXbmjo2PJkiUjIyMmJiaNjY0uLi4P/9nBwUE3N7eWlhaxaG1t3dbWtmDBAj0f6e/vt7e3l3pXcnNz33zzTd14k5OTo1KpWltbxTf1+fPnd3d36645awxCQkJKSkrkNbGxsRM1p2WK0TsBAAAwAXTnDxQWFsoXI5piu3fv/u53v3vPLCGOgDpw4MC4L75v3z5x9kVgYOCYsoQgCLt27ZKyhCAISqVSf5YQF1SVsoRCofje976ne86qVatSU1OTk5Ol/BAQEGCcWaKvr+/kyZPyGhMTk3fffXeaPvnECQAAgAkQExOj9WV/V1eX1lvj1BgeHn73v5599tmtW7cWFhZ+9tln8+fP1zrtD3/4w/iur9FopF6XcUzCzsjIkBd19wHUJc35FgTBy8vL1tZW95yFCxe6ubmtXr3a3d1drFm2bJlxPipFRUXyPTTEzornnnuOOAEAADB7zZ07Ny4uTqtyx44dBni9Uyj++c9/pqen19bWxsXFhYWFbdq06eTJk1obNdTV1VVXV4/j+mVlZY2NjYIg2NjYhIeHj+mzjY2N8lkTdnZ2r776qv6PtLS0lJaWSkX90741Go20BpTRxomdO3dq1WzcuHH6PvnECQAAgImxfv16rU6AoqIi+cCeqWFqalpeXv7222/Lt4Tz9vb+8MMPtc4cXweFNAn7rbfesrS0HNNn8/Ly5EWlUvnA8Ui//vWv5bOW9ceJurq6vr4+QRAWLVr0/PPPG+FDcuHChdOnT8tr/Pz8pvUO38QJAACAibFw4cL33ntPXqNWq9PS0qa+JfdcaikuLk6rgyI7O3usq9n29vbm5+eLx+MY6STvZxAE4dlnn9V//pUrV+TT2R977DEfHx8950tv6kFBQffbYNuwdFfZmtZdE8QJAACAibRhwwatV/b09HQj2YDC3t5eqVTKa27dujXW7fYyMzPv3LkjCIKnp+d3vvOdsbZBq6/mqaee0n9+UlKSfMe9oKAg/b0Z0mQV4xzp1N3dnZWVJa957rnnvv/97xMnAAAA8P994xvfiIqKktd0dXXl5OQYSfOio6O1asa6lK000mkcXROCILS2tsqL+leFys/P17p1+idaqNVqqffjnqs/GdyePXvEMCZJTEw0zl6Uh8e+EwAAABNJpVItXbpUPtzfy8trfJOeJ5xarXZycrp27ZpUo1AompublyxZ8jAfv3DhwosvvijOO+/o6HjsscfG9NO7urrs7Oy0au65TJMgCO3t7e7u7gEBAfL+k8uXLz/99NPisUajyc/Pj4iIkP61pqbG29tbTClXrlwxtgdDrVa7uro2NzdLNU5OTo2Njca5mu3Do3cCAABgIj399NOrVq2S19TU1Pz97383ijc/hWLNmjVa77haK7fqkZ6eLh5ERESMNUuIMx/Mzc21MsM9z+zt7VUqlStXrly8eLFU6ejoKGUJQRASEhJ27Nghj23SNuTGOdKpuLhYniUEQYiPj5/uWYI4AQAAMPF0J9fqTsA1lOjoaK3RNfv27XuY4Sp3796Vxv2Pb6STmZmZ1uimM2fO6J7W19cXHBzc2dn529/+9uLFi1K9fPmjr7766uDBg1lZWQrF/95mT5w4IR5II506OjrCwsKWLFli+n/MzMzc3NykM6fS9u3b5UU7O7vx3UbiBAAAwAzn4eGxfPlyec2hQ4du3LhhDG176qmnAgIC5DVNTU1lZWUP/GB+fn5PT48gCK6urlpXeHhaSzklJyf39vbKa/72t7+99NJLly5dyszMtLa2ln+d7+joKB5s27YtMTExKytL3ndx586dyspKcYfpwMBAsdLBwaGoqOjUqVPq//L396+qqmpoaJj6mRUNDQ1aGSY2NtbKyoo4AQAAgHvYtGmTvDg0NLRr1y4jaZvuhOyH2YBCPgl73LOHP/74Y3l/QnNzs5eXV3p6enl5+Y4dO8LDwwMDA9vb248dO+br6ysuhSSdnJ2d/dVXX73xxhvx8fH79+8PCgqSX7miomJwcFAQBHd3d63tybds2WJqarp582Yxqxjknqempsq7gOzs7GJjY2fIs64BAADAJNCapWBjY9Pd3W0MDevv79fabs/Kyqqvr0/PR65cuSJGCFNT0/b29kf56e+8846eV1MbG5uzZ89KJzs5OWmdYGZmdujQId3LJiQkiCf87Gc/k9d/9tlnixYtKi0tNeANb21t1drvb8+ePTPmOad3AgAAYFJ88cUX8mWLenp6UlJSjKFh8+bNe/PNN+U1AwMDubm5+rsmxC/XQ0JCHBwcHuWnp6WlpaSkaO3OIbYqISGhoaFBvlGd1qR2Hx+fs2fPvvHGG7qX1Z04oVar4+LiDh06dO7cuVdeecWwXVXyvUf8/Px0O4imLxaKBQAAmCzp6enyL+MtLS1VKpU0B8CAqqqq/Pz85DW+vr5VVVX3PHl0dNTJyamtrU0QhCNHjqxYseLRG9DS0lJZWVlXV9fZ2fmtb33L1dX1lVde0RqkJAjC7du3s7KyCgsL3dzc/Pz8lErlPcdZ3bx584knntBoNObm5j09PfPmzevv74+Kiurr68vPzx/HIlQTqK6u7oUXXpBWoDIzMzt//vzzzz9PnAAAAMADaDQacfqvVBMTEyNNQjCspUuXNjQ0yGvq6+vd3Nx0zywuLg4NDRUEYfHixa2trUa4tmlubu6PfvQjcfWniooKlUqlVCo9PDz27t2rtTTt1AsNDS0uLpaK8fHxW7ZsmUkPOYOdAAAAJouJiUlaWpr8/TsjI+Nf//qXMbTt4SdkS9tNrFmzxjj3SZCPdCosLPT29q6vr9+yZYvBs0RZWZk8Szg6On766acz7CEnTgAAAEwid3f3uLg4qTg6Ovrzn//cGBoWFRVlamoqr9m/f//o6KjWadevX//Tn/4kHsfExBjnTZbiRFlZmVKp7O3tVavVUgoyoI8++khe3LZtm7W1NXECAAAAY/Dpp59+85vflIpHjx693yyFqeTg4PDaa6/Jazo6Oo4dO6Z1WkZGxsjIiCAIAQEBzzzzjBHe3sbGxqtXr4rHt27d+uSTT8TjnTt36qajqfTHP/5Rvht6SEhIeHj4zHu8iRMAAACTy9raetu2bfIarS+tDUW3t0F3vNPevXvFA6PdwlnqmvD09KysrExMTBR7AK5du3bkyBFDtWpkZETeDWVpaam1KzZxAgAAAA9r5cqV4mxm0enTpwsKCgzeqtDQUDs7O3lNUVFRZ2envJ3idO0FCxZEREQYeZxYtWrV3Llz58+fL07LFgRh69athmrVzp07VSqVVPzkk09cXFyIEwAAABin1NRU+TYUH3zwgXzLZ4OYM2fOj3/8Y3nN0NBQZmamVJSmH0RGRlpZWRnhXVWr1adOnRKPpR0nfvKTn0hx6Ny5c1Pfqubm5o8//lgqvvjii4mJiTP1wSZOAAAATAUnJ6fs7Gxp9vN//vOfn/70pwZvlZ71nfr6+vLy8sRjox3pVFtb29PTI26n7enpKVZ6eHgEBASIx1pbB07B8CeNRhMdHd3f3y8W7ezsCgoK5syZQ5wAAADAI1m+fHlycrJUzM7Ozs/PN2yT3N3dpbdw0cWLF8+fPy8IQk5OzsDAgHiOl5eXcd5SaaRTUFCQQvG/N9uNGzeKBwUFBdJ86PLy8pycnMlu0tatW8vKysRjU1PT3NxcJyenGfxUEycAAACmTkJCgjSyXxyWc/PmTcM26X4dFNJIp3Xr1hnt/fzrX/8qHkgjnUSvv/66v7+/1P7bt293dXW9//77X3zxxaS2p76+Xj4D+ze/+U1QUNDMfqTZFRsAAGBKDQwMvPzyyxcvXhSL4eHhhu2j6OnpcXBwGBwclGpsbW2PHz/u7e0tCIKFhUV7e7t81odR3UlbW1ux5SqVytXVVf6v1dXVvr6+4lqxjo6OfX19X3755aRGo5GRkZdffrm6ulosrl69Oisra8Y/z/ROAAAATCkrK6uCgoLHH39cLB4+fNiwL502NjZKpVJe093dLe2QsHLlSuPMEuJMazFLODs7a2UJQRC8vb03b94sHre1ta1fv36yu1lSUlKkLPHCCy8Ywz56U4DeCQAAAAM4ceLEa6+9Jn53bmtrW1dX5+DgYKjG/OUvfwkODr5fO5ctWzZ973NlZeWlS5d8fHw8PDwm9QdduHDhpZdeGh4eFv+gtbW1zs7OxAkAAABMli+//PLDDz8Uj319fU+dOmVpaWmQlqjVamdn59bWVq16Z2fnpqYmExMT/lj6dXZ2+vj4NDU1idOvS0pKXn311VnyuzPYCQAAwDDi4+MjIyPF4zNnzkRHRxvqe16FQrFmzRrd+piYGLLEA929e3fFihVilhAEITk5efZkCXonAAAADGlwcHDFihXHjx8Xi0lJSZ9//rlBWtLU1OTq6ip/M1QoFC0tLY6OjvyZ9NBoNJGRkdL6s3FxcQbcitswWZSHAAAAwFAsLCyOHDkiLXL6y1/+8uDBgwZpiYuLi7T1myg4OJgs8UBJSUlSlli/fv1syxLECQAAAAOztLQsLCyUdid4++23KyoqDNKSmJgYedFod8I2Hvv27fvVr34lHr///vvbt2+fhTeBwU4AAACGNzAwEBoaWlpaKgjC448/XlNTM/XrAg0MDCxevPjrr78WBMHe3r6trc3c3Jw/zf1UVFQsW7ZMXMrpnXfe2bVr1+ycZ0LvBAAAgOFZWVmVlJSIG2Z3dXXV1NQYpA0//OEPxeOoqCiyhH4nT54Us0RSUtLvf//7WTtnnTgBAABgFCwsLLKysj766CMDtkEa78RIp4dhZma2Z88eQ82eN5abwHMAAABgJExMTFJSUpydnU1NTQ3SAF9fX5VKpVAoXFxc+HPot2DBguLi4uXLl8/2h5a5EwAAAMZGrVYrFIwi4W9EnAAAAAAwc5GoAAAAABAnAAAAABAnAAAAABAnAAAAABAnAAAAAIA4AQAAAIA4AQAAAIA4AQAAAIA4AQAAAIA4AQAAAADECQAAAADECQAAAADECQAAAADECQAAAADECQAAAAAgTgAAAAAgTgAAAAAgTgAAAAAgTgAAAAAAcQIAAAAAcQIAAAAAcQIAAAAAcQIAAAAAcQIAAAAAiBMAAAAAiBMAAAAAiBMAAAAAiBMAAAAAiBMAAAAAQJwAAAAAQJwAAAAAQJwAAAAAQJwAAAAAAOIEAAAAAOIEAAAAAOIEAAAAAOIEAAAAAOIEAAAAABAnAAAAABAnAAAAABAnAAAAABAnAAAAAMxO/y8AAP//+/YfLypb24wAAAAASUVORK5CYII=\n",
"text/plain": [
"<IPython.core.display.Image object>"
]
},
"execution_count": 1,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"from IPython.display import Image\n",
"Image(filename='./images/qkv.png')"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"ExecuteTime": {
"end_time": "2022-08-23T14:37:24.764368Z",
"start_time": "2022-08-23T14:37:17.596209Z"
}
},
"outputs": [],
"source": [
"import torch\n",
"from torch import nn\n",
"import math\n",
"from bertviz.transformers_neuron_view import BertModel, BertConfig\n",
"from transformers import BertTokenizer"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 1. model config and load"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"ExecuteTime": {
"end_time": "2022-08-23T14:37:43.593937Z",
"start_time": "2022-08-23T14:37:26.027208Z"
}
},
"outputs": [],
"source": [
"max_length = 256\n",
"model_name = 'bert-base-uncased'\n",
"config = BertConfig.from_pretrained(model_name, output_attentions=True, \n",
" output_hidden_states=True, \n",
" return_dict=True)\n",
"tokenizer = BertTokenizer.from_pretrained(model_name)\n",
"config.max_position_embeddings = max_length\n",
"\n",
"model = BertModel(config).from_pretrained(model_name)\n",
"model = model.eval()"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"ExecuteTime": {
"end_time": "2022-08-23T14:37:49.297031Z",
"start_time": "2022-08-23T14:37:49.282116Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"{\n",
" \"architectures\": [\n",
" \"BertForMaskedLM\"\n",
" ],\n",
" \"attention_probs_dropout_prob\": 0.1,\n",
" \"finetuning_task\": null,\n",
" \"hidden_act\": \"gelu\",\n",
" \"hidden_dropout_prob\": 0.1,\n",
" \"hidden_size\": 768,\n",
" \"initializer_range\": 0.02,\n",
" \"intermediate_size\": 3072,\n",
" \"layer_norm_eps\": 1e-12,\n",
" \"max_position_embeddings\": 512,\n",
" \"model_type\": \"bert\",\n",
" \"num_attention_heads\": 12,\n",
" \"num_hidden_layers\": 12,\n",
" \"num_labels\": 2,\n",
" \"output_attentions\": true,\n",
" \"output_hidden_states\": false,\n",
" \"pad_token_id\": 0,\n",
" \"torchscript\": false,\n",
" \"type_vocab_size\": 2,\n",
" \"vocab_size\": 30522\n",
"}"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"model.config"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"ExecuteTime": {
"end_time": "2022-08-23T14:38:37.060053Z",
"start_time": "2022-08-23T14:38:37.055373Z"
}
},
"outputs": [],
"source": [
"att_head_size = int(model.config.hidden_size/model.config.num_attention_heads)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"ExecuteTime": {
"end_time": "2022-08-23T14:38:42.881098Z",
"start_time": "2022-08-23T14:38:42.874824Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"64"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"att_head_size"
]
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {
"ExecuteTime": {
"end_time": "2022-08-23T14:41:53.748482Z",
"start_time": "2022-08-23T14:41:53.742105Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"tensor([[-0.0112, -0.0324, -0.0615, ..., -0.0383, 0.0031, 0.0059],\n",
" [ 0.0260, -0.0067, -0.0616, ..., 0.1097, 0.0029, -0.0540],\n",
" [-0.0169, 0.0232, 0.0068, ..., 0.0124, -0.0168, 0.0301],\n",
" ...,\n",
" [ 0.1083, 0.0056, 0.0968, ..., 0.0188, -0.0171, 0.0141],\n",
" [-0.0436, -0.1032, -0.1035, ..., 0.0138, -0.0488, -0.0453],\n",
" [-0.0611, 0.0224, -0.0320, ..., 0.0376, 0.0186, -0.0482]],\n",
" grad_fn=<SliceBackward0>)"
]
},
"execution_count": 21,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"model.encoder.layer[0].attention.self.query.weight.T[:, 64:128]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 2. data"
]
},
{
"cell_type": "code",
"execution_count": 22,
"metadata": {
"ExecuteTime": {
"end_time": "2022-08-23T14:42:30.683705Z",
"start_time": "2022-08-23T14:42:29.810137Z"
}
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/Users/chunhuizhang/anaconda3/lib/python3.6/site-packages/transformers/tokenization_utils_base.py:2227: UserWarning: `max_length` is ignored when `padding`=`True`.\n",
" warnings.warn(\"`max_length` is ignored when `padding`=`True`.\")\n"
]
}
],
"source": [
"from sklearn.datasets import fetch_20newsgroups\n",
"newsgroups_train = fetch_20newsgroups(subset='train')\n",
"inputs_tests = tokenizer(newsgroups_train['data'][:1], \n",
" truncation=True, padding=True, max_length=max_length, \n",
" return_tensors='pt')"
]
},
{
"cell_type": "code",
"execution_count": 25,
"metadata": {
"ExecuteTime": {
"end_time": "2022-08-23T14:43:26.668044Z",
"start_time": "2022-08-23T14:43:26.663985Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"torch.Size([1, 201])"
]
},
"execution_count": 25,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"inputs_tests['input_ids'].shape"
]
},
{
"cell_type": "code",
"execution_count": 37,
"metadata": {
"ExecuteTime": {
"end_time": "2022-08-23T14:46:49.614873Z",
"start_time": "2022-08-23T14:46:49.610494Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"dict_keys(['input_ids', 'token_type_ids', 'attention_mask'])"
]
},
"execution_count": 37,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"inputs_tests.keys()"
]
},
{
"cell_type": "markdown",
"metadata": {
"ExecuteTime": {
"end_time": "2022-08-23T14:14:25.826897Z",
"start_time": "2022-08-23T14:14:25.824419Z"
}
},
"source": [
"## 3. model output"
]
},
{
"cell_type": "code",
"execution_count": 26,
"metadata": {
"ExecuteTime": {
"end_time": "2022-08-23T14:43:51.861587Z",
"start_time": "2022-08-23T14:43:51.134234Z"
}
},
"outputs": [],
"source": [
"model_output = model(**inputs_tests)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"- last_hidden_state (batch_size, sequence_length, hidden_size) : last hidden state which is outputted from the last BertLayer\n",
"- pooler_output (batch_size, hidden_size) : output of the Pooler layer\n",
"- hidden_states (batch_size, sequence_length, hidden_size): hidden-states of the model at the output of each BertLayer plus the initial embedding\n",
"- attentions (batch_size, num_heads, sequence_length, sequence_length): one for each BertLayer. Attentions weights after the attention SoftMax"
]
},
{
"cell_type": "code",
"execution_count": 27,
"metadata": {
"ExecuteTime": {
"end_time": "2022-08-23T14:44:04.984971Z",
"start_time": "2022-08-23T14:44:04.981312Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"3"
]
},
"execution_count": 27,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(model_output)"
]
},
{
"cell_type": "code",
"execution_count": 31,
"metadata": {
"ExecuteTime": {
"end_time": "2022-08-23T14:44:36.364871Z",
"start_time": "2022-08-23T14:44:36.360873Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"dict_keys(['attn', 'queries', 'keys'])"
]
},
"execution_count": 31,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"model_output[-1][0].keys()"
]
},
{
"cell_type": "code",
"execution_count": 36,
"metadata": {
"ExecuteTime": {
"end_time": "2022-08-23T14:46:08.927665Z",
"start_time": "2022-08-23T14:46:08.921389Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"tensor([[0.0053, 0.0109, 0.0052, ..., 0.0039, 0.0036, 0.0144],\n",
" [0.0086, 0.0041, 0.0125, ..., 0.0045, 0.0041, 0.0071],\n",
" [0.0051, 0.0043, 0.0046, ..., 0.0043, 0.0045, 0.0031],\n",
" ...,\n",
" [0.0010, 0.0023, 0.0055, ..., 0.0012, 0.0018, 0.0011],\n",
" [0.0010, 0.0023, 0.0057, ..., 0.0012, 0.0017, 0.0007],\n",
" [0.0022, 0.0056, 0.0063, ..., 0.0045, 0.0048, 0.0015]],\n",
" grad_fn=<SliceBackward0>)"
]
},
"execution_count": 36,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"model_output[-1][0]['attn'][0, 0, :, :]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 4. from scratch"
]
},
{
"cell_type": "code",
"execution_count": 38,
"metadata": {
"ExecuteTime": {
"end_time": "2022-08-23T14:46:57.878825Z",
"start_time": "2022-08-23T14:46:57.868066Z"
}
},
"outputs": [],
"source": [
"emb_output = model.embeddings(inputs_tests['input_ids'], inputs_tests['token_type_ids'])"
]
},
{
"cell_type": "code",
"execution_count": 39,
"metadata": {
"ExecuteTime": {
"end_time": "2022-08-23T14:47:06.547071Z",
"start_time": "2022-08-23T14:47:06.542839Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"torch.Size([1, 201, 768])"
]
},
"execution_count": 39,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"emb_output.shape"
]
},
{
"cell_type": "code",
"execution_count": 40,
"metadata": {
"ExecuteTime": {
"end_time": "2022-08-23T14:48:17.703437Z",
"start_time": "2022-08-23T14:48:17.698015Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"BertLayer(\n",
" (attention): BertAttention(\n",
" (self): BertSelfAttention(\n",
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (output): BertSelfOutput(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (LayerNorm): BertLayerNorm()\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (intermediate): BertIntermediate(\n",
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
" )\n",
" (output): BertOutput(\n",
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
" (LayerNorm): BertLayerNorm()\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
")"
]
},
"execution_count": 40,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# model.encoder.layer[0]"
]
},
{
"cell_type": "code",
"execution_count": 42,
"metadata": {
"ExecuteTime": {
"end_time": "2022-08-23T14:49:09.030869Z",
"start_time": "2022-08-23T14:49:09.026963Z"
}
},
"outputs": [],
"source": [
"# emb_output[0].shape: 201*768\n",
"# query.weight.T.shape: 768*768, query.weight.T[:, :att_head_size]: 768*64\n",
"# 201*64\n",
"Q_first_head_first_layer = emb_output[0] @ model.encoder.layer[0].attention.self.query.weight.T[:, :att_head_size] \\\n",
" + model.encoder.layer[0].attention.self.query.bias[:att_head_size]"
]
},
{
"cell_type": "code",
"execution_count": 44,
"metadata": {
"ExecuteTime": {
"end_time": "2022-08-23T14:51:02.471342Z",
"start_time": "2022-08-23T14:51:02.467129Z"
}
},
"outputs": [],
"source": [
"# 201*64\n",
"K_first_head_first_layer = emb_output[0] @ model.encoder.layer[0].attention.self.key.weight.T[:, :att_head_size] \\\n",
" + model.encoder.layer[0].attention.self.key.bias[:att_head_size]"
]
},
{
"cell_type": "code",
"execution_count": 48,
"metadata": {
"ExecuteTime": {
"end_time": "2022-08-23T14:53:33.339672Z",
"start_time": "2022-08-23T14:53:33.335941Z"
}
},
"outputs": [],
"source": [
"# (201*64)*(64*201) ==> 201*201\n",
"attn_scores = torch.nn.Softmax(dim=-1)(Q_first_head_first_layer @ K_first_head_first_layer.T / math.sqrt(att_head_size))"
]
},
{
"cell_type": "code",
"execution_count": 46,
"metadata": {
"ExecuteTime": {
"end_time": "2022-08-23T14:52:07.447498Z",
"start_time": "2022-08-23T14:52:07.442701Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"tensor([[0.0053, 0.0109, 0.0052, ..., 0.0039, 0.0036, 0.0144],\n",
" [0.0086, 0.0041, 0.0125, ..., 0.0045, 0.0041, 0.0071],\n",
" [0.0051, 0.0043, 0.0046, ..., 0.0043, 0.0045, 0.0031],\n",
" ...,\n",
" [0.0010, 0.0023, 0.0055, ..., 0.0012, 0.0018, 0.0011],\n",
" [0.0010, 0.0023, 0.0057, ..., 0.0012, 0.0017, 0.0007],\n",
" [0.0022, 0.0056, 0.0063, ..., 0.0045, 0.0048, 0.0015]],\n",
" grad_fn=<SoftmaxBackward0>)"
]
},
"execution_count": 46,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"attn_scores"
]
},
{
"cell_type": "code",
"execution_count": 52,
"metadata": {
"ExecuteTime": {
"end_time": "2022-08-23T14:56:15.178493Z",
"start_time": "2022-08-23T14:56:15.173311Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"tensor([1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000,\n",
" 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000,\n",
" 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000,\n",
" 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000,\n",
" 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000,\n",
" 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000,\n",
" 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000,\n",
" 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000,\n",
" 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000,\n",
" 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000,\n",
" 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000,\n",
" 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000,\n",
" 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000,\n",
" 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000,\n",
" 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000,\n",
" 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000,\n",
" 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000,\n",
" 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000,\n",
" 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000,\n",
" 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000,\n",
" 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000,\n",
" 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000,\n",
" 1.0000, 1.0000, 1.0000], grad_fn=<SumBackward1>)"
]
},
"execution_count": 52,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"attn_scores.sum(dim=-1)"
]
},
{
"cell_type": "code",
"execution_count": 49,
"metadata": {
"ExecuteTime": {
"end_time": "2022-08-23T14:54:01.596921Z",
"start_time": "2022-08-23T14:54:01.592186Z"
}
},
"outputs": [],
"source": [
"V_first_head_first_layer = emb_output[0] @ model.encoder.layer[0].attention.self.value.weight.T[:, :att_head_size] \\\n",
" + model.encoder.layer[0].attention.self.value.bias[:att_head_size]"
]
},
{
"cell_type": "code",
"execution_count": 50,
"metadata": {
"ExecuteTime": {
"end_time": "2022-08-23T14:54:15.701673Z",
"start_time": "2022-08-23T14:54:15.698925Z"
}
},
"outputs": [],
"source": [
"attn_emb = attn_scores @ V_first_head_first_layer"
]
},
{
"cell_type": "code",
"execution_count": 51,
"metadata": {
"ExecuteTime": {
"end_time": "2022-08-23T14:54:24.153734Z",
"start_time": "2022-08-23T14:54:24.149518Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"torch.Size([201, 64])"
]
},
"execution_count": 51,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"attn_emb.shape"
]
},
{
"cell_type": "code",
"execution_count": 69,
"metadata": {
"ExecuteTime": {
"end_time": "2022-08-29T13:20:49.611750Z",
"start_time": "2022-08-29T13:20:49.607269Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"tensor([[ 2.3786e+00, -9.7945e-02, -2.8436e-01, ..., 9.6968e-02,\n",
" -1.8537e-01, 2.1132e-01],\n",
" [ 9.8886e-02, 2.2942e-01, -3.9613e-01, ..., 2.0846e-01,\n",
" -1.1224e-01, -2.1420e-01],\n",
" [-6.3356e-01, 1.1083e+00, -5.2859e-04, ..., -3.1870e-01,\n",
" 7.2633e-02, -1.0239e-01],\n",
" ...,\n",
" [-7.9659e-01, -4.9400e-01, -4.9216e-02, ..., -3.6446e-01,\n",
" 3.1565e-01, -7.1713e-01],\n",
" [-9.2551e-01, -5.0348e-01, -1.0398e-01, ..., -2.1418e-01,\n",
" 1.6604e-01, -5.9637e-01],\n",
" [ 1.2915e-01, -7.5900e-03, 1.8397e-01, ..., 2.6980e-01,\n",
" 1.3651e-01, 1.9180e-01]], grad_fn=<AddBackward0>)"
]
},
"execution_count": 69,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"V_first_head_first_layer"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### 5. all"
]
},
{
"cell_type": "code",
"execution_count": 60,
"metadata": {
"ExecuteTime": {
"end_time": "2022-08-29T13:13:35.213456Z",
"start_time": "2022-08-29T13:13:35.195874Z"
}
},
"outputs": [],
"source": [
"Q_first_layer = emb_output[0] @ model.encoder.layer[0].attention.self.query.weight.T \\\n",
" + model.encoder.layer[0].attention.self.query.bias\n",
"K_first_layer = emb_output[0] @ model.encoder.layer[0].attention.self.key.weight.T \\\n",
" + model.encoder.layer[0].attention.self.key.bias\n",
"V_first_layer = emb_output[0] @ model.encoder.layer[0].attention.self.value.weight.T \\\n",
" + model.encoder.layer[0].attention.self.value.bias"
]
},
{
"cell_type": "code",
"execution_count": 61,
"metadata": {
"ExecuteTime": {
"end_time": "2022-08-29T13:13:44.657849Z",
"start_time": "2022-08-29T13:13:44.639225Z"
}
},
"outputs": [],
"source": [
"scores = torch.nn.Softmax(dim=-1)(Q_first_layer @ K_first_layer.T / math.sqrt(64))"
]
},
{
"cell_type": "code",
"execution_count": 68,
"metadata": {
"ExecuteTime": {
"end_time": "2022-08-29T13:20:43.532343Z",
"start_time": "2022-08-29T13:20:43.527666Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"tensor([[ 2.3786e+00, -9.7945e-02, -2.8436e-01, ..., 9.6968e-02,\n",
" -1.8537e-01, 2.1132e-01],\n",
" [-6.2984e-01, 1.1037e+00, -2.0764e-04, ..., -3.1767e-01,\n",
" 7.2197e-02, -1.0126e-01],\n",
" [ 3.7932e-02, 2.9540e-01, -3.6121e-01, ..., 1.6694e-01,\n",
" -9.9637e-02, -2.0356e-01],\n",
" ...,\n",
" [-7.0623e-01, -3.0113e-01, 9.4959e-02, ..., -1.7217e-01,\n",
" 1.8647e-01, -4.4743e-01],\n",
" [ 6.2047e-02, -4.0626e-02, 1.6757e-01, ..., 2.2690e-01,\n",
" 1.4684e-01, 1.3030e-01],\n",
" [ 2.3748e+00, -9.7801e-02, -2.8357e-01, ..., 9.7254e-02,\n",
" -1.8481e-01, 2.1127e-01]], grad_fn=<MmBackward0>)"
]
},
"execution_count": 68,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"scores @ V_first_layer[:, :64]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.8"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
|
