From 8a1442995b0c9f120ca907b1009c2bc7ad969e85 Mon Sep 17 00:00:00 2001 From: monoid Date: Sun, 27 Feb 2022 19:50:13 +0900 Subject: [PATCH] feat: train, dev, test --- Batch.ipynb | 14 +- Training.ipynb | 732 +++++++++++++++++++++++++++++++++++++++++++------ ndataset.py | 8 +- 3 files changed, 656 insertions(+), 98 deletions(-) diff --git a/Batch.ipynb b/Batch.ipynb index 6eef219..1f59c22 100644 --- a/Batch.ipynb +++ b/Batch.ipynb @@ -3,7 +3,7 @@ { "cell_type": "code", "execution_count": 1, - "id": "c916dd3b", + "id": "5a4a1e30", "metadata": {}, "outputs": [ { @@ -25,7 +25,7 @@ { "cell_type": "code", "execution_count": 2, - "id": "d5861234", + "id": "710cd5b2", "metadata": {}, "outputs": [], "source": [ @@ -39,7 +39,7 @@ { "cell_type": "code", "execution_count": 3, - "id": "5accd3a9", + "id": "da018ffe", "metadata": {}, "outputs": [ { @@ -68,7 +68,7 @@ }, { "cell_type": "markdown", - "id": "d10fcb83", + "id": "69f05cf6", "metadata": {}, "source": [ "data를 준비" @@ -77,7 +77,7 @@ { "cell_type": "code", "execution_count": 7, - "id": "552fe555", + "id": "961edd10", "metadata": {}, "outputs": [ { @@ -114,7 +114,7 @@ }, { "cell_type": "markdown", - "id": "1cff8e03", + "id": "4178b576", "metadata": {}, "source": [ "간단한 collate function" @@ -123,7 +123,7 @@ { "cell_type": "code", "execution_count": null, - "id": "89eb64d8", + "id": "a5ff0049", "metadata": {}, "outputs": [], "source": [] diff --git a/Training.ipynb b/Training.ipynb index 292337a..1476832 100644 --- a/Training.ipynb +++ b/Training.ipynb @@ -79,21 +79,21 @@ "output_type": "stream", "text": [ "read train set\n", - "100%|██████████████████████████████████████████████████████████████████████| 150000/150000 [00:00<00:00, 208333.74it/s]\n", + "100%|██████████████████████████████████████████████████████████████████████| 150000/150000 [00:00<00:00, 208913.67it/s]\n", "read test set\n", - "100%|████████████████████████████████████████████████████████████████████████| 50000/50000 [00:00<00:00, 260420.34it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████| 50000/50000 [00:00<00:00, 259067.57it/s]\n" ] } ], "source": [ "from ndataset import readNsmcDataAll, make_collate_fn\n", - "dataTrain, dataTest = readNsmcDataAll()\n", + "dataTrain, dataDev, dataTest = readNsmcDataAll()\n", "collate_fn = make_collate_fn(tokenizer)" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 25, "id": "650c8a19", "metadata": {}, "outputs": [ @@ -101,7 +101,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "Some weights of the model checkpoint at bert-base-multilingual-cased were not used when initializing BertModel: ['cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.bias', 'cls.predictions.bias', 'cls.seq_relationship.weight', 'cls.predictions.decoder.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.dense.weight']\n", + "Some weights of the model checkpoint at bert-base-multilingual-cased were not used when initializing BertModel: ['cls.seq_relationship.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.bias', 'cls.predictions.bias', 'cls.predictions.transform.LayerNorm.bias']\n", "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n" ] @@ -135,13 +135,14 @@ " super().__init__()\n", " self.bert = bert\n", " self.dropout = nn.Dropout(p=0.1)\n", - " self.lin1 = nn.Linear(768,256) #[batch_size,768] -> [batch_size,256]\n", - " self.lin2 = nn.Linear(256,1) #[batch_size,256] -> [batch_size,1]\n", + " self.lin1 = nn.Linear(768,768) #[batch_size,768] -> [batch_size,768]\n", + " self.gelu = nn.GELU()\n", + " self.lin2 = nn.Linear(768,1) #[batch_size,768] -> [batch_size,1]\n", "\n", " def forward(self,**kargs):\n", " emb = self.bert(**kargs)\n", " e1 = self.dropout(emb['pooler_output'])\n", - " e2 = self.lin1(e1)\n", + " e2 = self.gelu(self.lin1(e1))\n", " w = self.lin2(e2)\n", " return w.squeeze() #[batch_size]" ] @@ -164,17 +165,54 @@ "model = MyModel(bert)" ] }, + { + "cell_type": "code", + "execution_count": 8, + "id": "0e36d9e2", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "\"\\nclass MyModel(nn.Module):\\n def __init__(self,embeddings):\\n super().__init__()\\n self.embeddings = embeddings\\n self.lin0 = nn.Linear(768,1)\\n \\n def forward(self,**kargs):\\n emb = self.embeddings(kargs['input_ids']) #[batch_size,word,768]\\n e0 = emb * kargs['attention_mask'].unsqueeze(dim=-1)#[batch_size,word,768]\\n e1 = self.lin0(e0)#[batch_size,word,1]\\n w = (e1.sum(axis=1))#[batch_size,1]\\n return w.squeeze() #[batch_size]\\nmodel = MyModel(bert.embeddings)\\nfor param in model.embeddings.parameters():\\n param.requires_grad = False\\n\"" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "class MyModel(nn.Module):\n", + " def __init__(self,embeddings):\n", + " super().__init__()\n", + " self.embeddings = embeddings\n", + " self.lin0 = nn.Linear(768,1)\n", + " \n", + " def forward(self,**kargs):\n", + " emb = self.embeddings(kargs['input_ids']) #[batch_size,word,768]\n", + " e0 = emb * kargs['attention_mask'].unsqueeze(dim=-1)#[batch_size,word,768]\n", + " e1 = self.lin0(e0)#[batch_size,word,1]\n", + " w = (e1.sum(axis=1))#[batch_size,1]\n", + " return w.squeeze() #[batch_size]\n", + "model = MyModel(bert.embeddings)\n", + "for param in model.embeddings.parameters():\n", + " param.requires_grad = False\n", + "\"\"\"" + ] + }, { "cell_type": "markdown", "id": "7969fead", "metadata": {}, "source": [ - "학습 과정에서 벌어지는 일" + "### 학습 과정에서 벌어지는 일" ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 9, "id": "8c2a4bc9", "metadata": {}, "outputs": [ @@ -476,12 +514,13 @@ " )\n", " )\n", " (dropout): Dropout(p=0.1, inplace=False)\n", - " (lin1): Linear(in_features=768, out_features=256, bias=True)\n", - " (lin2): Linear(in_features=256, out_features=1, bias=True)\n", + " (lin1): Linear(in_features=768, out_features=768, bias=True)\n", + " (gelu): GELU()\n", + " (lin2): Linear(in_features=768, out_features=1, bias=True)\n", ")" ] }, - "execution_count": 8, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } @@ -493,6 +532,38 @@ { "cell_type": "code", "execution_count": 10, + "id": "def10f6c", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'input_ids': tensor([[ 101, 9405, 62200, 14523, 48549, 119, 102, 0],\n", + " [ 101, 9294, 12424, 69592, 48549, 119, 102, 0],\n", + " [ 101, 9479, 68984, 48549, 119, 102, 0, 0],\n", + " [ 101, 9659, 22458, 119192, 12965, 48549, 119, 102]]), 'token_type_ids': tensor([[0, 0, 0, 0, 0, 0, 0, 0],\n", + " [0, 0, 0, 0, 0, 0, 0, 0],\n", + " [0, 0, 0, 0, 0, 0, 0, 0],\n", + " [0, 0, 0, 0, 0, 0, 0, 0]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 0],\n", + " [1, 1, 1, 1, 1, 1, 1, 0],\n", + " [1, 1, 1, 1, 1, 1, 0, 0],\n", + " [1, 1, 1, 1, 1, 1, 1, 1]])}" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "inputs = tokenizer([\"사랑해요.\",\"무서워요.\",\"슬퍼요.\",\"재미있어요.\"],\n", + " return_tensors = 'pt', padding='longest')\n", + "inputs" + ] + }, + { + "cell_type": "code", + "execution_count": 11, "id": "e027b926", "metadata": {}, "outputs": [ @@ -502,29 +573,29 @@ "torch.Size([4, 768])" ] }, - "execution_count": 10, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "hidden = bert(**tokenizer([\"사랑해요.\",\"무서워요.\",\"슬퍼요.\",\"재미있어요.\"], return_tensors = 'pt', padding='longest'))['pooler_output']\n", + "hidden = bert(**inputs)['pooler_output']\n", "hidden.size()" ] }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 12, "id": "ae9f8fba", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "tensor([0.1623, 0.1365, 0.1949, 0.1491], grad_fn=)" + "tensor([-0.0180, -0.0823, -0.0234, -0.0886], grad_fn=)" ] }, - "execution_count": 14, + "execution_count": 12, "metadata": {}, "output_type": "execute_result" } @@ -536,17 +607,17 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 13, "id": "5470c3f8", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "tensor([0.5405, 0.5341, 0.5486, 0.5372], grad_fn=)" + "tensor([0.4955, 0.4794, 0.4942, 0.4779], grad_fn=)" ] }, - "execution_count": 15, + "execution_count": 13, "metadata": {}, "output_type": "execute_result" } @@ -557,7 +628,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 14, "id": "b7eb8e67", "metadata": {}, "outputs": [], @@ -567,18 +638,18 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 15, "id": "7a324ed7", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "tensor(0.6989, dtype=torch.float64,\n", + "tensor(0.6937, dtype=torch.float64,\n", " grad_fn=)" ] }, - "execution_count": 17, + "execution_count": 15, "metadata": {}, "output_type": "execute_result" } @@ -589,17 +660,17 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 16, "id": "cb54294d", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "tensor([ True, False, False, True])" + "tensor([False, True, True, False])" ] }, - "execution_count": 18, + "execution_count": 16, "metadata": {}, "output_type": "execute_result" } @@ -616,6 +687,14 @@ "이런 일이 벌어짐. sigmoid 는 나중에" ] }, + { + "cell_type": "markdown", + "id": "9875ce48", + "metadata": {}, + "source": [ + "### Model Training" + ] + }, { "cell_type": "code", "execution_count": 28, @@ -921,8 +1000,9 @@ " )\n", " )\n", " (dropout): Dropout(p=0.1, inplace=False)\n", - " (lin1): Linear(in_features=768, out_features=256, bias=True)\n", - " (lin2): Linear(in_features=256, out_features=1, bias=True)\n", + " (lin1): Linear(in_features=768, out_features=768, bias=True)\n", + " (gelu): GELU()\n", + " (lin2): Linear(in_features=768, out_features=1, bias=True)\n", ")\n" ] } @@ -935,7 +1015,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 29, "id": "b9380dcd", "metadata": {}, "outputs": [ @@ -945,7 +1025,7 @@ "device(type='cuda', index=0)" ] }, - "execution_count": 20, + "execution_count": 29, "metadata": {}, "output_type": "execute_result" } @@ -964,7 +1044,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 30, "id": "74c4becc", "metadata": {}, "outputs": [], @@ -977,6 +1057,12 @@ " shuffle=True,\n", " collate_fn=collate_fn\n", ")\n", + "dev_loader = DataLoader(\n", + " dataDev,\n", + " batch_size=BATCH_SIZE,\n", + " shuffle=True,\n", + " collate_fn=collate_fn\n", + ")\n", "test_loader = DataLoader(\n", " dataTest,\n", " batch_size=BATCH_SIZE,\n", @@ -995,7 +1081,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 31, "id": "3cd5bf7b", "metadata": {}, "outputs": [], @@ -1007,7 +1093,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 32, "id": "65b5ccde", "metadata": {}, "outputs": [], @@ -1026,7 +1112,44 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 33, + "id": "68183cf4", + "metadata": {}, + "outputs": [], + "source": [ + "def freezeParameter(model):\n", + " for param in model.bert.parameters():\n", + " param.requires_grad = False\n", + " for param in model.bert.embeddings.parameters():\n", + " param.requires_grad = True" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "42d1a29f", + "metadata": {}, + "outputs": [], + "source": [ + "#freezeParameter(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "cc8a82a5", + "metadata": {}, + "outputs": [], + "source": [ + "TRAIN_EPOCH = 5\n", + "\n", + "result = []\n", + "iteration = 0" + ] + }, + { + "cell_type": "code", + "execution_count": 36, "id": "4835a0d3", "metadata": {}, "outputs": [ @@ -1041,7 +1164,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "Epoch 0: 100%|███████████████████████████████████| 9375/9375 [12:35<00:00, 12.41minibatch/s, accuracy=0.875, loss=2.58]\n" + "Epoch 0: 100%|███████████████████████████████████| 9375/9375 [11:41<00:00, 13.37minibatch/s, accuracy=0.844, loss=5.79]\n" ] }, { @@ -1055,7 +1178,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "Epoch 1: 100%|███████████████████████████████████| 9375/9375 [12:35<00:00, 12.41minibatch/s, accuracy=0.898, loss=2.18]\n" + "Epoch 1: 100%|████████████████████████████████████| 9375/9375 [11:48<00:00, 13.23minibatch/s, accuracy=0.84, loss=5.37]\n" ] }, { @@ -1069,7 +1192,35 @@ "name": "stderr", "output_type": "stream", "text": [ - "Epoch 2: 1%|▎ | 82/9375 [00:06<12:30, 12.39minibatch/s, accuracy=0.867, loss=2.08]\n" + "Epoch 2: 100%|███████████████████████████████████| 9375/9375 [11:44<00:00, 13.31minibatch/s, accuracy=0.879, loss=4.79]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch 3 start:\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Epoch 3: 100%|███████████████████████████████████| 9375/9375 [11:42<00:00, 13.35minibatch/s, accuracy=0.852, loss=5.12]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch 4 start:\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Epoch 4: 8%|██▊ | 717/9375 [00:53<10:48, 13.36minibatch/s, accuracy=0.918, loss=3.09]\n" ] }, { @@ -1079,20 +1230,24 @@ "traceback": [ "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[1;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", - "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_10708/1191029387.py\u001b[0m in \u001b[0;36m\u001b[1;34m\u001b[0m\n\u001b[0;32m 21\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 22\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mmini_i\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mmini_l\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mbatch\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 23\u001b[1;33m \u001b[0mbatch_inputs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m{\u001b[0m\u001b[0mk\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mv\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mto\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mk\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mv\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mlist\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmini_i\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 24\u001b[0m \u001b[0mbatch_labels\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmini_l\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mto\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 25\u001b[0m \u001b[0mattention_mask\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbatch_inputs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"attention_mask\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_10708/1191029387.py\u001b[0m in \u001b[0;36m\u001b[1;34m(.0)\u001b[0m\n\u001b[0;32m 21\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 22\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mmini_i\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mmini_l\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mbatch\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 23\u001b[1;33m \u001b[0mbatch_inputs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m{\u001b[0m\u001b[0mk\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mv\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mto\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mk\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mv\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mlist\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmini_i\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 24\u001b[0m \u001b[0mbatch_labels\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmini_l\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mto\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 25\u001b[0m \u001b[0mattention_mask\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbatch_inputs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"attention_mask\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_29640/636965671.py\u001b[0m in \u001b[0;36m\u001b[1;34m\u001b[0m\n\u001b[0;32m 17\u001b[0m \u001b[0mbatch_labels\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmini_l\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mto\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 18\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 19\u001b[1;33m \u001b[0moutput\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m**\u001b[0m\u001b[0mbatch_inputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 20\u001b[0m \u001b[0mloss\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mBCELoss\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch_labels\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdouble\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 21\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;32m~\\anaconda3\\envs\\nn\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m 1100\u001b[0m if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[0;32m 1101\u001b[0m or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[1;32m-> 1102\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 1103\u001b[0m \u001b[1;31m# Do not call functions when jit is used\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1104\u001b[0m \u001b[0mfull_backward_hooks\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_29640/2430988958.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, **kargs)\u001b[0m\n\u001b[0;32m 11\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 12\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m**\u001b[0m\u001b[0mkargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 13\u001b[1;33m \u001b[0memb\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbert\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m**\u001b[0m\u001b[0mkargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 14\u001b[0m \u001b[0me1\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdropout\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0memb\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'pooler_output'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 15\u001b[0m \u001b[0me2\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgelu\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlin1\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0me1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;32m~\\anaconda3\\envs\\nn\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m 1100\u001b[0m if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[0;32m 1101\u001b[0m or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[1;32m-> 1102\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 1103\u001b[0m \u001b[1;31m# Do not call functions when jit is used\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1104\u001b[0m \u001b[0mfull_backward_hooks\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;32m~\\anaconda3\\envs\\nn\\lib\\site-packages\\transformers\\models\\bert\\modeling_bert.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, input_ids, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, encoder_hidden_states, encoder_attention_mask, past_key_values, use_cache, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[0;32m 1004\u001b[0m \u001b[0moutput_attentions\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0moutput_attentions\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1005\u001b[0m \u001b[0moutput_hidden_states\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0moutput_hidden_states\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1006\u001b[1;33m \u001b[0mreturn_dict\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mreturn_dict\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 1007\u001b[0m )\n\u001b[0;32m 1008\u001b[0m \u001b[0msequence_output\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mencoder_outputs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;32m~\\anaconda3\\envs\\nn\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m 1100\u001b[0m if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[0;32m 1101\u001b[0m or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[1;32m-> 1102\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 1103\u001b[0m \u001b[1;31m# Do not call functions when jit is used\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1104\u001b[0m \u001b[0mfull_backward_hooks\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;32m~\\anaconda3\\envs\\nn\\lib\\site-packages\\transformers\\models\\bert\\modeling_bert.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_values, use_cache, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[0;32m 590\u001b[0m \u001b[0mencoder_attention_mask\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 591\u001b[0m \u001b[0mpast_key_value\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 592\u001b[1;33m \u001b[0moutput_attentions\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 593\u001b[0m )\n\u001b[0;32m 594\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;32m~\\anaconda3\\envs\\nn\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m 1100\u001b[0m if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[0;32m 1101\u001b[0m or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[1;32m-> 1102\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 1103\u001b[0m \u001b[1;31m# Do not call functions when jit is used\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1104\u001b[0m \u001b[0mfull_backward_hooks\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;32m~\\anaconda3\\envs\\nn\\lib\\site-packages\\transformers\\models\\bert\\modeling_bert.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions)\u001b[0m\n\u001b[0;32m 475\u001b[0m \u001b[0mhead_mask\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 476\u001b[0m \u001b[0moutput_attentions\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0moutput_attentions\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 477\u001b[1;33m \u001b[0mpast_key_value\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself_attn_past_key_value\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 478\u001b[0m )\n\u001b[0;32m 479\u001b[0m \u001b[0mattention_output\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself_attention_outputs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;32m~\\anaconda3\\envs\\nn\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m 1100\u001b[0m if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[0;32m 1101\u001b[0m or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[1;32m-> 1102\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 1103\u001b[0m \u001b[1;31m# Do not call functions when jit is used\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1104\u001b[0m \u001b[0mfull_backward_hooks\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;32m~\\anaconda3\\envs\\nn\\lib\\site-packages\\transformers\\models\\bert\\modeling_bert.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions)\u001b[0m\n\u001b[0;32m 407\u001b[0m \u001b[0mencoder_attention_mask\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 408\u001b[0m \u001b[0mpast_key_value\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 409\u001b[1;33m \u001b[0moutput_attentions\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 410\u001b[0m )\n\u001b[0;32m 411\u001b[0m \u001b[0mattention_output\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moutput\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself_outputs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhidden_states\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;32m~\\anaconda3\\envs\\nn\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m 1100\u001b[0m if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[0;32m 1101\u001b[0m or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[1;32m-> 1102\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 1103\u001b[0m \u001b[1;31m# Do not call functions when jit is used\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1104\u001b[0m \u001b[0mfull_backward_hooks\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;32m~\\anaconda3\\envs\\nn\\lib\\site-packages\\transformers\\models\\bert\\modeling_bert.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions)\u001b[0m\n\u001b[0;32m 304\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 305\u001b[0m \u001b[1;31m# Take the dot product between \"query\" and \"key\" to get the raw attention scores.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 306\u001b[1;33m \u001b[0mattention_scores\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmatmul\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mquery_layer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkey_layer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtranspose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m-\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 307\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 308\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mposition_embedding_type\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m\"relative_key\"\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mposition_embedding_type\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m\"relative_key_query\"\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", "\u001b[1;31mKeyboardInterrupt\u001b[0m: " ] } ], "source": [ - "TRAIN_EPOCH = 5\n", - "\n", - "result = []\n", - "iteration = 0\n", - "\n", - "t = []\n", - "\n", "model.zero_grad()\n", "\n", "for epoch in range(TRAIN_EPOCH):\n", @@ -1101,7 +1256,7 @@ " with tqdm(train_loader, unit=\"minibatch\") as tepoch:\n", " tepoch.set_description(f\"Epoch {epoch}\")\n", " \n", - " for batch in groupby_index(tepoch,8):\n", + " for batch in groupby_index(tepoch,16):\n", " corrects = 0\n", " totals = 0\n", " losses = 0\n", @@ -1129,7 +1284,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 18, "id": "81b69931", "metadata": {}, "outputs": [], @@ -1141,13 +1296,13 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 37, "id": "c3a73c68", "metadata": {}, "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYEAAAD4CAYAAAAKA1qZAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAAsTAAALEwEAmpwYAABU1UlEQVR4nO2dd7zURNfHf+c2uPTekd6ld1REBQQsKCrFgqKiCIj4CthARBQRHwQLPGAFFeURFRUFRRGxoRQFadJVepcOt837x2x2k2zKJJvdze6d7/3s56ZMZibZzTkzZ86cIcYYJBKJRJI/SYl3BSQSiUQSP6QSkEgkknyMVAISiUSSj5FKQCKRSPIxUglIJBJJPiYtGpmmpKSwzMzMaGQtkUgkScmZM2cYYyzmDfOoKIHMzEycPn06GllLJBJJUkJEZ+NRrjQHSSQSST5GKgGJRCLJx0glIJFIJPkYqQQkEokkHyOVgEQikeRjpBKQSCSSfIxUAhKJRJKP8ZUSWLt/LX7e9XO8qyGRSPzMp58C+/fHuxZJA0VjPYHChQszN5PFaBwBANhYucaBRCIxIDsbyMgA6tUD/vwz3rXxFCI6wxgrHOtyfdMTyGN58a6CRCLxO0qjdfv2+NYjifCNEiBQcHvK8ilxrIkk6ZgxA/jnH+D774Evv4x3bYw5dAiYMiUk5BKJzz8Hfo6xGTcvxo1G5TdkBGPA888D//4b0yp5hS/NQYA0CUk84uhRoHRpbj7YvJkf86Og7dEDWLQIWLkSaNUq3rVxBgXe21g813PngMxMXmasFMGxY0CpUkCdOsCWLeHnlywBOncG+vYF3n/fdTH53hwkkUSFnBz+/8iR+NbDDqUVef58XKvhexRFE0tFriibw4eNzyvfWYL2BHylBCZ3nRzcPpV1Ko41kSQdRPZpRDl7FnjgAeD4ce/yTFQOHbJPs2oVMHmyfToRvBL+jAFPPQVs2mSe5plngPXr7Xs6c+fy/7E2UXmEr5RA/6b9g9szVs2IY00kEgtefRV46SVg4kTv8oylScVLhg61T9O6NTBihDflefV8jh8Hxo4FLrvM+Pz588Do0UCHDvYNiHfe8aZOccJXSqBMoTLB7ePnZCtL4lNOnOD/0zxcjsPLnkosycqKbXleKQGl1W5nflOftys70RR4AF8pATWr962OdxXyB88+G9+WzL//Aj17AgcPus/jnXeACRMir8uZM0CvXsZeILt28VZhly7Avn382LRpkZWXnQ306QNs3Bg69sEHoe3584Fbbw0Jl5Mn+bNSytfz5pvAf/4TWZ30/PQTMHCgNwIuN5f/Z4zn+dNPxun+/JM/l6ws3oNYuDB0TqQex4/z57R/P7B1K9CtG9C9e+i5/fYbMGAA3zZTvko5WVnAt99qjzEG3HUX8Msv2mu+/tq+bn6EMeb5p1ChQswtE76fwPAkGJ6E6zwkDuA/6fiV/9xzvPwRI9znYXUP+/fzc2XL2t/re+/x8336hJ+79dbQ9amp3jy35ct5Hm3aMHbJJeF5pqTw/XPn+P706Xx/0CDj/KLxXSr3mpVlfP766+3LVc6fPcv3z57l+wUKGKdv356f//HH8LyPH7cvb+pUfn7YMMYuvzyUXnlulSuHjpUsaZzHmTOhNMqnWDF+7t9/+X7Rotr7AxjLzTWvlw0ATrMoyGO7j+96Ag91eCjeVZDEg2ibQ0TyTwm8DrHq1ivlEBnXTzFZ+MHM4MX3o5iOlLyUnoGTMr0YfBVZ/9zomau/L7M02dnu6xUnfKcEMlIz4l0F5/z4I/dDP3Mm3jWJHQsWAC1a2L/ICosXA02bhr8k0RZwRvmfOsW/L313Xnm57QSNOs9LL+X/c3P58/j88/D02dn83s0mqumF3fjxwO23m9+Dsp+dDTRpAnz1lfb86dP8/ozMLX/9BVSrxs1beiZMAG67TXtM+X6XLTOuu5o6dbgnkBmKElDqb/Tbyc0Fli/n22PGaM99/TVQsmRon8iZCWbGDG52LFTIOt0ddwDjxoUfP3kyVK4ZiktyAuE7JZCQPPQQn0Tyxx/xrkns6N8f+P330CCpHQMH8uezd6/x+Vj2BFav5t/XqFHaNKI9AfX577/n///9lz+P/v3D0+/bx+994EDjfPQ9gSeeAN5+2zidmj17gHXrgHvu0R5fu5bf38iR4XV57TU+5jF7dvi5xx8H3n03/DgAPPig8XE127Zxjxs9+pa/la+/+vek2OIV9PcJAIMHW9dJ/8w++UQ7oG/0u5s9G5g0yTpfQPYEJCoS1b0vEry613g8s/R0/l//wrrpCSgogsWqJagXOEo5dgrQTGimpmrzUVCUmdF9RPu3qjxbozKjYd5y03iItHyrZyh7At5wUdWLAABZuTF2P3NLorr3idCxY7iJAAi9ADVq8BakmsaN+WQqJ0yaxH3OmzZ1V087DhwIbSueSNnZQNWqvOV9+jRw4438uPrlHjIEaNZMvJycHODvv/lvQm9uMuPnn8XMLQozZ3LTo/K7270bKKyKNqCYU379NdTLIOL3ZSbAjL5jI5S8cnO5V9P8+drz+la2OrzDzJlcQakFpahAbtqUm7KM6mPGSy/xkA5q7r2X99j0VK7MTU127zKRtSdblHsCRNSNiDYT0TYiesTgfEkimk9EfxDRCiK60C5PXyqBn3ZxW+a1718b55o4JBl7Aj/8YG4iALg7nt49c/16/gKKoH5m06Z5b1Iz+k4UIZmdzQXo+PFat1B1C3r6dG5eES0nOztkp37tNe05vYAR/b0YXb9nT6jFD2jHo5Ry9eTlmSsBq+9YXw+Ax/CZMyc8jVFPQGHcOJ7HKVU0ANGBXrPfRaQNMOX6vXvFwz4o4x4xNgcRUSqAaQC6A2gIoB8RNdQlewzAGsZYEwD9Abxol68vlcCgloMAAF9t/8ompU+Q5iD/YlRPM3OQ1TVWwkZxEAR4K9fMiyRSJaCvj1mdUkxea6uegNN6EBk7QigmKivUz13UscCMePbCY28OagNgG2NsB2MsC8BcAD11aRoCWMKrx/4EUJ2Iyltl6ksl0LJSy9gXumcP/0EpcUCc4EclUK0a0KaN9hgRULCg+QsMADt28POffuptfZSWtvoZVa8OPPZYeNoXXuB1EIlEq45JozZ9KHzzTfg1zzzD/6uF0S23hLYXLOC/A9HZooxpW7TKAKaRgJowQWueEaFCBf7/vvvCyzViwwbj4y+8wOPlAHxCllIPfQuYKHwAf926cAF/9mx4GUa9Az2dO4e2P/iA/w5at+blliplf72+rgCfvEXEezTDhzu/3gl9+vD/586FX1+rlvP8QqQR0SrVRz8SXhmA2q1rd+CYmrUAegEAEbUBUA1AFatCfakEzufEIZKi8uK8+abza/2oBP75h4clVlDqZhfxULnG7mX24l7//tv4+JQp/P/Ro/Z5vPCC9Xn1DFwr9HbiWbNCLoF25OWJP4/nngtti15jpLBTUpz7zD/9dGhb8WoCuElMj1VgNat6ibBtW2j7gw/478DKtdQK5d1T3lszU1hikMMYa6X6vKo7b6Sx9D+iiQBKEtEaAPcD+B2AZffEl0pAPSB8zwIDt7BoEIkgV1+bne0uHPCZM9qu8dmzIUF97pz3tkazFpByXC8As7L4fZ09qzV5OEXkunPnwo8p5errZPVccnKMW6sidUlNFf8e8/KM0545o31WjGldII3u0wx92pMnnZse1EJbHXrCTNlZRUndssXcPfjIEfHfa6STv/S/Y6fv3rFjkZUfW3YDqKrarwJA02VjjJ1gjA1gjDUDHxMoC2CnVaa+VwKv/RZjzR6pEmjUiJtcnFK4sNYkUagQ91Y4fpzPcIx0oRH9fdkpgS+/1LYWy5UDihbl9eqpN0M6QOSlV+K2q+tcqBBw1VXadGXLWocy7tYt3NdclJQUoIqqF21lNsjL06ZVeO89oGvX0H3oYxJ17y5eH/0s12bNgEGDxK8HzO3vHTqEHyMCatc2z6t5c3OTYZkyfB1gEbxw11RPivv1V2fX5+YaT5zzJysB1CGiGkSUAaAvgM/UCYioROAcANwN4HvGmOVkHl8qgS61usS+UK96Alu3uq/D//4XfkwxiUTqNaO/L7PBQ7Ww+/HH0Pbx46HW3cKF7l/eSAYCFy/W7ttNVNO7B+qxugf987FKa6XYli61rkMk6GcKe43ZIipeEunAMGAeiE4Uo9XCfAhjLAfAUABfAdgE4APG2AYiGkRESougAYANRPQnuBeRra+2h7FwvaNFxRYoVqAYTpwXnI0ab8wUyJkzXFApA3uR5K0mO5v7vRu1PvfuNR5c03f59+4FSpQQK88IvT340CEetbFxY+vr8vK4Ddio7iKcOgUUKWJsx3bKjh3m5/RKwMo33MrkBDhvnfqBWJlJzGaQi3LyZOQtefXYmc9hjC0EsFB3bIZqezmAOk7y9GVPAEDsFUAkwcPMBGenTkDFiq6rZJr34MF8kpNRS7hy5dCkJzV162r3mzcPhdP1gnLleByb1TYhwHfs4N4g+pANorRowRVQ1ar2aSNBrwT0vRA1ZcqYnwOAm2+OvD6xxug3FA3WrYvs+r//Bl55JbI8Hn00susTHN8qgZjjhYeP/tpotTCUIGV6F0ql/C++CL/GyHb+2Wfhx4zyc4KdOUyxiy9a5DxvJf9YrMMr4u8ukSQBCaEEhi4UWMLOKxTBt2KFuL3SSIGoQwb89ps3dVPYv5//P3GCu8ZZRWW0suuqZ24q9VX3PETDHqhRXzN/fnjPQBlXEH22ubn8u1DjdrDXCfPmRb8MicQHJIQSmLYywhWcRFAL8l9+Adq2DU0qcnKtQvv2oe2WLY3jnihYtbitbPT16/NJMu+/z/eNXAZFJ6+0b8+Ftro8o7DIdryomqXeq1e4V5Mi/EXcGxnjk6vattUej5WpQiLJBwgpASL6i4jWEdEaInI5q8PnqAW5MtAk6pEjYkqycmW08i4RGahVTDBGglU01DMAbN8e/Wn4TnoCeXnAmjVRrY5Ekt9x0hO4jDHWjDEWocN6FPjtN21LOyuLT/1Xs327tUBRC3JFKOvT6/PdvZv3GpRrzWbAAlxAr13L66EE31JcGPVK4PXXQ9si4Rs2buTeQmZrBYuuv3vkiDbaplf8+Wdo24kScBJZU49IQDSJRAKhNSgB/AWgjOialZGsMaygrDMstN6wEsJLYdQovr9kiXkaPd9/z89ffDFj779vnH7ECH7su+/4fno63+/SJZTe7LNsWWh72LDQ9tGjjJ0/ry3PLA/9vag/1arZ18EPn3Hj+P8KFcTS9+oV/zrLj/w4+bgEPl9jmAFYTESrDYIaAQCI6B4l8FFOvBdWUPy/3Ux2Ub5KI3bu5P8V047SqhUxoain0atnjp47p+0JmJVth1UvxE8ovw3RgeEEXKlJIkkkRJXARYyxFuAz0IYQUUd9AsbYqywQ+CgtLfI5aL/cpfVM+XzL56BxhO1Ht9tfLCKUFy7kgaz27+czddXzBIwmBi1cCHz0Ed/Oy9NOvhIpT60YP/kktP3++9oJMz16mOcxeXLiC8Xx4/l/0YZCpJOJJBKJJUJKgDG2N/D/IID54HGto0rbKlqPkNlrZwMAVu+zmYykxqpVfdVVfGHsK68E+vYNCXXGjEPRquPW5OZqw/o6VQJqHnpIG1rXbDFyABgxIvIp8n5BtCdgN/lMIpFEhK0SIKLCRFRU2QbQFcD6aFdMz4cbPwQApKdYrFyk4MTDRTGjOFn/NDc35KvvBVbuo3oSvSegEG+ToUQiASAWO6g8gPnEBWsagPcYYxbN1eiSlhKhqYkx4D//Ce0r4XL37AmdV/Pkk3ztWzUDB2pD+1q13hWuvtq6TqK4jbvuN9zGopdIJJ5i2xNgfCmzpoFPI8aY4Ayq6OBICRgJ13XrjOPWKKtBMQZcf33o+LhxPGSxGidx4L3GaCUuiUQSf9q2DU3cTCB8GUVUgUBguoVzIu4J2E2eYiw82JpEIpHY8cMPofWrEwhfh424qu5VYcfSUtL4qkj33x9uV9aHvzUaG7AL+7tyZXwXr5ZIJIlJgsoNXyuBPBYeTiEtJY2vqPTKK+HhfceO1e4bmYOyssKP6UnQL1MiiQu33x6+6lt+JEHlRsIpgfTU9FAPQB9uQTnu5stI0C9QIok7s2bxYIPPPefsOitniVgxfbp3eSWoDEk4JZBCqiqPH8+9d4IXBNIrPuhG8fzNPHHUx5991llFJRJJ+DrIdngwqdRXSCXgPbl54ROK2r6umkS2YgX33lFQlMD2wKziF16IYu0kEh9TqZLza665BrjwQu/roka9CtiMGebpYoXbMC1GSCXgPUY9AQDmD1tRAlYTkbz80iUSrxFd/8GOr792fs3NN4cv4OMEkXdr8ODQdvnyfNZ+tOnTx/ycXZ1jUb84k5hKwGj5RAB44w3g99+1IQlKl9bOyL32Ws/qJ5F4jt16xdFGv7ayE0SUgL4BV6qU+/JEsaqXnQnLqYkrAfG1EshlBvFlbH5nh596VKsEjh4F3nvP24pJJF6zYAEPELhwITBNt5Ke1eDl2rXu12vWQ+S9EnjzTetrPv4YGDYM6N8f+OYb92U//DBfo+Ott7RxvQDjRZsGDQJGj+blqvn9d+1+v37AE0+E9q+/nu/rPRETmWjEp/ZiPQHGGOvwRoewdQXoCetY3h+2KMhYrVra4+PHxz/GuPwk5qdiRbF0NWtGVo4e9bmPPrK/buJE7fGNG/n/+vXF6zB3LmM5Oe7r/sIL2uMFCzK2f394WrN71t+3k8/KlaE8srO15264wfp5m9UPYGzSJPM6232HDoHP1xOIC3pzUFoukPeUwIXFi0enQhKJGTVqeJtftWqhbZEBR30axYxxwQXiZRJ5O7jZoAEXj7EmGgO0bgbaE4SEUgKFBeZ55QHhC5HH44co8R9GLsNqNm7U7m/YYP7bWbQI+Oqr0P4HH/CPwsyZkQmOn38OmWbcKIHq1fk6GXPnhqe1UgxOBejSpaFt9bN65RUeWNHIFLN8udja0f36OauLgt6k5cX7/+uv4UvWJgkJpQSY6O9T/0O2Wshdkn9o0cI6LlSDBtr9hg3N03brBnTtGtovVQq46abQftWqwB13uKomAK5AnEymMhLevXsDJUtqjxUpol3USJ+HEyVQpQrQqVNoXy1s+/QBypUzXjeiXTugaVP7/N97z3qRJTXqsvX3wFjkPbUqVfwxuS0KJJYSELjmpt/OAY8+qj2onlAmyb+kpIgJHzUXXeS+vOrV3V8LAPXq8f/lytmnVQs+Kw+jNhbrQTntBTRpot1XC2IlkFrBgs7y1NOokVg6q9a+055A+fKh7cqV7dN75dYbJxJKCUiSkCFDxNMuWRJ5eW+9xU05op4o06drW8716omZMgDg7ruBl14Cvv3WaS05Tz/NTU4dOvAV8ETo2BFYb7Lm0/jxwKefml/rRAksWxZualKEbfHioXE5dRh2keemrPXRvj3/P3q01sym9OSef168rnol8L//Wadfu5abfxYuFDNJ/fIL/yQovlYC+hnDiTkfT2LJhAniaS+/PPLyChfmppwrrhBLX64c0LNnaP+CC8R6E4pp5f77gcsuc1fXjIyQyal5c/vyAKBlS21LVs3o0dwcZIeI737HjkDRotpjirAdOND4GpHnpvSeKlbk/4sV05rZMjL4f7Upzg69Eqha1Tp9+fK8x9S9u5hiLFOGryUQA4ioGxFtJqJtRPSIwfniRLSAiNYS0QYiGmCXp7+VgG6eQIoc300+IvFL9won3mQtWkSvHpGgmI5EzSfRQBG2kXjnKIPWZs/50kv5fyffmX5MMEHDOxBRKoBpALoDaAigHxHpB66GANjIGGsKoBOAyUSUYZWvD95Ac/TmIJJKwP/cf792386m7rUScLP85o4dwM6d1mkqVOD/hw3THt+7l3+ijSK41GEX1DPnr7oK+O034M47Iy9D4bffnF1vpgR27QIOHBDLo3VrPmFLP66nMGUKXx2wWjXtoLRdvRJU8OtoA2Ab46s9ZgGYC6CnLg0DUJT4esBFABwFYLmgd2IpgTjVQ+IARVgqZFg2QrxXAi1bOr+mVCn7QVxFiOhNCxUrhkwX0UQpX+12qn/WzZt7I+yUe7Qzm5hdp69DlSpig9sKzZqZ/y7S00NB7kRXAFSmcyn4VyGkEdEq1ece3fnKAHap9ncHjql5BUADAHsBrAPwAGPWg6sJpQSkOSgB0He9U1Ot09udjzaiwklZd1pvB3eKkQBy4nro1udd7/5qhL5uToWlooDbtXN2nVs6dgxtGz0XxbQkOv4Tf3IYY61Un1d1542+EP2NXwlgDYBKAJoBeIWIilkV6mslcGUtrUeENAdFgRMn+HKdXqF/GdVrrm7dGp5e3+Lbs8d5JEsRV8yjR8OPHTvGTUFqDh40vv7FF4H9+/lAZSQcPx5+7MMP7a+LpPV65AiwenX48QYN+IQ4xRdfKUNd1pEjoe19+/jvxegeAD5Yu3s3cN117uvqhFtuMT936BCwZQv/PT34oPaezJ6lfnla/7EbgLp7VgW8xa9mAICPA5EotgHYCaC+Vaa+VgJTrpyCWT1nof0/PGREHYP3OF8RjVZz0aLhZoVIMFIChQvzbSOvFb0SqFTJ2aBf0aKh/K3QT5oCgBIlwq81KzstzdzrxggzQaPvSaSlAQUKiOerfr6ivYJSpYyjYRYowCfEWS3uovYUqlCB199KEYr41ceCMmX4/VWqxL8LkWdVokTUqxUhKwHUIaIagcHevgA+06X5B8AVAEBE5QHUA6Br6WjxtRJIT01Hwz1Z+PlN4NlvgJ9sAhImPV7bMp0INRGuuircHKTuCRhhdE9OxgmMegGRePBE016stJDVtmz1QK8VZmMSXqCEWYn2gjJ+QR2XKRKcuKl6AGMsB8BQAF8B2ATgA8bYBiIaRESDAsnGA+hAROsALAHwMGPssFW+vlYCAJB+mDf/mwg6F8QMfbjaKHGwvyoOklpARToTc9Ik3nW3om9f/v/VV7lvvR3Dh4crATdLCDpRAka9mBUrgKws4Px552VHqgTM5gRkZQEffcS3N24EsrN5/aZOdV8vrxTCbbcB584BtWt7k18ssZs/oaA8vw0bvGv8LFrEv9cYwhhbyBiryxirxRh7JnBsBmNsRmB7L2OsK2OsMWPsQsbYu3Z5+lsJHD+OUsu4fdh34/mRCmFR1AJRLQgiXYyjYEF7Aa204gsUEBfmRj0Bp4LVSXojQZiaysu180yywq3XklnPJz09lGdqKn+eGRnOn40bc5AIapNUIgVctOtp6nFierMjJcV5+T7E30qgXz9c8NbHAHw4KByrSU5qIaH2UY+0xSpSf7UwuPtusXx76tyWb78dGDOGbxcsKNZy09ctI8N88NdrgaWUrV672gn33sv/N27sTX0UjMxBbmPWKM9SP+chEVHmpUTSixE1ySUp/lYCW7YEN6PVE2Bu/MqB6Lk26s0JamGvXgXJTgmMHBl+jDG+ohLgTIkRhQt3M1q31gqqzp2BUaP4sfR0PgHJTnDr67Z4sflkLq8nAqWk8DxHj3Z3fa9e/Hqv488bKQG3vcGSJXk+A0wiCvjXjz6cW2/l91K6tPs8pk1LrN6Px/hbCah+jClRUgN7T7l0j4yRfzuZubbZCXGz+inmGpEXPV4vhh9CSfiNRBLMkoTC32+b6odf5UR0ijiRe8bdhWpBJdpKFkEneE9c1iG0ow4RbCco1RNpjPLXX2/l2x0NATRpElCzJt/u3Fl7zurentItLWcWXkChZk1eVjIxfDhw8cWxK++pp6zXVkgEnnuOmyOTeIUwtySMEkiNUlTp8wVctujVguqTT6LTamYMOZVVIQnUYXmtBCVj5r7/Zj0Bo/C60ewJjBwJbN/Ot5VwvcoiLFYrQ40Zo13ZVa9A9GzfbmwaSzTU5qApU4Affohd2WPGcK+aRObGG4GzZ43nS+RzEkcJRGkUPivd5SOIkTlIs5qauky7noDZebOegFVrP9qmCC+iTyY70ZwnIMnXJIwSKFPE44lNBmU4Ipp26wEDeJgCAES6cvr35wNZduXr70tZvEXpCYgogTFj+OpRokv8ueW667g3zSOB8Oh6QWcn+GbM4P8fftjzqvmOWCiB998HLrkkEWbQSjzAxUyeGKIeGPZbC8itEnjiCW5jffJJ82Uv37SYGj17Nv//8svW5eiF+iuv8P9m5iAjJVC/Pl9lKdqULg388Udo3+ma0BddlPwt5Fj2krp2jflsWEn8SJieQOZOm9mtscatOcisJa6gE2ZhPYHQCetyzM67MQeJ4pUgdqoE8hPJruwkMSdhegJ+g6WYO62+1QwYkNLCeFEOJy6aVuk+/piHHJg5k++PHcvjvyvKKTtbm1Zh0iSep3rJPn058+eHR9c044or+OzXcuXEF/mwo3JlYOhQoEsXPugeS08YvzJsGPDnn8CIEfGuSXyYPZuHtpB4Tr5UAr9cXB3tfvwLAJCTZ7nojilrD61DM5Nzd14HDPitSrgSaNvWvCVuw+7S6aiiPlC/PreFK0pAb1o6e5b/b9cuFAsf4F5DiklJjfpZOwkFXLcuX4zdS4hC5q5rr/U270SlWDHgXdswMMmLeqKkxFMSxhzkJXmqbM9ku5sncI5lWycwii1UqJA2Ho8RpUvj9k9uB40LVDKF/3ccNiMnoNyMQih7iZexWCQSSczJd0pgSQ2Apdjn+486ZPqoUcCmTVimikDLjOr266+4V1kkato0TOoAPK+a64V33+V5jRgR8tYBgLfe4qERBg8GJkzA22vfDp0LlMPMqjxrFrBsWfjxiy/mE6neesv8Jp0yZw6v56OP8gikI0e6j7EjkUj8AWNM6AMgFcDvAD63S1uoUCHmCU2bqqcFRfz5pxgYngT7vF2p4LFvGmYapp3ZIrT979l/2ccbP2Z4Emx6K37sp1H9QulV4ElehmbbIB1PYHxcnceW5V8wBrC/y6R780ytMKtnvPOSSPIBAE4zQXns5cdJT+AB8IUMYsM993junqhYVLbAfomyYMu7XDncNv829PqgF4CQWYbFKr5NrL1BIl1DVyKRJBRCkoyIqgC4CsDr0a2OitdeiziLe64Gzqs8ORXB/vjlwLfV+TYBuOLRyuh9o/5qYPSAasAvv2D7se1h50zNM14TUALn8mzGILxgzhzg99+9yWvhQmDlSm/ykmg4l3MOP/wdw7ARkqRGtDk7FcAoAKYO3ER0DxGtIqJVOTnuPG685rVW2n2lTX02A5jSXjlK2F4uDfN0K+sRA56p9je2Fc9FHgvddjRl/7/n/g07xgJlx0Tp3HyzaYz64+eO45/j/4jn1b070KqVfToLTmedxgcbPogoj2RkyBdD0HFWR2w9sjXeVZEkAbZKgIiuBnCQMbbaKh1j7FXGWCvGWKs0N0sKqonSLNVtqvDraiMLg7nJpdH0RsqYCACVOSiwrxlAdsmWI1vw5bYvUfK5knhv3Xuac0p5eTFQAjNWzcDGQxsNzzX+b2NUm+rR2qyCDF44GH0+7INVe1fFtFy/88dBPrvaqNEgkThFRFpfBOBaIuoBoCCAYkT0LmPs1qjVasGCiC7vcCdwuFD48d43hR8DoBHyerJyszQ9AYU8MFx5K7CuHLBXsF7jl43HmEvHhB2v90q94PbXO77W1i03l5cXgRLYfHgzTpw/gdaVW1umu++L+5CWkobsMeGmp10ndrmvgEuUnsfJ8ydjXnYiYNV4kUhEse0JMMYeZYxVYYxVB9AXwLdRVQBAxMHZll8AbC0TfvyYgWIgMn6ZzGSucpwBWFwb2OegJ/DEd08Et/ef2m+YJkwhKeYg8WLCqD+tPtq83kYordvJc35l2opp+G2fwcztBIb8t+K2JIHx5zwBDz1vfq1sn8aoJ7BVYzoKP68/xhjDsr8M/PUDnNX1uU5nnTaui76sQN1iNhCdZAxdNBQtX3W5hKjPserBSiSiODLeM8a+A/BdVGqixqESqPEAcMFxYNms8HPX3Awcn2h9vSJ4aw4DWu7jre6PG4TOawaGmfYahY83fYwb5xm4GAFofB9wSNcLEe7KB170WIwJSBIDimNMram/TEVOXg5GdMinMYySEH/GDnKoBP4qCZwORGPQW+9PGERvUEOMgi2qnaX4R8+OY6Fgasrr9+Oun4DafPvLbV/inT/eMS1jvW4pBKMxBlNyedpoKwHZqpSI8OBXDwKAVAJJhD/NQRG0dETNJup0TgbYVgdWe1ySFhoo7T6nOz7d/KnQ9R3e6IDUp1Lxy+5fzP1t1fXMyw3UMbrIQcbEQ35nEi/wpxKw6QmM7wgsrqk9pgh10ddCHZDNSSv4v62BRoOB76sLX6Jh+e7lAIClO5ei7Cig4kPa8+EDw7ExB/m5J7B632os3bnU8Nwbv72BL7d9GeMaxRc5MCzxEn+ag2zihh8qBBwooj2miDCnwtLMO8j8AmBjOftk53POW57PQx6OGngrqevCGAuuP8CIm5FSzBaZiRA/typHfs0Ximdjw+t494K7Tc8lO35W3JLEwZ89gcces02ypIbxcSMloI7+qbC1NP//Y4PCUXmZHvnmEcvzIuMCDAzZFbjG+bQekBswDRnx7A/PYsCnA4L7O4/txLwN8wRrqxUo175/LYo+m5wxhKb+MhU0jhJ6olU8B4YlyYc/lYANjIDZzcOPAcZKoOttQHGdTN5SBig7EphzaSlnA7WC7PjXemUuISXAGLIqlkWZkcCES6yveezbxzBrzazgfotXW6D3h72F66vOe8GWBTiVdUr42kRi5mq+CM++k/viXBNJLNl+dDseX/J4wveeiKgbEW0mom1EFNbSJKKRRLQm8FlPRLlEZODuEiIhlYAVRgPDWWnGXkKHCwOUkoIjZ494Xo/PNn9meV7kx6gI5iOFAZYC5DLznoAepaVr1XvQ1MdH5qBzOedcL/ZjRyILAf3cEj99Z37n6vevxoQfJ2g8/RINIkoFMA1AdwANAfQjoobqNIyx5xljzRhjzQA8CmAZY8wybHJCK4GfVestMt1/UeIlFMwEuro+eSwvbN8Iq/GH+764T6g+fhKO1aZWQ+EJhR1dM2LxCDy+5PEo1Sj+fPfXdyjybBEs2bEkpgPDs9fMxk///BSz8qJFVm5WvKvgBW0AbGOM7WCMZQGYC6CnRfp+AN63yzQhlYAiri66C0gJRGKwMgdZselw7JZIUCNiDspjeZrWnlmr/sp3rzTN47XfxEJyuzWJbT2yFQM/GxgWbmL70e2uvXYOnj7o+JrJyydjwo8TXJWXCHyz4xsAwM+7fo5puXd8egcufuvimJbphNy83Li37mMY26oyAHUQr92BY2EQUSEA3QB8ZJdpQiqBIMTNJGoSJbyCmdBVC319GnXv4fi545i3YR4Wb1+MZX+bh6swYtTXo7Bo6yLTcp1wy8e34PXfX8fqvdogs7Vfro3uc7qHpa8+tTpqvljTNFqpCLl5uaaxl0QxG1xljKHg0wUxfeX0iPL3GsU8Vig95FLmp95bvHj828dR66Va+Ovfv+JS/h8H/kCxicXCov+6JE0Jxx/43KM7b/SjNfsRXAPgJztTEJDoSsCARAmvINwTMDEHdZrdCb0/7G3ZCzDj+Z+fR4/3emiOmQmUw2cO49aPQ/EC9THs01K4l7HoeMXfx//Gzn93otH0RlixZwUWbV3kWJg98s0jqDi5Ig6dPuToOhFy8nJwPvc8HvjyAc/zNuPLbV+CxhHW7jcPoX42+ywAIDM9U3oHqfh257cAgAOnDgil93ocRfnOFm5d6EV2OUo4/sDnVd353QCqqvarwDyIcV8ImIKABFUCRq39FF2cf69oXK6xxzlyRJWAGrU5aM3+NZ7Wx+zlGPfdOMxZNye432BaA815RQm4iT762JLH0OO9Hnj3j3cdXffZFj7oHo0BfYVYtrI//ZPPNv9pl7ntXXm+yvMG5MCwH1AUcoy+i5UA6hBRDSLKABf0YR4oRFQcwKUAhMIYJKQSMCIY2M3jRlK0vlwz+37YwLCFechLRPPOZbmgcYQ5f3DFkJrC1+90owSULvzuE7sdXxst4tHKdlqmnDHsH5TvIhaNBsZYDoChAL4CX+/9A8bYBiIaRESDVEmvB7CYMWYcqlhHQiqBz+qFH1NeC6/NQesPrvc2wwCuegIOXESd4vRHPHfDXAAqc5CNK+rGQxvDylCEn5eC104Z+bn1LG387tH/hnLzcrUrAkZJcca60cAYW8gYq8sYq8UYeyZwbAZjbIYqzSzGWF/RPBNOCeR8txS7i4cfj5Y5KFocO3fM8LgmbASYkIuonh//+dHwOI0jzN8037ZcJ4iYg77e/jUaTW+Et9a85aoMPVYv9KPfPBrcPnLGubkoHoJYREAZCZto1fV8znlsO7otKnm7ZdvRbfjz8J/C6dPGp+GOT+8I7kdb+fu5cWFHwikByjMWhLFci9cLzFz9rMxBIhO/dp/YjUveusT0/CsrX7EtVwQlvaIEsvPCl6RUUF5e/QpfSh6ns07j+LnjYdd999d3QnVQowToA4Br515reb1l3nF4qa3K1LRqo9z6LPhMQdR5uY7pwkfxoM7LdcLGo+x4e+3bUapNCCWWVyL34hJPCZg87EOFgUW1gT4m6wgnIuX/Uz64zi4gZg6ye3F/+PuHsGNz18/Fq6v1jggcM4Gz+8RuHDh1wPYleHvt27ZC6+kfnkaJ50pYphGpE6AVpJsOOZ8DEg/h71dvn7M5Zx2ln71mNv7+9+8o1cYY0e8rauYgxHRgOCr4M4qoBZRr3BPISwHefe5mfO+Nv65v6PNhn+C2iDnoi61fWJ43arH3+6ifaXoz4b72wFpUmFwhlM7kJfh257doVamVZZ28JN4tspV7VqJc4XKoVsIgaqENjntj0TZxOKhPVm4W7vj0DlQtVhX/PPiP/QWCnDx/EgdO27t/xnuwPN6/u0hIvJ6AiTkIQNTCLMcSqxdbxBz00OKHbNM4wcrMI0JYaOxI8tJdv+fkHsvyrPKIltBo83obVH+xuqNrnNSFAn+xwImSUZ5rpJP49HR5pwvqvFzH9fVv/e7NOJQZfu3FOSFxpOYlATt3rrkgjHdrwAus7iE7Lxu7ju/CsbPGg8pek8fy8ObvbwqlNRPweSzPtMt8Ptd6zQU9o78drdnv8k4Xw3q8vfZt23kUTWY0MQxS50ZRnco6hd7zemsmLDHG8PiSxx15lzlt2Sdy61OUX/f8GtH1d352p2ZSodfPLBnMQYmhBMqVA4oV49sqJfBR74/wYrcXg/splIJRHUbFunaeYvVjOpN9BhdMvQClJllGhhXGLnTD8z89L9wTMJu4xRgLtpb+OPCH5px+fsBHGz/Cgs0LTMt44/c3AFgrSgaG2z+5Hc1nNjdNA3DzhX6gWl9vUd5Z+w7mbZyHsd+NDR47fv44Jvw4AZfOutT2esfzBCzSbzq0CedyrBdlEsULgckYw9ilY6PmbSRSx0FfDLJN45bgZLEEVsiJoQQYA1L5pCTk5qJacW5vJZDGRJJCKehRp4dRDkmB194ajaY3wubDm03Przu4TjivgQsGGh5XKzWrGbEAcOO8Gy09ehTbsOXAsIOX0SitmxadUX2U8RsnE/wiFSTHzh5Dw+kNcddnd0WUT7A+HrRud53Yhae+fwo95kT3vbT6Tdit8ucFsicQbYg0SqBphaaBw6TxmCFQwo8LWLVyoxFjf98pbxdXOZ9zXhO2N9YtJEd2bAdpj549igunX6hRmi//+jKW71pumF499jDuu3GoOqWqYToljRv09T+dzRsJy/5yFlAwmijPwanpT8ELEy8RRc12nwwm6MTwDtL1BNQvmLqllUIpSTFQY0Y0Zwwb4eZZFnxGu3oPA4vpi2KndNQrphn2BHTHDp4+iLfXvo3SmaWx4dAGPPvjs5h13SwAwLAvhwEA/nvVf8OuVQQ0EeHJZU+K1d1qnoDqnNnzVI57FV7ETdhqM/Oh28ZARmqGawWiQKCoNUakOSiWlC/P/2dmal4wvTko0TXz/zb8z/ScR+FqY4rfegJqRSrSE+g/vz9Gfj0Sq/auMk1j9JtT7lukZ+pVw0U0mFlWbhY6zeqEFXtWAAA++fMT0DjSzEkBgBs+uMFVPT7aaBvCXpj01HTN/tfbv9Z4/Ih8h+rnO27ZOE9/k3JgOFYQAZMmAdOmAVdfrXnB1K2eaHb7/MC8jeILx4ti9UJ4oVCjEfTOql7qwV6jF1O0Psq1wWU6A8pDOW4Xv14pR1/XT/78BCv3rAzu//TPT3hlhfEsbjVKPjl5OcH1I/Tfnejs1Q0HN2DZ38twyVuXYNaaWcFwHr/t+01YQOoXDFI/ayOF6fa9zEjN0Ox3fbcr7vzsTsf5KOXPWTfH0iHALbInEG0YAwoVAgYPBohQplAZAHyBjYJpIfNDoo8HxAOrFowXCjWX5fpKMWtMNg4GhvXCvN4roSiGRi1wxV9ef+/X/+96tHm9TXD/4rcuDikYC0FyLpd7/Dz747OmaURbpUqdsnKzMODTAZpImKItWqMFg6xwKyTTU9LtE9mg/+68NKv66bftlsSQmg01aynj5e4vY3qP6bis+mUY2mZo8HgymINijZceR0aT2XLzcoXXORaBxhE2HNoglNZO8LjppSh5qge/jcIJt329reac2/IUlDUX1G61eoEtap/WN5a8iIkfaWyjQZ8Pwj0L9AtphfcEzFA/Z7NotQpersUhzUGxYr428mXRAkVxX+v7QEQokFYgeHxgi4GoWtzcC0MSzm3zb/Msr5NZ4WutRnMNBDvUL2azGc3Cjhmx54R2FrJ6/MkJymCmOuSBF2GurephJZB2HtuJKcunaNIpqGf5mimQ3Lxc4VhMRg2xXSd2Wa4EN3P1TMP1sO2UgEgPQ1+fez+/1/YaK7JzszH4i8HYd3Jf8PuI5+88UhJDCZQsKZSsaYWmqFKsCq6pe02UK5Q8HD8fHr1TwWkX3ujl98vLsfZA+NKNRsKy9WutDa8XafHZCXG1Eqj9Um1c9OZFlunt0D9vq55A13e74v8W/x8Onzkcdu6X3b+YXqcwZukYNJze0PS8iAIbumiobZqs3CzNZDc3Jl6zHpJXfLH1C/x31X8xZOGQpLA8JIYScMg717+D8ZeNR9daXQEAYy8da3OFxAinXVyrSVPx4MT5E2HH7NZnMFOKli1wQSGjVgLbj20Pc8GMNICckYvo4TOH0Xte76DnTx7LMxWsDOZjAvP/nG943AlqE5rCyfMn8c2Ob4L7Dac1ROYzmRGVE2YO8lhQK8/Xy7hY8SQplUDxgsUxuuNoNCvfDAA0g8cScZz+sPVhIQDg6x1fe1WdiDmXc850MR9RInnZnSzBeS7nHLq80yXsmYoINLVwmvjjRMzbOE87hmGitKzuzW5BF7fP5ZaPb9HEgNp+bDsAoPe83o7yUd9TtHsCwXxBnoynxJukVAISb3D6wzZazMbN2sPR4o5P7tDsqwXXZ5s/A40z9/f3ojXpZExg5Z6V+GbHNxiycIgmjdqz5b4v7tO0ooP5WAhkxqwn70WzRWuU96bDxuMMIu7QjLGg0rArR4Q1+9dg78m9wunP5ZwLuhAnck8gMWYMuyQZ3LfiSSJOTrNCPxFPLXRnr51teI2SJuiDb6AYRRcbz86NLCy3ni1HtqDLO13AxjJN3ZT/vef1DhOmeSzPvCcQiXeQekazRatcTyTKdfba2cYmPxMzmR3NZzbnM5RHi81QXrRtERZtWySU1s/InoAk3+JkIXKrQVfl3JtrrMNu2/YEPGpNnjh/AvM3zTdsTVeZUsU0mJuTeQJWPPPDM8FtuzGhSOb2KDOe7XAyydJo3EKPV4EH/YJUApJ8i92LO2/DvODsVyslIbqQj50SOJV1Cq+tfs2xMjh85jBe/OVFzXW9PuhlmDaP5WHnvzsNzzEw14ro193Gcf+dzlnQY9Zr0SsXAmHPiT1YsmNJXEwz0hzkcxL5C5JED7vfRe8PQ4OTU3+dCoAHVdPH2FHswnbYKYGnvn8KAFC7VG2kpaQJ1RHgYwMfbvzQ1ZKWXtH5nc6GxzUeNEamNJcm2+zcbO0EMTA0m9kMh88cxtnHna2N7AWyJ+BTksGHVxI93Ly4fx//G9WmuhO2IgvMAKGQ0KKczeZC7+jZo47rpMYrc5AaO3OQrRnO5HxOXk6YAlHmQETa6Dt0+pCns4r9jq0SIKKCRLSCiNYS0QYiGheLinnByItGok+jPhjcenC8qyLxIWph8dEm7yJfmiGyYLobimQUAcB97iNhx7Edjq9ZtXdVcLKZEepnfODUAdR+qbZmlTG3YwI5eTkaBaHvFVjVw47mM5ubrkxnpigT2dog8g2cB3A5Y6wpgGYAuhFRu6jWSmHAAKFknap3wiMXPRJ2vFRmKcy9cS6KFyzudc0kScCxc8fwzPfPhJl34s0171+D139/XTh9agpfayPSwGijl462nQ+gp/VrrdH+jfZhx5XFd9QC89c9v2L7se146deXgsfcrhJ32/zbHJmS9K6k+gFg9brde05qQ4eoKft8Wdw076aw44lsDrIdE2D8m1BW4kgPfKJ7xwUKAA88ADz3HPCm/ULnS29fGtXqSJKT2z+5HQDw/vr341yTcN5e+zYAMZu54srrRWvUbm1mUepPq4/dD+42NAep6+m2J7BgywIMbmXcw3fzHCpOriiUzmot7URF6BsgolQiWgPgIICvGWNhrgBEdA8RrSKiVTk5LicIMQbMmgWcP8/XEJBIYoCRr7lfcLRmss9ao93mdLN1p7ScuAZm3VMwuV+R56Cu19PfP227etn+U/tx1XtX2eabiAgpAcZYLmOsGYAqANoQ0YUGaV5ljLVijLVKS3PpdPT55yETUJSUwBc3fxGVfCWJy64Tu+JdBUu2HtkqlM5vrdF9J/fZ2s/tegJGAe8U/rvqv8FttbK4f9H9tnVT91DGLB1jm77DGx2wcOtC0/N+U8BOcNQXY4z9C+A7AN2iURkcVwXvipISqFy0sm2au5vfHZWyJRI3NJ3RVCjdqG9GRbkmztCv/KegFphWSoAxZqkEzJi1ZpZtGqeBDc3mVijESgETUTci2kxE24gofCCUp+lERGsCjjzL7PIU8Q4qS0QlAtuZADoDcDZ6JIpa8EdJCZQtXNY2Teeaxj7PEkk8OJsTe793EU5lnbI8T0SGQQVFF6Cp83Id95Wzofuc7qBxhI2HNnqSXyx6AkSUCmAagO4AGgLoR0QNdWlKAJgO4FrGWCMA4aPYOkR6AhUBLCWiPwCsBB8T+NxZ9QVJUVUnSkqgUtFKwmlvamj8/ObeMBfdakenMySRqPGzmaHR9EaW5w+ePmi4aNGM1TOE8rdrfUfCD//8AAD4due3nuQXo55AGwDbGGM7GGNZAOYC6KlLczOAjxlj/wTqddAuU1slwBj7gzHWnDHWhDF2IWPsKReVFyMGSsCOzjU7a1onHat1DG4PaT0EvRv1xjX1rvGd/VWSnPhlUR4jInGtXbTV28Br8Z4Y6pGyTlOcawIf/XqblQGoB7B2B46pqQugJBF9R0Sriai/baGR1dljomgO2jFsh+HyhwqplMoXRddNOll2xzKMXDwS/1n+H1QrXg2v9HgFgL9fTknyYDURK5HZcmQL2lVpF/fGlFfleyQPchhjrSzOGwlF/Q2kAWgJ4AoAmQCWE9EvjLEtZpnmGyVQo2QNw+MFUgtgTq85yMnLQd+P+vKidc/a0MPBx910icQPEMj0PRn+1XAM/2q4Z2W5fR/NQog7JTcvsol6guwGoF5EvQoA/QIIuwEcZoydBnCaiL4H0BSAqRLwV+ygOJmDbmh4A0pmhq9jrLQSgguLqOOkq1oQrSpZKW+JJH+SCA2l1ftWm55jjGHPCfPZw5q0YNh+dDsOnrY1wUfCSgB1iKgGEWUA6AvgM12aTwFcQkRpRFQIQFsAxiv3BPCXEoiBd5CeIa2HhB3TeywoP2Z1D6FT9U6uy7yu/nWur5VIJOFEw6z0/M/Po8qUKthyZIvtfIY8lofaL9dGpcnijidOYYzlABgK4Ctwwf4BY2wDEQ0iokGBNJsAfAngDwArALzOGFtvlW++MQcZkT0mG6mUGnbc7AelVg6PXfKY0CQTI+JtB5VIJPYoS3f+9e9ftjZ/xRwUafwmOxhjCwEs1B2bodt/HsDzonnmSyWQkZqBVpVaBWO2A/brsgLankAkKyLJQWWJxFuiYXpyItAT+Z32lxKI0ZiA3RqipuYgj+qUCLZSiSSRmPPHHM/zVOYQKD0CKxJZCfhrTECtBFLiX7Xgwt2C5puvb/taKF0i/2AkEj/ywi8vRC1vkbUmEvmdjr+kVaNuacdYCahb+XVK8enqHS/gE8WGthmKSkUroXej3sbXBsxExQoUEyorkX8wEkl+Q2SxnUR+p/2rBGI8Y1jd2m9aoSn+euAvDGs7DABQp3Qd7Pm/PY5CTqi5osYVmv1o/WB+v/f3qOQrkUisifaAcDTxlxLwgTlI6RFUK1EtojGA448cNz03pmPIq6hN5Tauy1DTqGwjQ08nJ5TOLO1JXaxoUbFF1MuQSGKN7Al4RRzNQemp6QCAwumFPclPbRrSDwRffMHFQdPS/7X7PxwdFdkC4QDww4AfNN5ObnjqsuiFhVK4rUl4QDGJJNGRSsArXnwxtB1jJdCpeieMv2w8Xr3mVcfXinr7lCxYEqM6hMd8j8TdNJh3ZsngWrNuicX8Baf32qxCs+hURCLxkERWAv5yEV2kiiwYYyWQQikY3XF0VPJWhOu8m+bhipra8QEGZikY9z+0HxUmVxAqJ1JzUCxcV50qgRIFS0SnIhKJh4gMHvsVf/UE1PjARVQUszC2io3daJ6B+hqrsYfyRcoL1yNSc1AscKoE3PaSrq13ravrJJL8hn8lbQIpATPWD16PFXevCO6bKQsvzEEAHJmDjAak/WgO6lStk6tyLqt+mavrJJL8hn8lbQIogW33bwubILbn//Zg6/18YfAKRSqgdeXWeKnbS7i8xuVoV6VdWB6MWZuDrOhRp4dm30lPYH6f+WHzGmKyRJ7DxT+GtxvuqpwH2j7g6jqJJL/hX0mbAEqgVqlaYesRVypaCbVL1dYca1y+MZb0X4LM9MzgsQ5VOwTzEBWMH/f+GOvv4wEBr6l7DVpWbKk572RMgEAY1maY5hhjDBWKiI0/2HFgxAHD404VntvBbq9CfEgkyY5/JW2Sv8T3t7kfW4ZuQZvKbUwF4xc3f6HZv77B9WhUrhEOjzyMD3t/GJbe6ZjA+MvHa/YZGF7u/rKjPMwwq0usxgQkEokY/n3DEqAnoMeJTZ2IUKc0D0+hFnTHHj4W3NabexRKFyqNjNSMsB6Ek1azUUvZyzEBr8Y/IvV4kkgk1vhX0iaQEojU9KAWjE5cIhUb/sMXPQzAuPU98+qZjvIzUwRVi1XF/D7zhfMyeyayJyCR+Av/+hQmkBKIFDslckWNKyzDLWSm8bEGo1azmRB1OkC77r51KF6wuHB6s/yduLwCzpXAXc3vQs96PR1dI5HkZ/wrafORErDjm/7fYFKXSWHHqxbja05XLlYZgHFPwImJp27puqbnnCgAwFyx1SlVB6sGroo4HzNql6qNa+pd4+iaaDG87fB4V0EiscW/kjaBlIDiZaP3Coo2d7e4G5/2/RR3Nb8LgHGruXud7gAQNuCrF65rB63F1XWv9sxN1Kqn0bJSS9NzfuKj3vZx5GNF11pd410FSZLiX0mbQErglia3gI1lKF0o+lE41RARrq13bVCgG7WaqxSrAjaWYUjrIZZ5NSnfxPO6WR1/9/p3PS0vGkQ6UO7lvAs5+U0SLfwraRNICSQCeqHsdExAoVzhcmLl2eR/S5NbXJVvh1pwRxo6QjRGvJk7rJdBxdx+XxKJHf6VtEk+T8AIr4OllS/sbBBWhO3DtguFvjbtCURZmKlb3yULlowor9y8yBYK8dTlNh++D5LY4F8lEIM4Nn5iQb8FWHPvGs/yO/f4Ofzz4D+aYwNbDAxuuzVVFMkogpKZ9sLVC2F/aOShiK6P1BwTaUs+WmE43IwPlClUJgo1MefNa9+MaXkS9/hXCeQlbnxuN1xd92pUK1HNs/wKpBVARmqG5tiL3V40SR3Cq9ar3ZiAHY3KNgoKru/v+D54XG8bL5VZSrOvrn+sbPpmE9qMyncb6VXdK9HfswiRKlSnyPkdiYN/v6l8pgRiQWZ6JsoWKgsg+hFDzXoCoj2Ei6peFNy+pNolQfu+WolkpmVaCrdIW+IXFL9AKJ3ZTO08loeCaQVNr7M6p0c9PuF2FnW14t41MuyIlRJIhPDpfkcqgXyGV7Zlu/ELdTnPdX7ONn3jco3xfJfnDa8HeKwlQKtEUlNSw4SNWvBHoug6VuuIOqXqCKXVC6LpPaYH67Jq4Cr0b9o/eE5d/1k9ZwnXR90TcBtUz4nSiZQutbrEpJwm5Zvguc7PBfdf6vZSTMpNJvyrBPLZmECiUjCtYDBshRq1sFNPQjNSQg9f9DCW37UcIzqMCB4LE+7MemEeI9QKoVHZRpZp9Yh6QQHhLXOljowxNCrXCE90fCLsHMAjzoqi7gkkgqlF6XFGG72ilz0D5/j31yR7AnGhfdX2AOwHH+f0moN2Vdrh1KOnMLHzxLD0RITNQzdjcKvBuK7+dYYC++XuL+PNa9/ExM4TUTijsPZ6XXplkFZ0RTb1NQBQKL2QZVo9jDHhXpNe8Ch1tFtRzgk5eTkR5+H1XBArRBVVl5qR9xj0vUOJM6QSyKeY2csvKH4B2FiGK2tdaXl9jzo9sPyu5cGX7qtbv8KJR04EzxMIdUvXxbSrpmmuU7+wQ9sMxYDmAzTnlYV39AK4VqlawXI/7/d5WF7B+zIZGHY6PqD3DLqlsfm8Bn1IDXVPQI9bc5zGHESpphFmrXir51uuyjbiq1u/wu1Nbzc9L3qfXntgJXtPgIi6EdFmItpGRI8YnO9ERMeJaE3g84RRPmqkEkgyapeqjTub3Wl6XrQV6caeXrRA0VA5+slpgkKh34X9AIS3JGuXqo2DIw7igbYPBHsrdnlGMj7AwISf1YyrZmj2w3oCiLwnoDYHERE+7/c5Hmr/kKM89L2tSFAP3PuJZFYCRJQKYBqA7gAaAuhHRA0Nkv7AGGsW+Dxll69UAknG1vu34o2eb9imsxOKkXrWmHoHCZpwjK4vW7gsiCg4wGnUGjYT/JH2BKzQhwtRei2NyzW2vM5Jr0BfHyJytYTmpdUuFU5bo0QN03NeORhE+jvTm+2SfP2JNgC2McZ2MMayAMwFEHHIXKkEfMgdze5A70a9o5K36MsbqQup2zAVSmRUq2B8hdILYecDO20nJKlNOI57AirhkpGaYendpM/78hqXY9XAVRjWlgcW9EJgGs1ediNAb2hwg3DaHQ/sMD1nNWbiZMDba1flBB8TSCOiVarPPbrzlQHsUu3vDhzT056I1hLRIiKy9YiQSsCHvNXzLfzvxv9FtQw7AdK6cuuolGunDHo16IXFty7GkDbWAe+ql6iOAmkFwo6rhUrP+j2x+p7VALQt6cpF+XtTv0x90/zV6YsVKIaJnSda1kdPy0otQ4H9TAazzZ6FkZujV3GIsnKzhNK90PUFV/kPbzscm4duFk4fcU9Ad32Cx1jKYYy1Un1e1Z03ujn9A/wNQDXGWFMALwP4xK5QWyVARFWJaCkRbSKiDUTkvA/qhnysBKKJ6EvSqXonHBxx0LtyBVvDRIQutbp45gZpdL/KgKZVGXksT3NtkYwiuLv53cZlEKFPoz5C9Xns4scMj6tNR0PbDA07bxTMzk0rWlQJtKncxnHeADeNFckoojlm5S4a8cAw047dJPOYAHjLv6pqvwqAveoEjLETjLFTge2FANKJyDJmiMiblgPgIcZYAwDtAAwxGYzwFqkEooqIAClbuCz2P7QfR0Yd8azcWAdCUwS9usWoCB4rJWDUQjU69vBFD6NJ+SaW96U+9+glj+LBdg9qzneo2kHbQzDISy0s9QPPgLblrpjUjBBVAl7GPbIakGaMoUTBEsIT8+xoX7W9Y3dgL6lVslY0s18JoA4R1SCiDAB9AXymTkBEFSjwAyKiNuAy3vIFtlUCjLF9jLHfAtsnAWyCsR3KW6QSiAoXX3AxAHG/+fJFyjuKVfP7vb9rZnAqxKqbHmYeMHDXVASqehBRb8KoWaJm2LVG9zCx80TbXov6WRMIL1z5AthYZ55LhmMCgeuqFa+GB9uHFIvaS0tPzZI1bcsSrZMoVq1zBoajo446MiHpr7+1ya0A+NhN2UJlMeHyCa7y8oJoTuRjjOUAGArgK3A5/AFjbAMRDSKiQYFkNwJYT0RrAbwEoC+z+TId1ZiIqgNoDuBXg3P3KAMaOTk5Ydc6pmRkYYAlxsy+bjbW3LtGKBKoG5pVaIZRF40yPR9rm61Rq/nS6txDRv3C6t+TyVdODrs2Mz3Tthwj1BE8zVr8ds/FyGxStjA3swxvN1xz3OqdVwSmwl3N78LiWxdblm1ECqUgPSU97LjRfVh57CgDzJH0EMsXKQ82luH86PNITw2vUyxxa0YThTG2kDFWlzFWizH2TODYDMbYjMD2K4yxRoyxpoyxdoyxn+3yFFYCRFQEwEcAhjPGTujPM8ZeVQY00tIitMsNHw7072+bTOKczPRMNK3QNLj/1wN/4cCIA1EvN5pmoJ/v/BktK/IlK/UCUClXLUSVkBBmrbZLLrgEBdMKBlvwtzbmgjNa9mYR00suyw2LAlskowjYWBauBCzy038PBVILaOL8KD1FuzplpmeGlWvGU5eZu6pblTOtxzT8OOBHy7zd9FgiXbdD+V0YKbyp3aZGlHc8EFICRJQOrgDmMMY+jm6VAEyZIlcWixHVSlRzFCcnUqKhDNpXbY9JXSYB4BFHNeXB3BykKIHr61+vEUbli/DFeDLTM3HikRN44coXwvIAgMGtBge33djQrVr/YzqO0eyXKVQmGBrc7hlWL1HdcV3s6lQ0I9zE1LBs+NCgUd2s3J2thHjjco3RoWoH0/NuYGOZsEnMiCOjjmDHMO46a3SvBVLDPdb8joh3EAF4A8Amxpg7vzFJvifaZqDLa1yOE4+cQOeanbXlUrg5SLGvp1AKssdk48PeH5oKo6IFiob5nhsJkUi9XKwE+9Qrp+Lpy58WzqtikYr44MYPhFaWExlDASDs/WTGkv5LDI9bNUAiNRNFg1KZpYLeT0a/ab/VVwSR5vZFAG4DcLkqHoXzwCUSSZQxGhA1elGDA8MpqUhLSRMezDMTmIAHk+sslORdLe5yFAaageGmRjfhpoY3AdCuKAdwE6DiJhtmPtONgyh2f30Pywyz+7i8xuWGrqKRxjOK1uptIiRCNFcRRLyDfmSMEWOsiSoexcJYVE6SfER7MRs9Vt5BaoFVuVjI4U2016JOF4kwMpp9e1Wdq1znF8w3UCd9+IpqJaqhbeW2zvLy4HtbdscyAHxCIMAHjK0cFJTnu+iWRdg0ZBMOjzyMbfdvi7geehbdssjVdUat/kScrJYcqkzie+LVTTbyDjKaJ1CsQDHMvWEuv8akrl4rMKtn0rZKSEgbjWuIYLQGQ1iawHPpUrOLJlSHvqwaJc3jCKmxKqtB2QZgYxnm9JoDQDzEQ7fa3VC/TH2ULlQ6LGKr0TMxGqtQozfdKeYdIx//zLTMMMcJ5R5TKAXdanfDfa3uCzuXSEglIIkpse6+G/UEqhSrAsD5gu03NORxdy6vfrkmbwAoncmDyP1fu/9zXEe7Z+I0DpNyr1aT4vR5Lr5tMbbev9VUiLWs2BLr71tvWa4oBVIL4J4W92Dp7Us1xx9q/xCeveLZ4L5aESqItLS71OqCP4f8GdzXjztk52aHXbN56GasHLgy7LilpxUIi25ZhOlXTXdUP78hlYAkJsTr5VDKVbf+apWqhX+G/4PHLjEO4WBGx2odwcYyXFjuQtM0dUqLz3x1GmJaGfS+pYn52gZqhGZG24wJqGlQtoFtmSL3QUSYec3MMM+f/3T9D+5teS8AoHiB4oYuuVbLiaqpV6ZeUNn/dOdPmnPqBXoU6paua2ia0oelUI4p95EMSCUgiSlxGxPQCYuqxasKCxTTvGOk2JRy6pSuAzaWBX35zVDuw2g9A32eot5BfhF4Tuqx4u4VWHr70rCItIoSMPJM+qyvJgqD4W/CKty5X56TE6QSkMSEuI8JOFA+dsL9zuZ3onej3hjdcXTwmBvl1rJSS9zd/G68e/27wrGHRFC8cKxarGZ5NivfjOdRONyTR7SVH03MWuVGVCxaEZ2qdwo7np3HzUFKT0Od5zX1rtGk1XtWASHFYGhmS0BzUFKH3JP4j3iNCXhJ0QJFw0J91y1dF4B58Lb1963H8t3LNcfSUtLw2rWveV4/ZU6BegBTj5lynNh5Im5qdFPYesR+EW5euGUqPQHF/dXsN3nmsTMokFYAR85o468pE8L6Nw2PapCIPQGpBCQxId5jAiLKJxJT1YPtH0SLii1wWY3LDM83KtcIjcrZru9hiNNnp8wpmHDFBBAoLF4QYG4mS09ND67zbHaNFcpaDQDw/g3vW65O5gYvhGyv+r3wyspXuKfRcfN0ZrGiCqQVwIlHThgGYfSLsnSCNAdJYkq8xgREZvR2r9MdTcs3xdhLxzouJ4VSTBWACFbCw21QtDKFymDmNTMdTTQzQy181b2EwumhMNEL+i3AzY1vDu73vbCvoYdPJLgdx1lz7xq8dg3vdU3pNgWHRh6ynPmrxkjxGM0kT1RkT0ASExJhTKBEwRJYM2hNlGtkjPJ8lLkKAPdb335su7AJJDMtE2dzzoqVJ/hc9EL2hwE/oEGZkJfQtmHbUHFyRQDA1XWvFio7Ety2tJtWaBoMnJiWkqaJ7CqK4gZshTQHSSQ2xG2eQBzDC4igCLdqJaoFj60YuAJHzx4VzuPgyIOG6w4YlufwuSj103smVShSQbh+0SCSnuXUK6di8MLBaF6xuWf1SURzkFQCkpgQ9zGBGJuhvKBUZilHC/rol3VMZMxa1F62tFtXbm04QWzz0M2uVyeTPQGJxAa/zBPwK7F6PopnjNHCMEb4RbhFOrdDBMXTyw2J2BOQA8OSmDDhCr7knxKrP1aoewJbhm7BuvvWxbR8UVpXag0ArmzVbuh7YV+MaD8CEztPjEl5XpGRmoH3er0X3J/WY1rMyk6UhoRTZE9AEhNubXKroatitFH3BJyEdIg1k6+cjDua3RGzOqanpuP5rs/HpCyv6de4H85kn8HFF1yMemXqxbs6AHjU1y+2fuGbHpMTpBKQJDWJMiaQkZqBlpVaxrsaCcNdLe6KdxU0fNj7Q+w/tT/e1XCFNAdJkppEGxPwK4lo644lBdMKRrSsZzyRSkCS1CRKT0AiiRfSHCRJapRZnYUzCtuklLjluvrXGS7I4gal5ybqtRQLkr0XJJWAJKkpU6gMJnWeFFzSUOKMB9o+gMnLJ1uGSJjfZ75n5RUvUBxPdHwCfS6MbGF7iTgUjW5y4cKF2enTp13UJqBxZdddIvEFjDEwsKRZVN0NR84cQZnny6BUZikcGXXE/gKXENEZxljMu6yyJyCRSEwhoqQ3h9ihmKjcziL2O/5SAl9+CRy3iO0qkUgkMaZUZik8e8WzuKHBDfGuSlTwlzlIIpFI8inxMgflX0OfRCKRJBhE1I2INhPRNiJ6xCJdayLKJaIb7fKUSkAikUgSACJKBTANQHcADQH0I6KGJumeA/CVSL5SCUgkEkli0AbANsbYDsZYFoC5AHoapLsfwEcADopkKpWARCKR+IM0Ilql+tyjO18ZwC7V/u7AsSBEVBnA9QBmCBfqtrYSiUQi8ZQcxlgri/NGvrp6z56pAB5mjOWKRjSVSkAikUgSg90Aqqr2qwDYq0vTCsDcgAIoA6AHEeUwxj4xy1QqAYlEIkkMVgKoQ0Q1AOwB0BfAzeoEjLEayjYRzQLwuZUCAKQSkEgkkoSAMZZDREPBvX5SAbzJGNtARIMC54XHAdREZbIYEeUBOOvy8jQAOR5WJ9HI7/cPyGcAyGeQH+8/kzEWc2edqCiBSCCiVTaDI0lNfr9/QD4DQD6D/H7/sUS6iEokEkk+RioBiUQiycf4UQm8Gu8KxJn8fv+AfAaAfAb5/f5jhu/GBCQSiUQSO/zYE5BIJBJJjJBKQCKRSPIxvlEConGykwEi+ouI1hHRGiJaFThWioi+JqKtgf8lVekfDTyXzUR0Zfxq7h4iepOIDhLRetUxx/dMRC0Dz24bEb1EogFS4ozJ/T9JRHsCv4M1RNRDdS7Z7r8qES0lok1EtIGIHggczze/Ad/CGIv7B3z223YANQFkAFgLoGG86xXF+/0LQBndsUkAHglsPwLgucB2w8DzKACgRuA5pcb7Hlzcc0cALQCsj+SeAawA0B48mNYiAN3jfW8R3P+TAEYYpE3G+68IoEVguyiALYH7zDe/Ab9+/NITEI2Tncz0BDA7sD0bwHWq43MZY+cZYzsBbAN/XgkFY+x7AEd1hx3dMxFVBFCMMbaccWnwtuoaX2Ny/2Yk4/3vY4z9Ftg+CWATeBjkfPMb8Ct+UQK2cbKTDAZgMRGtVsUML88Y2wfwFwZAucDxZH42Tu+5cmBbfzyRGUpEfwTMRYopJKnvn4iqA2gO4FfI30Dc8YsSEImTnUxcxBhrAb5M3BAi6miRNr89G8D8npPtWfwXQC0AzQDsAzA5cDxp75+IioCvejWcMXbCKqnBsaR4Bn7DL0pAJE520sAY2xv4fxDAfHDzzoFAVxeB/8rScMn8bJze8+7Atv54QsIYO8AYy2WM5QF4DSEzX1LePxGlgyuAOYyxjwOH8/VvwA/4RQkE42QTUQZ4nOzP4lynqEBEhYmoqLINoCuA9eD3e3sg2e0APg1sfwagLxEVCMQRrwM+MJYMOLrngLngJBG1C3iE9Fddk3Aowi/A9eC/AyAJ7z9Q3zcAbGKMvaA6la9/A74g3iPTygdAD3CPge0AHo93faJ4nzXBvR7WAtig3CuA0gCWANga+F9Kdc3jgeeyGQnqCQHgfXCTRzZ4a+4uN/cMvnLS+sC5VxCY9e73j8n9vwNgHYA/wIVexSS+/4vBzTZ/AFgT+PTIT78Bv35k2AiJRCLJx/jFHCSRSCSSOCCVgEQikeRjpBKQSCSSfIxUAhKJRJKPkUpAIpFI8jFSCUgkEkk+RioBiUQiycf8PxeZlA97jOsJAAAAAElFTkSuQmCC\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAD4CAYAAAAHHSreAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAAsTAAALEwEAmpwYAABPbUlEQVR4nO2dd5gUxdPHv3UBOA7JOQfJICgZRJIkEURFRFSCoIKKGQVUggnDKxhAkR8iCKiABBGRHEWiBEmSM0eQDEe6u37/6J3d2cmzOxtmrz/Pc8/O9vT09NzudnVXVVcRYwwCgUAgEMiJi3QHBAKBQBB9COEgEAgEAhVCOAgEAoFAhRAOAoFAIFAhhINAIBAIVCSE82ZxcXEsKSkpnLcUCAQC15OamsoYY2GdzIdVOCQlJeHq1avhvKVAIBC4HiK6Fu57CrWSQCAQCFQI4SAQCAQCFUI4CAQCgUCFEA4CgUAgUCGEg0AgEAhUCOEgEAgEAhVCOAgEAoFAhSuEw+mrpzF9x/RId0OQWTh+HJg7N9K9EAgiiiuEw8PTHkbnXzqj64yuuHTjUqS7I4h1GjQA2rePdC8EgohC4Uz2k5yczALZIU3DyO89GyISFAlCCHm+byIRliBKIKJUxlhyOO/pipXDO/e8E+kuCDIjQjgIMjGuEA7vNnvX771IbSoIC+J7JsjEuEI4AMDsR2d7j99d8a5+RYHAKYRwEGRiXGFzkJDbHoTdQRAyJJvDzZtAYmJk+yIQQNgcTMmWkC3SXRBkJsTKQZCJcZVwmNZpWqS7IMhMZGREugeCWOD6dWC6+/ZpuUo4tK/o8z0/eP5gBHsiyBSIlYPACd54A+jcGVi5MtI9sYWrhIOcwxcPR7oLglhHrByco2dP4P/+z/51aWlA3brAokX6ddatA+64A0hNNW9vyhSgbVv7/QiGo0f56/nz4b1vkLhWODSb2CzSXRDEOpl95XD1KnDypLr8xAmuKrHDhAlA//7AtWtASopx3YwM4NAhfpySAmzYADz1lH79V18Ftm0DNm/WPn/jhm+AfuIJYP584NgxXm7EgQPG583qp6fz55AcHFw22XCtcBAIQo7LfsyO06ABUKSIurxYMeChhwJrs3VroGhR4zoffACUKQPs3esrMxLUZkK8Rw+gZEnufSZRogTQrZv+NYsWAeXKAT/+aNy2xB9/8PrTZHbRwYP5cxw5wt+77PskhINAoEcgK4dRo4Dt253vixzGgPff57PfULJtm+941ixg4ULf+z/+UNefNw+YPdu4zVWrzO87eDB/PXAAeOstfnz8uP/grsVbb2mrlqQ+paVpl0vMnMlXFQDwzz/89e+/zfurrH/kCFeFffghLztzhr8K4RBaNj2zKdJdEMQy1675jgP5MffrB9So4Vx/tNixA3jnHW7kDBcPPcRn/Ua0awc8+KBz95w4EZg0yff+hx+060lCfMUK4OOP1eelz9FM2D/8sM8eIV0TF8AQ2b49V4VJCLVSeCiXt1ykuyCIZSpV8h0HanNwehBo04arQSSkGbA0Sx40yDcAKTl1ip/7+Wft8x9/zM/fumWtL/L/iZVVgB7//stfP/lE//4//eT/Xlnn22/5tWvX+spSU/mKg8i3CkhP5685cvhfr/f5EnEPI+n46FH+OmeOuu6AAer//blz/u8PHza+X5TiOuGQGOfbsSpiLAkcR9IPA8EN8leu8NcNG3zG0EBZsEBbhSR9/4cP1792507++u232uffe4+/WjUwy39zwfjuf/65//0PHeKD/IkT+tcoZ/Gffaauc/KkT1hIKw29z/HWLe69tGOH/j1PnQJGj+bH48apzytXKidO6Kv75BMPF+A64ZAlPov3eP3x9RHsiSDmCWby0aMHf61bFyhd2one+NBbJWhh9gzSeattOrUqkoSV1F6DBvzPaABV9lGrL5Mn+4SIFXXSE08A1arpn//hB58AkFYgRkyerH+ufHnz66MI1wmH+Lh473H97+pHsCcCx/nyS99yPhqQDz7Vq/PBycgoKh+EZswALl70b2fECJ+BFeBqDyI+i27cGLh8Wd3mhQu+48aNgV279O//wgvqsvff56/LlwObNOx1kjunVcOrPNaU3qArDaJDhwKffmrcntTG2bP8Vet/IHHtGrB4Mf8/EAH792vX+/pr/jp1KvDSS8b3l7j7bvM6mzcDTZrw1Y5kNJewMpGIjzevE00wxsL2lz17duYE3Wd1ZxgKhqFwpD2BA6SnM5aSElwb/Cdm/7rjxxnLyAju3so+AIwdPcrYqVP+5UuXMpaaqr7u6lXGzp71v37iRP9nUj6fvC7A2PTp6nbff9+/TocOjG3ezI/vuEPdjpyLF/3P1avH2H//MXb9uroPFSvq/y+0+gow9sILjJ04wf9P8jrXrzN24YLv/blz2tczxljWrNrntP5mzGAsKcl6/VD/yZ954EDz+rdu6X7tzABwlYVpnJb+XLdyAID7K9wf6S4IlLz/PveJl+vsw8HWrdzvfswY59t++mmgUCH/1ULz5kDt2uq6NWsC+fL5lzGT2aRSTaKcWZ4+Dbz9tv41ZqqgXLnU7efPzw3cTpCayvcslCjBPYskMjKA3Ll97/Pm1b5+2TJrqhqJW7eiK0qu5L4KGNt9JEK4ciCiNkS0m4j2EdEAjfN5iGgWEf1DROuJyECXxnGlcMiRJYd5pcxOejrfTGS0THeSefP46/HjobsHY8BHH/l7g+zezV+XLQuszf37gW++0T4nebssX+5fLhl55cg3bEmYCQflebnBddEibddNIm3DqBWk9pXPY4ae4JX+94Dv8wes7/No3ly998CIdeuAS1GUQ/4dmxkq7diKbDVL8QBGA2gLoAqAx4ioiqLaIABbGGN3AOgG4Auzdl0pHO4uaUE/mNmZMYPPOt98M7z3DdEPAACwdCkwcCDQp4/6fmYDsR6NGwPPPWfsrWPm36+HvE/K/mkNinLh0KoVDzehVUfynrFLoDPXvn21y1ev9h3LP/e6dQO7jxkjR4am3UDRcm2NDHUB7GOMHWCM3QTwM4AHFHWqAFgCAIyxfwGUJqJCRo26UjjIVw4s0EHBrfToAZQqZV5PMjTqzbS6duU/6Oef56/Sbs5ACcfnIKl35M+k9Eyxi7QKsdv/nj3N6xgJBy11SocOfO+AkYA1E7579/I6WvXkwoHIf5dzsJ/f1KnBXS8wI4GINsr+npGdKwZA7i99zFMmZyuAhwCAiOoCKAWguNENXSkc5Hy+9vNIdyG8TJxoTa9vNqOWNhhJnh1yL5pAkO6zYQMfcA8dMvasAYA1a3wePXIOHvQd79jh2yegNeAZCYcjR3wqoOXLtVcHUpsLFtgTMBMm8Gf88kt+rRby/71c3bZ4sb7XU4cOxvc1C1onD3GhRLlPQK4u2rMHePJJvneBMf/QGYJoII0xVlv2N1Z2TmvGoPzhfwQgDxFtAdAPwGYAhjo9U+FAROOJ6DQRbZeV5SWiRUS01/Oax6ydUDFl25RI3Tq6kQaCcK2spPu8+CLXJZcpA1RRqj1l3LgBNGyoPRiWLes7rlaNB03Tw0gIlioFVK3KhVSzZrxvetc/+KBvY5ZVypThrpJ6Bl55n+T/i5YtuZunFnLXVS3++st3bFeFZ6ZWmjyZh+SYOZOHwBa4hWMAZFvoURyA325CxtglxlhPxlhNcJtDAQAHYYCVlcMEAMpv/wAASxhj5cH1WCrreKipX5zvcaBQ6rjdSloa8Pjj/Pinn7Rn53bYuJGroazOrLduNa8jhUKQ/OulnbJGLFnCXxcs4MJl7lxfLJ9ff/XVe+89f+8ZKY6+1mxY/v05aPhbsY98li/tmJaw8j9yGuXKQS/8xb59oe+LwEk2AChPRGWIKAuALgD8DCJElNtzDgB6A1jJGDO07psKB8bYSgCKYCF4AID065sIoKNp9x1GCqOx8cRGrDi0Ity3jzznz/s2Dsm5dk098Fg1YEpqlwsXuGpIMpp26MCFTEoKH5SVAx1gHBv/2jX/gHaAT9Bcvcpn2MpNRVrIk8WsXcsDnGkxeLBvhzLgv4q6dcvnwZWW5h/F87//zPtgByOBZxSyIRi0PKkklCsHveQzixc71x9ByGGMpQF4AcACALsATGOM7SCiPkQkeW9UBrCDiP4F92p6yazdQG0OhRhjKZ6OpQAoqFeRiJ6RjChpdtzWTEiM9/k7T9/pvvysQZM3L/dZV5I/v9oP36o9oXNnPojnycN99iUvFelzS0jgbd92m/paIx11cjJvU45c5RJqLxS5XaJ9eyBnTv5eLkAA/eB0gWL0fddKouMEkg1JC6sRRoVwcB2MsXmMsQqMsXKMsQ88ZWMYY2M8x2sYY+UZY5UYYw8xxnRmBj4SwtDpsQDGAkBycrJjCnB5AL44cr1d3TmMUiVeugR89RWQLZv2+d9+898XMXky8L//+VRACQmB5SpgTL2ykHvrvPaaeRvrFXG0XnlFu56R6ujoUf9QylNCbK8Ktb3n33/teQlFj+ulwAUEKhxOEVERxlgKERUBcNrJTlkhIc7XdSEcLMAYT6f43XfG9eRhkaVB1Wo4Zy30vHLs7IwFgHr1/N/rpYTUMqRKM+ZQzdYjxfXrQJcuke6FIEYJdFSdA6C757g7gF8N6oYEuVopnqIwoNXq1XxwDXW2LjmjRumfy5bNXDAA/oLg2jVu9JXK5DNhIm2bhxK5EfvFF/mARgQU1NVEBo5eWGrhtBBdOB2l1g0ULhzpHtjGiivrTwDWAKhIRMeIqBe4z2xLItoLoKXnfViJerWS5EMeaFgHqygHXz3MUixKKFcJH37oK1Oqdtau5Z5MVvv31VfWVTn799u3Rch3TsvREg7BJKoRBEdmVG+tWxfpHtjGVK3EGHtM51QLh/tiC/nKISqFg0So9c63bgFZs/LjhITgVECA2oialuZ7hnbt/M/16cNXRkarI6X6qHdva/24/XZr9aygZYi95x7n2hfYQxmgMDNgtFcnSoniUdUY+cpBnuMhaghGlfHXXzz1oxVSU7nXTUqKM1EflYO53ICrRBIKvxpoFe3aFgTuQr4pzyp2vxOhDOYoEWz4mBgk5N5KoUIuHFJvGXjoOEVqKpCUFNigf/UqkD27/7XXr/Pww1oDeqNG/NXKF3baNL7h68YNvnIIFqM0jXo8/7z+uWhIqh4NfYhV9DzfjChuGNJHTTjCdCtdrQXuXTkkJSZ5j79YZxp9NjguXuS++nohD7SQBMHevTyx+fjx/ueTkoAHlIETA0BS+TDmjHAIVi2lJBpWDpktOGM4CWS1qjfBekxHg211f4Zyo6UdKlcO/NoYxbXCITkxOXw3k3bOTppk/RrpByDtC9Aywv3+u71+bNmiLpMGvitXzOPyWMFqonmrRINwkIIMCuxTpozx+aQk4/NWefZZ9QRKwupq3YqgSkkBvtCYTDZpwgNBCry4Vjg0LNEw0l2whqTScMKd8s471WWScLAraPQIZvalRTSodORhNwT26NzZ+HyyQ5O0d9/VVlF99pn1lYNSOFSs6Dvu2ZPvji9cmMcJ06K+yEkvx7XCoc3tDqU6NGL/fv7FNAs9bYS0Yvj1V18wPDmHDnEdrBSD/3//850j4qEM4uP9UxLKMdL3B8KTTzrb3jPPmNcRRC9SqBE9kpICT4YkwZj+vpdXX9WfWJ054zv+5ht1PXmK1fHjfb/F/PnFhMECrhUOIY3GevMmD143dSr/4k6ezMsZ4wHTVq70r3/0KN99e/w4P960SfsL/eOP6rJZs/y9MZSD6WOP8dm31lLYDTi1ohFEBrPQJjlzGqtbp03T3lOijGsVLL17q39zGRnAihXq3yvA9wSNGGGt7a5d9e0hMYxrhYNyb8Oaow7qC196iSeMP3TIczPPva5d4/kFmjThSVEkSpYEihThK4CSJYFatayHZLZqANbTxwoEoUTaQ6NHfLx2AEiJ5s2BuzXS+jZpYr0Peg4Fksoyf35tZ4yMDL6fpXFj9bnERF98rurV9e9dtSrfuPnEE9b7GyO4VjiQIvnRt39rhE749Vd7ydhXrQI++cTnuy2lkJSEg3yXsVFoZAA4rRNuasYMoGhR33uzHM9Oh5EWWGfQIO3Q2nLdtpYdKLMgBWk0WsXnzu3/XgoMaceDTG63GjbMdyy1oXd/K/auq1fNd/kDwH33Wd97FCO4VzgovhAZTOOL0LEj8PTT1hpMT+ezjDffVH+pJOEg97wxm/HruZV26mSe6lEQHSQlAVmyqMvlA1u5cs7es2pVZ9uzQ6dO9urnyGF8/vXXfYL0u+94SlXJu8nO/og8ebi6df16/5VMgQJcnTRvnq9Mvov/0UfN286eXfszlpCPM1qh6tu25a9SwMeWLYGlS33nhw7lE0IX4tpNcEo0hYNVTp8GChWSNaYjHOSJ7c2Ew+HDgfdHEB2ULu0bHMqWBQ4c4Md16/K4UgBXSfzyi7X2WrfWzzctsX175AIFTp/u7L0//dR3/NRT/ucqVNC/LmdO/98akS+oonzgjYvzd+AAuKagSBHuZRWIJ1WfPv65teVorUTy5zdeBQ0ZYr8PUYJrVw5KpmybAmZnqXr+vC8WvmRbkFC6nyrPA9YD2QncyZIl2t5lADB7Ng+ktmQJ94iRBIURc+ZYFyLbtjkbinvsWB4GZcIE/l6u1owUtWppl//zD7Bnj/n1enm7CxfmauFAbXSff84N2FLQR7mwlGsOBg4MrH0XETPCAQD2n99vvXLXrvwHuF/jGulLIH0xVmikIXUwq50gyqhdmxtSifgsFPBPY5o7N189NG/OZ6/KXBNatG9vroaRqFbNp6YINLS5XFVSqxZ/pgYNAmvLCs89Z75hTkm5cupIutWr+1bxH3+svqZDB/76wQf67TZoEPj+i6xZuQG7eXP1OSlfef36fNMeoK+2fu89c2N+lBNTwsHWymH+fP5686Z6WSitHIzaGzWK60LF7tvYQ/6Dz56dfw+6d/eVWQ0ZUbOmtXp6ubABro4pW9ZaO3Ik92vAZ/+SBqu8ee23Z8bo0T61m1X27eP7E7RgDHjjDXV55cr83F132e+jHaR4TvKYS9Wq8XuvWQOUKsWPtTyhAL6idDraQJhxtXCY0dnf0GMpdPd///lnBIuLUwsBaTVhFvrhwgX93ZbRhtzLI1rQ+vFHA2aTDCPhIFdDzJ1r7X4ZGcDw4Xz2bUZSEvfPb9lSv84ff/i/l1w1S5XikxqjKLpaKJ932TJ928nSpdxQPHasvXtEG5UqcRXTtGmR7knEcLVwaFDcf5lsaWNcgQI+VQG/SL9uNMQFcgqzMAiRwK2ugVrfGUkHLo/bX6yYseFVIj0dGDCAz76VMOZ/v65duX++kSdOmzZ8cAO4cVV+/fPPAyVKmPdJzq5d/oNk06ZAq1badZs14w4eVr0EoxUivt9J7qiSyXC1cFAKg4CS/nz7LbB8ufa5WMpY5UTEVqdxyjNGLwNcOJH2q5Qq5V++ZQuP6muE1iRE63+zdq3Pk0bu/aMVQqV6deDUKe3wJfIQ2EZJj6S9OuXLA488ol9PEJNE4YgRZqxuoXc7RsKhZUuuj7aqBnEKp4SDU8HfgqF+fT5j//RTPsuWchYkJWlHLv3gA+Ctt/ix1RVq3ry+z9HM/x4wNmZ/+CG3p7Rt6x+gTo5yA9uCBT6PJ0HM42rhoNwlnZ7Bf2TX065j0JJByCTDvjWMhEPVqtx1L5DBukYN66FC5GjFwgmkzUceUc/WI0G2bD5df7Vq5vUHDfIJB7OdvNWrczuYnhAMJF+FFVdM5efTqpW+OkkQc7harVQoh78+cMnBJQCA8ZvHY+Ram8npYx0j4WDV+0Zuq5EIJpGOnnCwasxcvJj7szsdmTbcmK0cJk3ihl69/QmhSmYUqc14gqjA1cIBAMrm8bn5PTuX+x6nZYg9CCqMhIPVePlNm6rLAh2YlIZWgKs5AOMEMnIVSosWfO+A1f4rkdKxhptXX/V/byTcGOPP2KyZfp2HH+avderoq5gCQQiHTI2r1UoAkBDn/wgfrvoQObLobDa6cSMMPYpSjISD1UFAa1OP2cauli2BhQut3UMa5JV1GzUCVq/mx5cuGbelJXT06skJ10CoJUytxAAyom7d0KwehHDI1Lh+5aAUDsMWvoUcpy/4V5KSgmTmeEdWVg7yuDVKypVTe6x07eofulwLaYBZt46rR6RZstYgLqm3iPzzQHz5Jd+d+vXXxvcyY9Gi4K6PBJEaoEeNEsIhk+N64aA0Sk+eCTzVYQji5WpcyWvDqXy3bsTKyqFZM6BKFe06w4ap1TdTpnBffiOka+rW5THx5SGu9YQDwEMkS23nywfMnAn07es7rzRCS2EV9ChfHrj3Xn6cK5dx3UhGRrWLk2okOW634wiCxvXC4Wa6fwC8B/7lr3Faq+zPPw95f0JK+fKBX6slHHr35q9WdPZE/qoLpd4c4CszpdFUb/aptXJQqpWk+FVyv3yArwSVeRamTTMOhb55M389e5Zn61Ny9ixXWR0+rJ2cJlIYqYvOnvXPIigQOIjrbQ5K4SChOSS5fU9DtWrA3r3a56pW1U5MI6HlkSTtlLVqD5APVFrpI0uWVHveGAke5X2V7/WEg1bmsaxZeUROOV99BfTrx1cKkhuoXlwhqTxUM3Etvv6a5ygIFKdiJL3/vi+xlUDgwfUrhy/aaOdW1lw5uB2jQdwo1aHetdJgb3XlIG2Keu45fbdKSThI7qhGfdYTDtKr1Jbd3d2S+lAv5Ha00Lcv8P332ufCqe9/6y3gs8/Cdz+BK3C9cHig0gN+75nnN6USDsuWhadDocRowBg3juv1jVi6lKtehg/nBmJl3go5XyiELhHQsCEwa5Z6INm7l+cLAHxtSgLHzspBWS6tHKzuw5DYs4enfBUGVUEmgYjaENFuItpHRAM0zuciot+IaCsR7SCinmZtul44AEDR29SzWJVw0IrPHg1o7R3QwyhqZ3IycP/9vvd586rbbtaMq14GDOCCRDmQA1wNAwDduvlfW6cOf+3YUZ3i8fbbeb4AQJ0LQykcjAZsaZ+DVGfAAP9yqxQvHhq7wf3320tvGQzSTmSjcN4CAQAiigcwGkBbAFUAPEZESs+S5wHsZIzVANAUwGdEZJAfNUaEw75++7zHkkxwjVrpxx+t123enG/80kNuEzh71ny1JOny5fsX+vTh7cjj6jBmPZGLcoBXCgPpnomJ/ucYU9sW3nqLl0dL0MDffgOuXQvPve68kz97NBnHBdFKXQD7GGMHGGM3AfwM4AFFHQbgNuLRSnMAOAfAcLdwlPzqgiMpUe2iWuVMBDoycyaPhCl3uTSDSO0JZMQPP/hcPEeP5slI8uWz31eAG5UvXwZefFH7/J9/8nDNdli6lKfRlAy7ypVDly48DeagQeZqJacIVXgJgSB8JBDRRtn7sYwxKc5MMQByF7xjAJTpCUcBmAPgBIDbADzKGDMM6hUTKwct1nzncINWUjw++KD18NGSyicpifv/W6VoUZ/6qHhx4LHHfCoIu4NgcjKPIqq3/6NRI5+7q1UqVeLqID17RmIi8MknfGUinZOex+lBXNgcBLFDGmOstuxPHoBM64uu/DG1BrAFQFEANQGMIqKcRjeMiZWDHBaq8cDpgeabb4Bbt7ibpTIqZ/78PGOdHHn2ulD1yUmkgd7MW+nQId8mRSvXCAQCJccAyDM4FQdfIcjpCeAjxnMp7yOigwAqAdD1pY7ZlYPjOD2rTU72uZ8qhUPp0ur68oxUeoNoT48Dgh21Vqiw6iZbqpR65eKUcMiRg4f/njjRmfYEguhkA4DyRFTGY2TuAq5CknMEQAsAIKJCACoCMEz6HXMrB1ciFzydO5snatcTDiVLRo9+XcsTyowyZXjeAqc8guLjeSY2gSCGYYylEdELABYAiAcwnjG2g4j6eM6PAfAegAlEtA1cDfUmY+w/3UYRg8IhJEPjlCk8XMSVK861KR/Y5SsHK4OpHfXLrl3ccyncBCIcpk3jKVulLGoCgcASjLF5AOYpysbIjk8AsJWpSaiVzOjVi0cfNcvWZVcVoiccrHgu2REOlSpFJm+BtO+hUyfr1+TJw436gshhlFNakKkISjgQ0Sue3XbbiegnIgrTDiF9HDdId+9uWuVmuTL6woMx4J13jBuQxyOyIxyimUqVeD87dox0TwR2WLHCHd8vQcgJWDgQUTEALwKozRirBq7r6uJUx6IOyaPmhx9Up45fVkTGtJLmMlxqJYFAIAiAYNVKCQCSiCgBQHao3afcjzIInJQ8XhbvhyktHU8/bd6uPCSEQq2UWvcu42uFcBAIBCEmYOHAGDsO4P/AXaRSAFxkjC1U1iOiZ4hoIxFtTEsz3K0dNGXPAcm3gmjg6695TH850kCsFz7aCspl+s6d/uEpFCuHh+setNaeEA4CgSBEBKNWygMev6MM+K67ZCJSbfVljI2VdvUlhDhGzlKL7uyn9eK4lSnDwz7UU+48B88FUaSILwNZMANz5cr+7wcN8h3HxeH0rfNYWRI43r8P8NFHQP36/vUHD+ab58yisAoEAkGABKNWuhfAQcbYGcbYLQAzATR0plv2yXcVKHXRvF6PB4BCbwCj6xhUWrvW5+EjCYH27YETJ0ITlbNhQ+B///O7X5OngJSXewFvvgmsWeNfv3Fj4MIF/9WHQJMxG8fg0IVDke6GQOA6ghEORwDUJ6Lsnkh/LQDYjNLmHC1N9o1JZGjlvFHmIwbseWyYVbXSlp3EOwJLXL5xGX1/74tmE5tFuisCgesIxuawDsAvADYB2OZpy4KbTmj4aYa1etVO89c/bveVsYoVVPV0h3PJEO0XYM8B1z9ZoDrSTnIqsEmGJ+jk2dQIbAIUCFxOUNNUxtgQxlglxlg1xtiTjLEbTnUsVORP5a+/V/SVHbx0hB/IZvgnLnPHq9NXT/s3EBcH3LjhlynNVDTYWYWIlYNAIIgCMt1IpLVJrkmD3fi2FoB77/WWSbPNk1dOqS/IksX5QbxbN+Cpp3iyd4FAIIgwmU44aNkcjucC+rSHn5vq5ZuXPUcOqIwqVjSvk5QEfPcdD9ctiEpOXjmJ3f/tjnQ3BIKwEHOB98zQEg4SP237CQWTC6JFWYNUnIHw5JNA+fL8LxJB8ASOUOSzIgAANkSElxDEPkI4yOg6sysA/uN//x7gjylAaoWygd2ocWOgalV+TAQ0aMCPbawMmIhxIxAIIoT7hcPGjUAdo00L/lgdbueXB2gogJltwaoHMEivXGn7kovXLyJXtlz27yUwhMROcoHANu63OXz5pa3q8pVD0+5Ag16B37p7R+nIuvBIy0jDf6nqHBvrj69H7o9z45edvwTeIYEmYgUmENjH/cLBLM+CArm30ooywNoS6jqz/52tKjt37Zxq4P7Nsz1iTo0kVf1PVn+CH7f9qCp/Zf4rKPBpAVy+cdlbtillE+qN4yE7lhxY4p3pqgL6CQQCQZhwv3CQ50KwgJXh9sGp6oQznad3xiPTH8GRi0dAwwgf//kxzmcHbhsI/F+rHKr6by5+E4/PfFxV/ssuLmB83lDAkOVDZP0TAkEgEEQedwuH9HS+Ic0GgSYDOniBR0o9evEoAODtZW8DAK5kBTIs/heXHFiCk1dOqsoT43wutOFUgTDGsO7YOqF2EQgEKtwtHBISgFmzTKvd19V3bOStZMSB8zx4U+/fegPwH8SlMA1m3DvJt8lOag8AEuJ8fgEMzDR8BmMMt9KDiU3OmfzPZNT/rj6m75wedFsCgSC2cLdwMGFqVaBZd+APWeikYOfIRy4eCbIFTrsf23mP5cLBCuM3j0eW97Pg8IXDQfVh91m+oWvP2T1BtSMQCGKPmBYOaXHA8jL+ZdOqBtlmBk/6I7cNBKKWuZHmU4dZFQ6MMXy36TuM3zIegG9wDxahVhIIBErcv8/BgHQN7czmIsG1KQkHOYEYkeXX+KmVFAP1qsOrsPXUVrxQ9wUsP7Tcq9bSqmsXSX0Vq0bwWH0ugSAcxLRw0LIvBGJzkAsEyb4gH5i19i3YIZ7081HfM+EeAMALdV/AlZtX/M4FO/jFEV84ipWDQCBQEtNqJaeEQ93/WU/HeerKKdAwezfR28FrNmgHvXLw3NeqQd1tCKEnEAROphAO/Rv295YFMlxsPrnZct2JW60lspYPXHLvJKMBTSlEgl05RJtaaeaumcj1US5cT7se6a4IBJmeTCEc6hevjxbdgJ+qBb7PwSpabqgXr19Er1/943TIB2T5oH/q6ilsOLHB0r2cWjlEywy7/6L+uHTjEo5fOu5Ie9Ei9AQCN5IpbA7xFI+lZYGlAQZY1UJv4FHO7g+cP4ARa0Z4PYy81+sMyL/v/d17/Pm6z037kJaRZujtxBgDA/PaF/z6mknSkQohIRDYJ7ZWDr/9Bkydij871gLgEw6NSjbCyNYjUTp36ZB3QTnglvuyHEZvGK2qZ0XP//P2nw3bfmfZO0h8LxGpt1J122g0vhHi343XPQ84N3jK3XODwan+RMuKSCBwI7ElHMqVAzp3RoNmTwLwCQcC4eX6L2PWo+a7qYPh5JWTljeU+amVApzBbzm5BQD8gvgpWXNsje45J9VKSw8uRbYPsmHV4VUBt5FZVjICgRuILbWSJ69zfDYeJfVqFl6cznhwPrnLaCgoPqK4915Os/30dsfb9LqyOjBTX3JgCQBgxeEVaFyqcVBtOTXjF+okgSBw3Lty0Aq4J+n7e/QABg/Gu03424LJBQGEdrCYtHWSbcHw3O/PWa5b/ZvqjietkWbqTriyOtE3kZRHIIge3CscevZUl3lWDsiSBRg2DKlZwtedbrO72b7mm43fOJKwPtBBNRTeStGk54+mvggEoYSI2hDRbiLaR0QDNM73J6Itnr/tRJRORHmN2nSvcJg7V11mMkhG42CRlpGGn7b/5Fh7Z1PPWq7r5D4HLXvB4gOLkZ5hX80m1EECgXWIKB7AaABtAVQB8BgRVZHXYYx9yhiryRirCWAggBWMsXNG7bpXOGTRWBbEue9x4igOF29cdKStRfsXIf+n+TF/33xL9ZUrjlaTWqHU56WC6oM0sC85sAQtJ7XEB6s+sHyt0wZpIWQEmYS6APYxxg4wxm4C+BnAAwb1HwNgOiN132gqoSUczFYOUThYaO0/0MNs8Fx7bC0AYPWR1bp1ft/zO9YcXYM1R31eTNKKatGBRQGHJFcKmhOXTwAA9p7ba3rttB3TUPrz0iEz5gsEMUACEW2U/T0jO1cMwFHZ+2OeMhVElB1AGwAzzG7oXuGQmKguc6FB0xFDrkdomOWePnThEO7/6X40HN8QDcc3tO2tlHorFc0nNseuM7t069hV3U3+ZzIe/eVRHL542NAlNxCiUY0Y7Xyx9gv0ntPbvKIg3KQxxmrL/sbKzmkNInpf/vYAVpuplAA3CwctQhyoLhTYWTlcunHJ8LzXhsAYes/pjfdXvu93XrlZzq630rKDy7Ds0DK8tvA13XvbgTGGJ2c96Xvv+T4rP6e0jDTHQmoIjHl5wcv4bvN3ke6GwB7HAJSQvS8O4IRO3S6woFIC3CwcKlZUl2W4L7qoHeHQZUYXzXJpUJXaymAZ+G7zd3hn2TuG7dn1VjJbmcjP6Q30WnW973Xqvjz/ZRQfWRznrplOdvyYu4c7LVy5eQWnr562da1A4CI2AChPRGWIKAu4AJijrEREuQA0AfCrlUbdKxy0MBnkKuSrYHg+Euw7ty/oNqRB1crgLUfPW2nwssHYd24fWvzQwm9QNVodSPdec2wNcg7Pie6zu1t/AA9KwSIhxZu6cP2CrfbkiZHGbx5vUDMytJrUynZ4d4FACWMsDcALABYA2AVgGmNsBxH1IaI+sqoPAljIGLtqpV33CgctXb2JcEjOkhyizgRO68mtg25DGkylwfvj1R9r1lMO7q8ufJVfr/i/vbfyPZT/qjyWHlyK0evVcaGMVgML9y/E5Zs+24GRTUXZjp56S64uCxQ7K7RwsejAIkv1jl06hhFrRoS4N9HBxC0Tsf74+kh3w3UwxuYxxiowxsoxxj7wlI1hjI2R1ZnAGNNWP2jg3vAZAQiHWEUaNEM9ABoN9E64oSoH/7OpZ7EpZZPtFZEW0SgcrNLx5474O+VvdKzUEWXzOBhaOArp8WsPAAAbkjl/y9GEe38xWigGl9H3jcaguwf5la3oscIbTiNWYGDIYBl4Y/EbhvV0M84ZDLpa1zgWNVVpc1C8b/djO7Sa3Mqb/MeplcObi97Eg1MfDLitcCOp0wLZUCgQBIp7Vw5aA0XWrH5vn6ujjl10T6l70Ob2Nvhh6w+h6lnYYYxZGjj0ZvfBenGtPrIas/61H/FWT60kle/6j7vM3kq/xcuDEEryZ//kr08AADvP7ESVAlX0LokavGpDF7pqC9yLO4XD2rXAhQv+ZTNmAKWC293rVj768yPsOWctVLgWVl1Z9YTL3d/fHfC9jfohJTGSNsfZEWLHLh3ze6+lVmo+sTlOvn7SbjcjhghpLggn7lMr3boFNGgArFcYrR56yHITVn9k+bPnt9OziDFqwygs3L/QtN7XG77WLDeakQ9bMUxdXzZIHzh/wEIPtVHeVykcpBDrequiJQeWYFPKJs1zJUaW8HsfH6cO134rg69Ijl065qjK5nradWw7tc2x9qJxf44g9nGfcNDay/Dkk+oyB6iYT2MvhYv5cv2XmuWMMUsDkKTWyGAZeG3Bazh84TDKfVnO0r1PXz2N1FupuHzjMtr/1B5HLx5V1VEO0NLKwatuAsOJyyew4/QOAMC9k+5FrbG1LN1fb0Jw7NIxlBhZAoOXDbbUDuALDaJHj9k9cMeYO3D+2nnLbVrBTK108PxB/HnkT0fvKWfo8qHC9TYT4U61kpLcuW1VL5RcyFI9N3u42GHM32PwUv2XDOtksAw8NuMxADwD3ZKDS/DnUWsD0aELh1DmizIAgMr5K2PXf7uQP3t+jGk3xq+eXAgAvtm+XK1UbAQPGfNaA/UubQmtzXJ6ObRPXuFqpfn75+ODFtaCBBYbUQz7+u1DubzagnHl4ZUA+I70PEl5LLVphFVbS9kvuSdTqDx9tFaRgtglqNGPiHIT0S9E9C8R7SKiBk51zBbVqtmq/m6zdy3ViyUDoJld4fO1nxueP5t61jvoJsbzuFanrpyydO/DFw57jyUjM4FUg7gkBKQwIV6bg2dF8eGfH3rrfrbmM817nbl6Bvk+yacq1xP0ge6hMApQmHIlhbcdouRMEsVHFEfrya2RnpGuyjcuEARLsFPjLwDMZ4xVAlADfHdeaFH+iOPigKefttVE1oSs5pUQWwZAM536lZtXLLeVJZ5HxD188bBJTR5YT2vm+/2W71F0RFHNa/ov6g+ArzgAHltJasuIKzevYOTakZrnUq6koOhnRbH3rC9KLBH5hRxxGqe+P3LBdeH6BdAwwm+7f8Pxy8excP9CfLX+K++qTiBwioCFAxHlBHAPgO8AgDF2kzF2waF+6aO0OdSsGVA01r+e+gtTO03VPJc1nguP7InZbbcbrUgDrB7SLF0P+SzYrrrtr6N/2ar/55E/AzLC9vujH4b/OVzz3C87f0HKlRS/kBoAQiocrHL1pjqaAWMMC/cv9Ps/EBF2ntkJAH7PaWYDETgDYwyrDq/KNA4CwawcygI4A+B7ItpMROOISBWfgoiekWKQp6UZD1CWUH4wAX5QDUo0QOeqnTXP5U3i2fN632UeurhDxQ4B3T/cmOVKkDaaadFtVje/89KM3ipvLX3LVn0A2H56u636O8/sxIQtE3TPS6owyR4A8Jm9Mmz5icsnQMPIG7RPD7mw/GLtFzh4/qBhHSO0wp38vP1ntJ7cGmM2jvFbeUlCLJZUntHE9bTrWHpwKRhjGLxssJ/6cMq2Kbhnwj2Y9M+kCPYwfAQjHBIA3AXgG8bYnQCuAlDlLmWMjZVikCckOGD/DoPUlgaSGoVqmNaVBEm0Y6ZWmrpDexUFAJP+mYQ5u1VBHkPKHWPusFX/rm/vMjy/5eQWzXK5BxYA/H3ibwDAt39/a9iepDI6f+08Xl7wMlr80MJOd/3Q8mqSVHZyQUwgzVAp4Z7JxvLM+cU/XkSLH1pg2o5peG/le3hk+iPec/vP7fd7jXWCEQ7HABxjjK3zvP8FXFiEFuUXMwQzKMm/PpYINstaJNUuVriRfiOg66wYpG+k3dANFy79X7VSvVq1OVy5pW/vISK/vimDLALA0kNLDdtfeXgl/kv9z1JfJM5fO297hRgLSGo76f+ltaKOxoySoSBg4cAYOwngKBFJmwFaANjpSK+Mb+xoc9UKVsMbDf1jEkkulNE+INoh2E1ekfxfhOrHKDdIG93jvh/v0/SA8mtLQxBYVf1oCSZNgaAQFBJ6GwElmkxoguYTm+uen7hlIr5a95VfWbERxbzux6q+xfDgqAzyqLT5ZCaC1fP0AzDFk2DiAICewXfJBIeFw7a+fCfruM3jvLND787cGMppHOyzXLyunhmHi1AKJqVBWmvgW3pQf2buhIrFLNqtN18HSJXYyQzp2m2n9XdsS5FQ+9Xr5y27lnbNUvuxhtEmwlhWp2kRlCsrY2yLx55wB2OsI2PM2S2h2jf1f1+njiPNyndDSysHMw8fNxHsymHwcus7iMON1m5rK8gN0noCyGr2OadnlVpCiogsG6QzWAY++vMjTXVX0H3LRINkLK+SzHDfFmDlF/OLLxxp9rfHfvMeSysHKzPW7jW0M54Z7eCNBLvP7o50F0LGoKWDzCvpoDRISzpnaWKgFwLDaBOckyiFgNXcHb/v+R0DlwxEno+1d2inXE6xtbdFzrW0a6p85LGGlgDMbGol9wuHrNY2tJmRL3s+3FPqHgDAAxUfAAAUyF7Ar06urLlU1zUt3VSzvXbl2znSL6cIxpsm2vn3v38Duo6IUP6r8gD4YDBr1ywMXDIQgM+VVm/m2G12N9xI8xnBb6bfVO3nsJybW8NeoWVzAGSurCbGbiPXZAAoOqKoqYeXHoX+rxCSP4y+rIqhIDOtkpS4Tzi8/nrIml74xEKcfeMshjUbhpOvnUSR24r4nbez+SszL0fDzcYTG4Nu4/jl43homvXIvoC/2vHSjUtoNL6Rd+UB8O8AYww0jDBw8UDddow8reRCgDHm/V7J7xMoe8/tNa+kQayvGgDx+wXcKBy+/z5kTWdNyIq8SXkRR3EolEMdnE+5rHy53ssAgDK51V4dTs043BI23I3IP6Ob6TftXw+mGkTOXD3j17400/9o9UcAgEFLBmHU+lFYcWiFt96P237UbFurn1J7UvymQJBvMFR6KQk4Rr/fzLKacJ9wiBC97uyFxU8u9isb0ZonfT/w0gEMaTLE75xTHjaZ5YsYCQzTo1rY/3Ar/ZbK0C9v8+KNi6p7DP9zOPr90Q9NJzbVbXfKP1Ow4cQG3g/ZhERaiQTLh6t8AQxfnP9i0O0Fy59H/vSGYTdj55mdoGHk3awoCB1COFhkXIdxuLPInX7hkOU/3KFNh/rVD1Q4FL3NPxidXlhoQfCcvnpa95wV42PeT/Ki5rc1/crkg3fl0ZWx79w+S30Z+/dYbD25FQDwxKwn/Hak+60iglB3SHaIaFILbTm5BY2/b4xq31iLrPzbbu44orej/8L1C1i0f5Fj/dPzGssMCOEQACt6rMCKHisM6wT6Iy6du7Tf+7a3t0X94vVNr0uMSwzofgJtpJWD2UCgFDDKz73ZxGaW7vfs3GdVggYA/k7525tzwnJSJp2EPA9OfdBSX4LhbOpZQ6GrZMjyIeaVZHg3qen8HzpN64RWk1vhbOpZ07bWHF2j66r8/RZ99XVmWc0L4WDCv8//i+19/YPA3VPqHq9nkx7ZErIFdD/l3orOVTtjVc9VuKOQcawhKeWlwBkCnR0qV4xyG0QgzN833+99MCuH+fvmY+/Zvdh8cnNQfTIi/6f5Uej/tJNp/XnkT/yx94+g2jfz0pIM9WbeWgDQcHxDtJzUUvPc1lN8Fee3Q9pz76u3rmYKASGEgwkV81dE1YJVbV/XpFQT7/GJV62HVJbrsNkQhioFqiAhLgENikcmj1JmJdAkQMp8EvKd6cEOKAwsaFtWhVEVbO3ROHnlpGPpTht/3xj3/XifI20xMNxKv4VSn5fCjJ0zvOXK8Be613s+C7PQI1qMXDsSn6z+xPZ1bkMIBweZ0Zl/SRuVaAQiwuGXDyPltRSVS6ySduXbIWfWnAACD3MhXS9whoMXDqLv3L62r5u3d57uOaMQFlYJ94y1yGdFUPizwravM9uRf+3WNbyz9B3b0X7lK7pz187hyMUjeG7ec77zFoV6sELWKIpxrCCEg4NIm+akL3DJXCVROIf6hzWvq28AYUMY5nadi90v7Mbfz/yt+6My+rKv770er9Z/NZiuCzQY8/cYywZlKxgFv7OCfJ9DOAnEzTfhPeOwbZ+v/Rzvr3rfVpuNxjfyZgn02ySoERzPdOWgcV4vXM6o9aNAw0gVQsaqDcitCOHgIEZfSGlPBAC0Ld9Wdb5wjsK4q8hduiuH5Cz6O1LrFKuTaTwowo1dNYhRRj0rdiGzeF5uixQ8f998LNy/UFX+6+5fDa+bsGWCSjDLd6DvOLMDxy4dA+D/u1OuHCZsmYCJWyaq2tf6Py7Yt0BVdurqKfT7o5+qHODRbuuOq6t5bv3x9fhg5Qea59yCEA4hQMtoNrIN10VrheCQ81mrzzTL3232bvAdE4Qco1wgl25cMr3eaFPai/NfDDqAYqD0X9jf9jWzds1C2ylt0Xpya5WdY93xdTpXcXr+2hO1x9YGwPNpKD2wFuxfgNr/q626Trly6PlrT2/UWTlrj61VlWlNzIwCL646sspvd/63G7/1eknVG1cPby97GwC3a7hxhSGEQwjQW0FMenAS/n7Gt3lHK8Vom9vbaF6bI0sO43u68MsXiwSadEji1YX66sG5e+Z6Yz+Fm/9b83+2r5GHIyn1eSnb10sRZc0CBGp5FBn9HjalbEKTCU1U5VaTMwHq3/jWk1vR5/c+6Da7m1/5jJ0zUGtsLUzZNsVy29GCA3k7I8hC9XI1mnnijie8xzfevuFoxjkRCyZzsP+8tRSV4f4+dPy5o6mqKBzIVw795mmrg1Iua4cesfM/U6adlVxn5+2dh6HLh3rLpaCQu87sstx2tODulUNLbR/lSGNlBpIlPos3b4RVRrQaoXtOrBwEcgJRP03dHrgHTigFgx17mvTbG71+NEZtGKVZRykE5u6ZG3jnwO0a8nhVw1YMC6q9aMHdwiFG2fLsFk3bwysNXtG9RqwcBHICCWPeZUaXEPQk9Jy9dhbjN48H4IucPGKt/kRKSfuf2gd1/56/9kTv33oH1UawEFEbItpNRPuIaIBOnaZEtIWIdhCRcYgHuF2tFGU4NXuvUbgGahSuEZF7C2KDd1eGz4HBifDhQHARYnvN6YVsCdksq920CMVvKByTNiKKBzAaQEsAxwBsIKI5jLGdsjq5AXwNoA1j7AgRFTRr110rhxWmwi6iSJvd7i55d8jv5aS9QiAIhqpf248goIVehFirhuLHZz5uWmdzymbNlYLLdzzXBbCPMXaAMXYTwM8AHlDU6QpgJmPsCAAwxkwDYLlLODz3nHmdCFIhXwXsfG4n3mv2XsjvpYzdFEm10oanN0Ts3oLIcezSMeT6yNg1O9oYt2mcZvmbi98Myf3seEAFQTEA8kTqxzxlcioAyENEy4nobyLqBhPcpVZywUavygUqh+U+ermFI4GdDHmC2KHEyBIhv0fqrVRHJj4L9i3Q3NsgJxQTLAfbTCAiecrDsYyxsZ5jrYFReeMEALUAtACQBGANEa1ljO3Ru6G7ftXyAfEVfeNsZmDSg5P83jv1JZzTxV6sG8BYxXXq9VPBdMcR+tTqE+kuCAKk3Y/tHJn4tJnSBkNXDMXlm5cd6JUx5fL4crC8s+wdp5pNY4zVlv2NlZ07BkAuqYsDUEb7PAZgPmPsKmPsPwArARgaNt0lHOTkzh3pHkSUjpU6+r13auWgJWTMwoUbueTK80wUStYO5axHwxINbdXXo189bX93QWTQSouqx/JDy9FwvDPfA8A4TtRHf37kyD2CMYoHyAYA5YmoDBFlAdAFgHKW9yuAxkSUQETZAdQDYLj5wl3CQb5yKBH6Ja2bCGbl8NPDPxme/73r74bnjdRKcvXXkVesh4oGgEerPmqrvm4fwqP3FVhECp5nlT1ndTUftjGKpmoW0iNaYYylAXgBwALwAX8aY2wHEfUhoj6eOrsAzAfwD4D1AMYxxrbrtQm4yeZw6RKwTRbyuEePiHUlWljSbQmu3rwadDsPVX7I8Lw8HtTj1R9XhQLQEw4nXj3hdy5LfBZb/XLKlmEWzE4QXk5ctp7fRGANxtg8APMUZWMU7z8F8KnVNt2zcuja1f+9C4zToaZ5meZoX5G75d2W5baA25HPrLXUU7dlvc2b27pFmRaq83o2h4S4hKBm7b3u7BXwtXKU0VAr5w+P04BA4GbcIxx2ydRjOYyD0MUyJXOV1Cx/veHr+LSl5UmBH1bCEyQlJAHwV191r9EdD1R8wCs4lDCwoGb/SYlJAV8rR7lyKJFLqCQFAjPcIxziZF1Ny7xqgp3P7cR//f9TlWdNyIrXG74eUJtWBnBJgMjj4Her0Q2zu8w2zGFgNS5O/eL1/d4fePGApeusIDeKA8L1ViCwgnt+JfJB5rp58vBYJTlLMvJlz+dom35qJR3DtlRHLhyqFKgCgAsmPQIdiMvkKRPQdVrULFwT37T7xtsXIRwEAnPc8ysRNoaAuDLwCtgQ5k1hqoWV2X2FfBUA+Gwb91e4XzMFqhw77rXyuq/Ud3YPCxGhT+0+yJMtD38vvJcEAlPcIxzi3NPVaKFivore9KInXjuB9hXUMWX2vODvJqgc0I++wnflT3loCuZ0mYMu1brgg+YfYHyH8Yb37t+wPwomm8b20mREa3VEzc3Pbg6oLS3EykEgMMc9vxIhHIIiIS5B0zZQPl953WuSEpJQPGdxAECubLnQvmJ7EBEGNR6EAsn6KxEA+KTlJyAilb7/x4esb4CSU7NwzYCu08JuHg2BIDPinhFXqJUskT97fu+xnrqodlF17l2JpqWbIne23ACcCcmhHIiVO7sl7N6reM7iWN59eUB9GtpkKDpV6YRtfbeZV3aIfEnO2okEglDjHuEgsMS/z/+LX7toZ+WSBuCBdw/UHazyJOXB8VeP276vXIW08emNuvWSEpPAhgQvdAomF0ST0uo8wAD36DLqQ+EchTH9kemoVrAaqhWsFnRftGhVrpXfe2UUXUHmYtWRVZHugm3cIxwyMszrCJAvez6Uz6uvKgK4QfbASwdw+nXtkO5WkrQrkdRPAFCraC3L10m8WFc7lr8eWeP1PaQqF6iMu4rchZfrvYyl3ZaqzstXVHLvKyeJozg/lZqeIBNkDoRwCBVHj/pvghMETc6sOVV2gza3twHgn6TdKoF6ACUl8JXE43doJ2rZ8uwWHHzpoKrcaG8FwJ9hZJuRaFammV+Zsq/SCmprn622+25EHMXhuTq+/CNf3/e1o+07wbCmsZHrWBAa3CEchg+PdA9iguEthqNhiYa4t+y9qnOnXz+N2Y/OBhDYQP/kHU8G1KdxHbSTr0jUKFwDpXOXVpUr7Smj2monk5ejtRKa2mkq3rnnnYDUS0ZJnZQeUbmyWUuK06lKJ1t9+KhF4JFEBzcZHPC1gtjHHcJBaVjt2DEi3XALt+e9HU1KNcG49v4Db6X8lbD6qdW4Las6DlOB5ALezWyJ8Ykomauk6nojXqj7gu653Nlyq/ZZLH5yMVY/tRpdq3fVucoefev0BWA/uF+R24rg3WbvBuTe+vY9b+uuOAJdSWVPzG65bp2idfDm3f4ZzL5o80VA95Uz/ZHpQbchcD9BR2X1JLfeCOA4Y+z+4LukeRP/90WKhOQ2sUJifCKW91ge8PVxFIfDLx+2dY00k29Wupnq3Jn+Z1SDZYuy6gB+ZkztNBWP/qIdxjuO4vDv8/8iT1Ie0z46iV6uiziKC/lmOy21n5GQy5MtD85fP2/arl6sLEHmwomVw0swSRoRNMoftXBrjTriKA7b+m7DnMfUmeQS4hIc2VsgBf/To2L+igFvvAO4F5dTJMQloFTuUn5lu1/Ybenatre3Dfi+j1R5xHv8RsM3/M61KtcKKa+l4NBLh/DXU39ZbtOp6LgCdxGUcCCi4gDaAbCuf3CCK1fCejuBNaoVrIYcWdwbMffDFh861lZSYhJerMc9sKSQI1Jo87J5yhpeO6PzDM3yj+/92PS+hXIYZ9srnKMwSuUuhQYlGpi2JWFmFxLEJsGuHD4H8AYAXX9AInqGiDYS0ca0QKOpKlcKbQOfWQncSyjUQqEiKSEJcRSH7X23Y2+/vQCse3/phSpXCl4zV+NBjQf5vbfqthtIylkRksScG2k3It0FWwT8iRLR/QBOM8b+NqrHGBsrJcVOSAjQxKEcFLp0CawdgauR6/CfvuvpCPYEmNl5puF5SRVTtWBVzdl87zt7276nXRuG0kPq4o2Ltu8pIQVe1EMIB3NCtacmVATziTYC0IGIDgH4GUBzIprsSK+UuGjGKAg995W/D0/c8YTt60a2Honsidm94UGC4cHKD6rK+tXt5z2uV7ye4fVft9Pe9/B4de39HhJreq2x0Ds1pXKVCsqTySyPsxAO5jgRjiacBPyJMsYGMsaKM8ZKA+gCYCljzP4v1gpCOAh0eLney5Z9/Z+44wlcHXQVifGJ5pVlrOttLfH8l22/tFTPaAWgDLvhdx2RX1IkK4PN2TfOInVQKg69fAiV8ley1D95u0VyWPMM1EsVK/CRmVYO4UMIB4EOI9uMVPn6O03dYnXxZRtrA78Rcl1+Ynyi7R3Kytm5FdtA3qS8hulWpTYvDbikavfuknfjxGsnLPVNHj4lMxCIR1kgtpxI4ohwYIwtD9keB0AIB0HE6Vevn9dO8FTNp/zONSkVWNykwU0G48rAKyiUbOxhJNGtRjfN8mD2JUirmGCN/Uu6LQnq+kjSt3Zf29fMe3webr5909Y1YuUgEISYcM3AJj04CUObDPW+l8Kh3573dr96y3sstxRpVtr9LFfvJGdJRs+aPQEYq2a61eimiuwqqX/W915vem89tOJN1SxcE4VzFMb7zd633I5y5WBFYElpZgPFyspLL0S8nHrFjO1DethVT2Yam4NAEG7C4cr6actPvcftyrfDkKZDVHWMfuSf3PsJNj2zSfNcsZzFMP/x+fjxYf+ER9KKQCvxkpRxTz54K9svlrOYbn/M0LJ/3Jb1NqS8lmIrkqzys6lbrK7pNf/0+UdVNrXTVMv3fOeed0zrWJlIhGvQdtvKIejwGQKBUyzrvgznrp0zrRfKH/PrDV9H/0X9AQTmgdO/UX/D861vb60qk1xdH6v2mOqc9Kx2+rKm1xpcvnHZUl3vyoEIi59c7E0rG2oKJRfS3DXfuWpn3RApeZPy+n0/rEwWrHxXrK5E8yXlw9lrZy3V1UIIh1Bw7VqkeyAIA01LNzU8H+pYRUqUA7I0Q7fqwWOVvEl5cWXgFc2gezmz5gQAVeBCQH9Qk3s0meG1OYBsx7vqdWcvfLf5O1vXSEgDJYEsC/thTYfhraVv4dKNS+aVPaRnpJvWsXp/o3qty7XGgv0LjK/PjAbpkDNpUqR7IIgCahSuASCwDWSBoBQOz9V5DtMfmY4eNXs4fq/kLMmaM+GHKz+Mb+//FsOa+fTrTqrXrObu0BKI8n0dqnZNBLkUIt1OzK06RevYHmDNZutvNjL2dCuTuwzuLHwnAOPB3YpTgttWDu4QDi6TuILQUPS2omBDGB6u8nBY7qcUDnEUh05VOoU1jAcR4Zlaz2imGXVCvWY169/6p/2N3nEUhxqFayB1UCquv3UdgH+4cbO0qDMf5TvM7eyPqFe8nu0B1ux/9NG9xntkutfojsXdFpu25ZSKK5pwh3AQrqyCCBCtsZyqF6yOXnf2ws8P/xx0W5IANBu4iucsjoYlGgLgdgEpj0VSYpI3D8jp108jOZHbLIY2Heq9tnHJxqr2pF3q8v9x9YLVA3sIA5TCRB680IpgTMtIM1wFyVPB2u1LtCOEg0CgQ7SGhIiPi8e4DuNQuUDloNuSkjRZSZL0Vduv0LBEQ0x4YIJm5rzkLMne4ICSrQQA5nadq9umfOXwT1+195ISpRA7+dpJ7Ou3z/u+fYX2WNd7HX7v+ju2PLvFOyBLn2WpXKW82fYkwfRQ5YdQp2gdr2CTk5aR5lO9aQiRf/r+YzkplhAOAkGMEK3CQQstg7UVhrcYjvTB6ZaEw11F7sLqp1Yb7rjWWoHIBYUSu/9jaYCVQpoUylEI5fKWw2etPgPA96DULVYX95W/DzUK1/DWl69UJj/IQ8BJK4I8SXmw/un1KJe3nOp+aRlpXnXZ83WeV52vlL8Set1lLd+FMEiHArFyEEQANwmHPf324OgrR21fR0SOPqc0AFr1LDMSNEbtKzPwKVcIEtLGRS1PMGVd5UZAAKhcoDKyxGdB+uB0vN9cf1Ngm9vbAACmdZqm++xi5SAQxAjhdp0NhtzZckdVfCOlvWZ4i+GaQQVX9Fhhq906xeoAUBuy5a6xcr69/1t8fd/XmgELVbGqZKuezc9uxqZnNnl3r8dRnKENqmbhmmBDGB6p+ohuHSEcQoFYOQgiQCQN0idfO4mU11Iidv9AaVSyEQC1t9KAuwfg0arqzW1WI8VKzH1sLtb2WqsKXSGtKJQDfu5sudG3Tl80KM4z35XNU9ZrRJbvhtfiziJ3BvUdyJXVP59GKL2ViKgNEe0mon1ENEDjfFMiukhEWzx/g83adMcmOCEcBGGkXrF6WHfcWpjuUGGW7jNamfLQFOw5u0fTzmCWMEhJx0odMfvf2X5lubLl0syV4V056IwVr9R/Be3Kt0PF/BUBwFIsrEDRW5mEauVARPEARgNoCeAYgA1ENIcxtlNRdZWdAKnuWDnkcG9eYoH7WPDEAmx5dkuku+FKsidm9+rrldxd8m7seG4HMgb7D5Jf3/e11zUWAH555BcAQM1CvJ0n7ngCZ98wDlvxbO1n0a58O7za4FXN80TkFQx6WLG9rOyxEt+0+8a0HqAOzBdCg3RdAPsYYwcYYzfBk689EGyj7hAORZwNVyAQGJErWy7vbmyBs1QpUEU1u+9bp6+fgfnhKg+DDWEokMw9sHIk5kDepLyG7eZNyou5XeeiYHLBgPs2/ZHppnUal2qMPrX7WGpPmXEwmDStABKIaKPs7xnZuWIA5N4IxzxlShoQ0VYi+oOIqpreMJjeho0M2UyjdOmIdUMgEFhnWNNhmnsHrCIZsJ+s8aRTXTJEGYo9UDpX7YxpO6YhIc5/eL2Zbi//g4I0xlhtnXNaujTlMmUTgFKMsStEdB+A2QDUYYDljYbT9zY5OZldvXrV/oW1agGbPGGKXeYrLBAI3AMN82x4C8ImkZ6RjhvpN1BvXD1sP70dANC1eldMeWhK4P0iSmWMaUpaImoAYChjrLXn/UAAYIwNN2jvEIDajLH/9Oq4Q60kBIJAIAgDc7rMQYeKHYJqIz4uHtkTs3vdat9r9h7G3j/Wie7psQFAeSIqQ0RZAHQBMEdegYgKk0efR0R1wcd+Q0OO+9RKAoFAECLaV2yP9hXbO9KWZFu5v8L9Ic2TwRhLI6IXACwAEA9gPGNsBxH18ZwfA6ATgL5ElAbgGoAuzERt5A7hkG4ek10gEAiiCW9QwzBoPhhj8wDMU5SNkR2PAjDKTpvuUCulpUW6BwKBQGALSa3ktp3REkI4CAQCQQiwmkgpWnGXcFhgnIZPIBAIooVwqpVCgbuEQ+Xg49cLBAJBOJCEg1ArhZJsniBeidazLgkEAkEkkTYARmtGQTPc4a30xx/A9OlAIXcGIxMIBJmPyQ9NxpiNY1CnaJ1IdyUg3LFDWiAQCDIxRjukQ4U71EoCgUAgCCtCOAgEAoFAhRAOAoFAIFAhhINAIBAIVAjhIBAIBAIVQjgIBAKBQIUQDgKBQCBQIYSDQCAQCFSEdRMcEWWAJ5oIhAQAmTk8q3j+zPv8mfnZAfH8CQASGWNhncyHVTgEAxFtNEiwHfOI58+8z5+Znx0Qzx+p5xdqJYFAIBCoEMJBIBAIBCrcJBzGRroDEUY8f+YlMz87IJ4/Is/vGpuDQCAQCMKHm1YOAoFAIAgTQjgIBAKBQIUrhAMRtSGi3US0j4gGRLo/oYCIDhHRNiLaQkQbPWV5iWgREe31vOaR1R/o+X/sJqLWket5YBDReCI6TUTbZWW2n5eIann+b/uI6EtySU5GnecfSkTHPd+BLUR0n+xczDw/EZUgomVEtIuIdhDRS57yTPH5Gzx/dH3+jLGo/gMQD2A/gLIAsgDYCqBKpPsVguc8BCC/ouwTAAM8xwMAfOw5ruL5P2QFUMbz/4mP9DPYfN57ANwFYHswzwtgPYAGAAjAHwDaRvrZgnj+oQBe16gbU88PoAiAuzzHtwHY43nGTPH5Gzx/VH3+blg51AWwjzF2gDF2E8DPAB6IcJ/CxQMAJnqOJwLoKCv/mTF2gzF2EMA+8P+Ta2CMrQRwTlFs63mJqAiAnIyxNYz/Un6QXRPV6Dy/HjH1/IyxFMbYJs/xZQC7ABRDJvn8DZ5fj4g8vxuEQzEAR2Xvj8H4H+lWGICFRPQ3ET3jKSvEGEsB+BcKQEFPeaz+T+w+bzHPsbLczbxARP941E6SWiVmn5+ISgO4E8A6ZMLPX/H8QBR9/m4QDlo6tFj0v23EGLsLQFsAzxPRPQZ1M8v/RELveWPt//ANgHIAagJIAfCZpzwmn5+IcgCYAeBlxtglo6oaZbH4/FH1+btBOBwDUEL2vjiAExHqS8hgjJ3wvJ4GMAtcTXTKs3SE5/W0p3qs/k/sPu8xz7Gy3JUwxk4xxtIZYxkA/gefqjDmnp+IEsEHximMsZme4kzz+Ws9f7R9/m4QDhsAlCeiMkSUBUAXAHMi3CdHIaJkIrpNOgbQCsB28Ofs7qnWHcCvnuM5ALoQUVYiKgOgPLhhyu3Yel6P6uEyEdX3eGl0k13jOqSB0cOD4N8BIMae39PX7wDsYoyNkJ3KFJ+/3vNH3ecfacu9Rev+feAW/f0A3op0f0LwfGXBvRG2AtghPSOAfACWANjrec0ru+Ytz/9jN1zgoaHxzD+BL51vgc+AegXyvABqe35E+wGMgmfXf7T/6Tz/JADbAPwDPiAUicXnB3A3uPrjHwBbPH/3ZZbP3+D5o+rzF+EzBAKBQKDCDWolgUAgEIQZIRwEAoFAoEIIB4FAIBCoEMJBIBAIBCqEcBAIBAKBCiEcBAKBQKBCCAeBQCAQqPh/vK0nliTtrE0AAAAASUVORK5CYII=\n", "text/plain": [ "
" ] @@ -1170,7 +1325,7 @@ }, { "cell_type": "code", - "execution_count": 91, + "execution_count": 45, "id": "cab7889f", "metadata": { "scrolled": true @@ -1180,8 +1335,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "gpu allocated : 1776 MB\n", - "gpu reserved : 1910 MB\n" + "gpu allocated : 2778 MB\n", + "gpu reserved : 3272 MB\n" ] } ], @@ -1193,7 +1348,7 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 46, "id": "29ffab84", "metadata": {}, "outputs": [], @@ -1203,7 +1358,7 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 21, "id": "4b9b9579", "metadata": {}, "outputs": [], @@ -1216,7 +1371,348 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 88, + "id": "80d0ee50", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 88, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model.load_state_dict(torch.load(\"model.zip\"))" + ] + }, + { + "cell_type": "code", + "execution_count": 89, + "id": "5c9b570c", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "MyModel(\n", + " (bert): BertModel(\n", + " (embeddings): BertEmbeddings(\n", + " (word_embeddings): Embedding(119547, 768, padding_idx=0)\n", + " (position_embeddings): Embedding(512, 768)\n", + " (token_type_embeddings): Embedding(2, 768)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (encoder): BertEncoder(\n", + " (layer): ModuleList(\n", + " (0): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (1): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (2): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (3): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (4): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (5): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (6): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (7): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (8): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (9): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (10): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (11): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " )\n", + " )\n", + " (pooler): BertPooler(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (activation): Tanh()\n", + " )\n", + " )\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " (lin1): Linear(in_features=768, out_features=768, bias=True)\n", + " (gelu): GELU()\n", + " (lin2): Linear(in_features=768, out_features=1, bias=True)\n", + ")" + ] + }, + "execution_count": 89, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "device = torch.device('cuda')\n", + "model.to(device)" + ] + }, + { + "cell_type": "code", + "execution_count": 38, "id": "fff7a7d0", "metadata": {}, "outputs": [ @@ -1224,7 +1720,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "100%|███████████████████████████████████████████████████████████████████████████| 3125/3125 [01:26<00:00, 36.25batch/s]\n" + "100%|███████████████████████████████████████████████████████████████████████████| 1250/1250 [00:34<00:00, 36.39batch/s]\n" ] } ], @@ -1232,7 +1728,7 @@ "model.eval()\n", "collect_list = []\n", "with torch.no_grad():\n", - " with tqdm(test_loader, unit=\"batch\") as tepoch:\n", + " with tqdm(dev_loader, unit=\"batch\") as tepoch:\n", " for batch_i,batch_l in tepoch:\n", " batch_inputs = {k: v.cuda(device) for k, v in list(batch_i.items())}\n", " batch_labels = batch_l.cuda(device)\n", @@ -1250,12 +1746,12 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": 23, "id": "4e9a90b5", "metadata": {}, "outputs": [], "source": [ - "def getConfusionMatrix(predict,actual,attention_mask):\n", + "def getConfusionMatrix(predict,actual):\n", " ret = torch.zeros((2,2),dtype=torch.long)\n", " for p_s,a_s in zip(predict,actual):\n", " ret[p_s,a_s] += 1\n", @@ -1264,7 +1760,7 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 39, "id": "b7a513c9", "metadata": {}, "outputs": [ @@ -1272,7 +1768,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "average_loss : 0.3252932393981423, average_accuracy : 0.86136, size :50000\n" + "average_loss : 0.33939967472716237, average_accuracy : 0.86565, size :20000\n" ] } ], @@ -1287,24 +1783,24 @@ " total_loss += batch_size * item[\"loss\"]\n", " total_accuracy += batch_size * item[\"accuracy\"]\n", " total_size += batch_size\n", - " confusion += getConfusionMatrix(item[\"predict\"],item[\"actual\"],item[\"attention_mask\"])\n", + " confusion += getConfusionMatrix(item[\"predict\"],item[\"actual\"])\n", "print(f\"\"\"average_loss : {total_loss/total_size}, average_accuracy : {total_accuracy/total_size}, size :{total_size}\"\"\")" ] }, { "cell_type": "code", - "execution_count": 45, + "execution_count": 38, "id": "1ac327de", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "tensor([[21382, 3487],\n", - " [ 3445, 21686]])" + "tensor([[8716, 1638],\n", + " [1194, 8452]])" ] }, - "execution_count": 45, + "execution_count": 38, "metadata": {}, "output_type": "execute_result" } @@ -1315,7 +1811,7 @@ }, { "cell_type": "code", - "execution_count": 46, + "execution_count": 122, "id": "3e71d4d2", "metadata": {}, "outputs": [], @@ -1333,7 +1829,7 @@ }, { "cell_type": "code", - "execution_count": 49, + "execution_count": 242, "id": "6756408c", "metadata": {}, "outputs": [ @@ -1341,7 +1837,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "f1 score : 0.862197756767273\n" + "f1 score : 0.7763830423355103\n" ] } ], @@ -1351,7 +1847,7 @@ }, { "cell_type": "code", - "execution_count": 50, + "execution_count": 86, "id": "f28f64e9", "metadata": {}, "outputs": [ @@ -1653,12 +2149,13 @@ " )\n", " )\n", " (dropout): Dropout(p=0.1, inplace=False)\n", - " (lin1): Linear(in_features=768, out_features=256, bias=True)\n", - " (lin2): Linear(in_features=256, out_features=1, bias=True)\n", + " (lin1): Linear(in_features=768, out_features=768, bias=True)\n", + " (gelu): GELU()\n", + " (lin2): Linear(in_features=768, out_features=1, bias=True)\n", ")" ] }, - "execution_count": 50, + "execution_count": 86, "metadata": {}, "output_type": "execute_result" } @@ -1677,21 +2174,17 @@ }, { "cell_type": "code", - "execution_count": 90, + "execution_count": 51, "id": "cc727fd9", "metadata": {}, "outputs": [ { - "ename": "KeyboardInterrupt", - "evalue": "Interrupted by user", - "output_type": "error", - "traceback": [ - "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[1;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", - "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_10708/4160447663.py\u001b[0m in \u001b[0;36m\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0msen\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 2\u001b[0m \u001b[0minputs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtokenizer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msen\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mreturn_tensors\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m'pt'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpadding\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'longest'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 3\u001b[0m \u001b[0moutput\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m**\u001b[0m\u001b[1;33m{\u001b[0m\u001b[0mk\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mv\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mto\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mk\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mv\u001b[0m \u001b[1;32min\u001b[0m \u001b[0minputs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m}\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 4\u001b[0m \u001b[0mprob\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msigmoid\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 5\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"긍정적 output :\"\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mprob\u001b[0m \u001b[1;33m*\u001b[0m \u001b[1;36m100\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m\"%\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;32m~\\anaconda3\\envs\\nn\\lib\\site-packages\\ipykernel\\kernelbase.py\u001b[0m in \u001b[0;36mraw_input\u001b[1;34m(self, prompt)\u001b[0m\n\u001b[0;32m 1008\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_parent_ident\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"shell\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1009\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_parent\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"shell\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1010\u001b[1;33m \u001b[0mpassword\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 1011\u001b[0m )\n\u001b[0;32m 1012\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;32m~\\anaconda3\\envs\\nn\\lib\\site-packages\\ipykernel\\kernelbase.py\u001b[0m in \u001b[0;36m_input_request\u001b[1;34m(self, prompt, ident, parent, password)\u001b[0m\n\u001b[0;32m 1049\u001b[0m \u001b[1;32mexcept\u001b[0m \u001b[0mKeyboardInterrupt\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1050\u001b[0m \u001b[1;31m# re-raise KeyboardInterrupt, to truncate traceback\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1051\u001b[1;33m \u001b[1;32mraise\u001b[0m \u001b[0mKeyboardInterrupt\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Interrupted by user\"\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 1052\u001b[0m \u001b[1;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1053\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlog\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwarning\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Invalid Message:\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mexc_info\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;31mKeyboardInterrupt\u001b[0m: Interrupted by user" + "name": "stdout", + "output_type": "stream", + "text": [ + "웹 gpu\n", + "긍정적 output : 54.185086488723755 %\n", + "부정적 output : 45.814913511276245 %\n" ] } ], @@ -1717,9 +2210,72 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 40, "id": "b40f071c", "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████████████████████████████████████| 1875/1875 [00:51<00:00, 36.45batch/s]\n" + ] + } + ], + "source": [ + "model.eval()\n", + "collect_list = []\n", + "with torch.no_grad():\n", + " with tqdm(test_loader, unit=\"batch\") as tepoch:\n", + " for batch_i,batch_l in tepoch:\n", + " batch_inputs = {k: v.cuda(device) for k, v in list(batch_i.items())}\n", + " batch_labels = batch_l.cuda(device)\n", + " output = model(**batch_inputs)\n", + " loss = BCELoss(output, batch_labels.double())\n", + " \n", + " prediction = (output > 0).to(device,dtype=torch.int64)\n", + " correct = (prediction == batch_labels).sum().item()\n", + " accuracy = correct / prediction.size()[0]\n", + " \n", + " collect_list.append({\"loss\":loss.item(),\"accuracy\":accuracy, \"batch_size\":batch_labels.size(0),\n", + " \"predict\":prediction.cpu(),\n", + " \"actual\":batch_labels.cpu()})" + ] + }, + { + "cell_type": "code", + "execution_count": 52, + "id": "871b72d4", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "average_loss : 0.3305340794156926, average_accuracy : 0.8672333333333333, size :30000\n" + ] + } + ], + "source": [ + "total_loss = 0\n", + "total_accuracy = 0\n", + "total_size = 0\n", + "confusion = torch.zeros((2,2),dtype=torch.long)\n", + "\n", + "for item in collect_list:\n", + " batch_size = item[\"batch_size\"]\n", + " total_loss += batch_size * item[\"loss\"]\n", + " total_accuracy += batch_size * item[\"accuracy\"]\n", + " total_size += batch_size\n", + " confusion += getConfusionMatrix(item[\"predict\"],item[\"actual\"])\n", + "print(f\"\"\"average_loss : {total_loss/total_size}, average_accuracy : {total_accuracy/total_size}, size :{total_size}\"\"\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "62333944", + "metadata": {}, "outputs": [], "source": [] } diff --git a/ndataset.py b/ndataset.py index a7469c0..23bf620 100644 --- a/ndataset.py +++ b/ndataset.py @@ -7,13 +7,15 @@ from ndata import readNsmcRawData, NsmcRawData def readNsmcDataAll(): """ - Returns: train, test + Returns: train, dev, test """ print("read train set", file=sys.stderr) train = readNsmcRawData("nsmc/nsmc-master/ratings_train.txt",use_tqdm=True,total=150_000) print("read test set", file=sys.stderr) - test = readNsmcRawData("nsmc/nsmc-master/ratings_test.txt",use_tqdm=True,total=50_000) - return NsmcDataset(train),NsmcDataset(test) + testBig = readNsmcRawData("nsmc/nsmc-master/ratings_test.txt",use_tqdm=True,total=50_000) + test = testBig[:30_000] + dev = testBig[30_000:] + return NsmcDataset(train),NsmcDataset(dev),NsmcDataset(test) class NsmcDataset(Dataset): def __init__(self, data: List[NsmcRawData]):