1116 lines
85 KiB
Plaintext
1116 lines
85 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 1,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"'C:\\\\Users\\\\Monoid\\\\anaconda3\\\\envs\\\\nn\\\\python.exe'"
|
|
]
|
|
},
|
|
"execution_count": 1,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"import sys\n",
|
|
"sys.executable"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"파이썬 환경 확인.\n",
|
|
"envs\\\\nn\\\\python.exe 으로 끝나기를 기대합니다"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 2,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from preprocessing import readPreporcssedDataAll\n",
|
|
"import torch\n",
|
|
"from torch.utils.data import Dataset, DataLoader\n",
|
|
"from dataset import make_collate_fn, DatasetArray\n",
|
|
"from transformers import BertTokenizer\n",
|
|
"import torch.nn as nn\n",
|
|
"from read_data import TagIdConverter"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 3,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"TAGS_PATH = \"eng_tags.json\"\n",
|
|
"DATASET_PATH = \"engpre\""
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"변수 설정"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 4,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"tagIdConverter = TagIdConverter(TAGS_PATH)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 5,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"PRETAINED_MODEL_NAME = 'bert-base-multilingual-cased'\n",
|
|
"tokenizer = BertTokenizer.from_pretrained(PRETAINED_MODEL_NAME)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Tokenizer 로딩"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 6,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from transformers import BertModel"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 7,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Some weights of the model checkpoint at bert-base-multilingual-cased were not used when initializing BertModel: ['cls.predictions.decoder.weight', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.bias', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.bias', 'cls.seq_relationship.weight']\n",
|
|
"- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
|
|
"- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"PRETAINED_MODEL_NAME = 'bert-base-multilingual-cased'\n",
|
|
"bert = BertModel.from_pretrained(PRETAINED_MODEL_NAME)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"버트 로딩"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 8,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"class MyModel(nn.Module):\n",
|
|
" def __init__(self,output_feat: int,bert):\n",
|
|
" super().__init__()\n",
|
|
" self.bert = bert\n",
|
|
" self.dropout = nn.Dropout(p=0.1)\n",
|
|
" self.lin = nn.Linear(768,output_feat) #[batch_size,word_size,768] -> [batch_size,word_size,output_feat]\n",
|
|
" self.softmax = nn.Softmax(2) #[batch_size,word_size,output_feat] -> [batch_size,word_size,output_feat]\n",
|
|
" #0부터 시작해서 2 번째 차원에 softmax.\n",
|
|
"\n",
|
|
" def forward(self,**kargs):\n",
|
|
" emb = self.bert(**kargs)\n",
|
|
" e = self.dropout(emb['last_hidden_state'])\n",
|
|
" w = self.lin(e)\n",
|
|
" return w"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 9,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"MyModel(\n",
|
|
" (bert): BertModel(\n",
|
|
" (embeddings): BertEmbeddings(\n",
|
|
" (word_embeddings): Embedding(119547, 768, padding_idx=0)\n",
|
|
" (position_embeddings): Embedding(512, 768)\n",
|
|
" (token_type_embeddings): Embedding(2, 768)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" (encoder): BertEncoder(\n",
|
|
" (layer): ModuleList(\n",
|
|
" (0): BertLayer(\n",
|
|
" (attention): BertAttention(\n",
|
|
" (self): BertSelfAttention(\n",
|
|
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" (output): BertSelfOutput(\n",
|
|
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (intermediate): BertIntermediate(\n",
|
|
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
|
|
" )\n",
|
|
" (output): BertOutput(\n",
|
|
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (1): BertLayer(\n",
|
|
" (attention): BertAttention(\n",
|
|
" (self): BertSelfAttention(\n",
|
|
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" (output): BertSelfOutput(\n",
|
|
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (intermediate): BertIntermediate(\n",
|
|
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
|
|
" )\n",
|
|
" (output): BertOutput(\n",
|
|
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (2): BertLayer(\n",
|
|
" (attention): BertAttention(\n",
|
|
" (self): BertSelfAttention(\n",
|
|
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" (output): BertSelfOutput(\n",
|
|
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (intermediate): BertIntermediate(\n",
|
|
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
|
|
" )\n",
|
|
" (output): BertOutput(\n",
|
|
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (3): BertLayer(\n",
|
|
" (attention): BertAttention(\n",
|
|
" (self): BertSelfAttention(\n",
|
|
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" (output): BertSelfOutput(\n",
|
|
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (intermediate): BertIntermediate(\n",
|
|
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
|
|
" )\n",
|
|
" (output): BertOutput(\n",
|
|
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (4): BertLayer(\n",
|
|
" (attention): BertAttention(\n",
|
|
" (self): BertSelfAttention(\n",
|
|
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" (output): BertSelfOutput(\n",
|
|
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (intermediate): BertIntermediate(\n",
|
|
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
|
|
" )\n",
|
|
" (output): BertOutput(\n",
|
|
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (5): BertLayer(\n",
|
|
" (attention): BertAttention(\n",
|
|
" (self): BertSelfAttention(\n",
|
|
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" (output): BertSelfOutput(\n",
|
|
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (intermediate): BertIntermediate(\n",
|
|
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
|
|
" )\n",
|
|
" (output): BertOutput(\n",
|
|
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (6): BertLayer(\n",
|
|
" (attention): BertAttention(\n",
|
|
" (self): BertSelfAttention(\n",
|
|
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" (output): BertSelfOutput(\n",
|
|
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (intermediate): BertIntermediate(\n",
|
|
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
|
|
" )\n",
|
|
" (output): BertOutput(\n",
|
|
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (7): BertLayer(\n",
|
|
" (attention): BertAttention(\n",
|
|
" (self): BertSelfAttention(\n",
|
|
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" (output): BertSelfOutput(\n",
|
|
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (intermediate): BertIntermediate(\n",
|
|
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
|
|
" )\n",
|
|
" (output): BertOutput(\n",
|
|
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (8): BertLayer(\n",
|
|
" (attention): BertAttention(\n",
|
|
" (self): BertSelfAttention(\n",
|
|
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" (output): BertSelfOutput(\n",
|
|
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (intermediate): BertIntermediate(\n",
|
|
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
|
|
" )\n",
|
|
" (output): BertOutput(\n",
|
|
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (9): BertLayer(\n",
|
|
" (attention): BertAttention(\n",
|
|
" (self): BertSelfAttention(\n",
|
|
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" (output): BertSelfOutput(\n",
|
|
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (intermediate): BertIntermediate(\n",
|
|
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
|
|
" )\n",
|
|
" (output): BertOutput(\n",
|
|
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (10): BertLayer(\n",
|
|
" (attention): BertAttention(\n",
|
|
" (self): BertSelfAttention(\n",
|
|
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" (output): BertSelfOutput(\n",
|
|
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (intermediate): BertIntermediate(\n",
|
|
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
|
|
" )\n",
|
|
" (output): BertOutput(\n",
|
|
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (11): BertLayer(\n",
|
|
" (attention): BertAttention(\n",
|
|
" (self): BertSelfAttention(\n",
|
|
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" (output): BertSelfOutput(\n",
|
|
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (intermediate): BertIntermediate(\n",
|
|
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
|
|
" )\n",
|
|
" (output): BertOutput(\n",
|
|
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (pooler): BertPooler(\n",
|
|
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (activation): Tanh()\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" (lin): Linear(in_features=768, out_features=10, bias=True)\n",
|
|
" (softmax): Softmax(dim=2)\n",
|
|
")\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"model = MyModel(tagIdConverter.size,bert)\n",
|
|
"model.cuda()\n",
|
|
"bert.cuda()\n",
|
|
"print(model)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"`tagIdConverter.size` 만큼의 종류가 있음"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 10,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"bert current device : cuda:0\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"print(\"bert current device :\",bert.device)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 11,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"device(type='cuda')"
|
|
]
|
|
},
|
|
"execution_count": 11,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"device = torch.device(\"cuda\")\n",
|
|
"device"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 12,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"datasetTrain, datasetDev, datasetTest = readPreporcssedDataAll(DATASET_PATH)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 15,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"my_collate_fn = make_collate_fn(tokenizer, tagIdConverter)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 16,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"BATCH_SIZE = 4\n",
|
|
"train_loader = DataLoader(\n",
|
|
" DatasetArray(datasetTrain),\n",
|
|
" batch_size=BATCH_SIZE,\n",
|
|
" shuffle=True,\n",
|
|
" collate_fn=my_collate_fn\n",
|
|
")\n",
|
|
"dev_loader = DataLoader(\n",
|
|
" DatasetArray(datasetDev),\n",
|
|
" batch_size=BATCH_SIZE,\n",
|
|
" shuffle=True,\n",
|
|
" collate_fn=my_collate_fn\n",
|
|
")\n",
|
|
"test_loader = DataLoader(\n",
|
|
" DatasetArray(datasetTest),\n",
|
|
" batch_size=BATCH_SIZE,\n",
|
|
" shuffle=True,\n",
|
|
" collate_fn=my_collate_fn\n",
|
|
")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 18,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from tqdm import tqdm"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 19,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"100%|████████████████████████████████████████████████████████████████████████| 14041/14041 [00:00<00:00, 468033.78it/s]"
|
|
]
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"212812/272841 = 0.7799854127495501\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"total_l = 0\n",
|
|
"total_o = 0\n",
|
|
"\n",
|
|
"for item in tqdm(datasetTrain):\n",
|
|
" entities = item[\"entity\"]\n",
|
|
" l = len(entities)\n",
|
|
" o = sum(map(lambda x: 1 if x == \"O\" else 0,entities))\n",
|
|
" total_l += l\n",
|
|
" total_o += o\n",
|
|
"\n",
|
|
"print(f\"{total_o}/{total_l} = {total_o/total_l}\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"O token 이 77%를 차지함."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 20,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from torch.optim import AdamW"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 21,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"optimizer = AdamW(model.parameters(), lr=1.0e-5)\n",
|
|
"CELoss = nn.CrossEntropyLoss(ignore_index=tagIdConverter.pad_id)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 22,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from groupby_index import groupby_index"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 23,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"epoch 0 start:\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Epoch 0: 100%|████████████████████████████████████| 3511/3511 [02:21<00:00, 24.82minibatch/s, accuracy=0.954, loss=1.2]\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"epoch 1 start:\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Epoch 1: 100%|██████████████████████████████████| 3511/3511 [02:20<00:00, 24.97minibatch/s, accuracy=0.986, loss=0.288]\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"epoch 2 start:\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Epoch 2: 100%|██████████████████████████████████| 3511/3511 [02:22<00:00, 24.65minibatch/s, accuracy=0.995, loss=0.192]\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"epoch 3 start:\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Epoch 3: 100%|███████████████████████████████████| 3511/3511 [02:25<00:00, 24.20minibatch/s, accuracy=0.99, loss=0.313]\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"epoch 4 start:\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Epoch 4: 100%|██████████████████████████████████| 3511/3511 [02:23<00:00, 24.47minibatch/s, accuracy=0.988, loss=0.345]\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"TRAIN_EPOCH = 5\n",
|
|
"\n",
|
|
"result = []\n",
|
|
"iteration = 0\n",
|
|
"\n",
|
|
"t = []\n",
|
|
"\n",
|
|
"model.zero_grad()\n",
|
|
"\n",
|
|
"for epoch in range(TRAIN_EPOCH):\n",
|
|
" model.train()\n",
|
|
" print(f\"epoch {epoch} start:\")\n",
|
|
" with tqdm(train_loader, unit=\"minibatch\") as tepoch:\n",
|
|
" tepoch.set_description(f\"Epoch {epoch}\")\n",
|
|
" \n",
|
|
" for batch in groupby_index(tepoch,8):\n",
|
|
" corrects = 0\n",
|
|
" totals = 0\n",
|
|
" losses = 0\n",
|
|
" \n",
|
|
" optimizer.zero_grad()\n",
|
|
" for mini_i,mini_l in batch:\n",
|
|
" batch_inputs = {k: v.cuda(device) for k, v in list(mini_i.items())}\n",
|
|
" batch_labels = mini_l.cuda(device)\n",
|
|
" attention_mask = batch_inputs[\"attention_mask\"]\n",
|
|
" \n",
|
|
" output = model(**batch_inputs)\n",
|
|
" loss = CELoss(output.view(-1, output.size(-1)), batch_labels.view(-1))\n",
|
|
" \n",
|
|
" prediction = output.view(-1, output.size(-1)).argmax(dim=-1)\n",
|
|
" corrects += ((prediction == batch_labels.view(-1)) * attention_mask.view(-1)).sum().item()\n",
|
|
" totals += attention_mask.view(-1).sum().item()\n",
|
|
" losses += loss.item()\n",
|
|
" loss.backward()\n",
|
|
"\n",
|
|
" optimizer.step()\n",
|
|
" accuracy = corrects / totals\n",
|
|
" result.append({\"iter\":iteration,\"loss\":losses,\"accuracy\":accuracy})\n",
|
|
" tepoch.set_postfix(loss=losses, accuracy= accuracy)\n",
|
|
" iteration += 1"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 42,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"del batch_inputs\n",
|
|
"del batch_labels\n",
|
|
"del loss\n",
|
|
"del optimizer"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 43,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"gpu allocated : 1355 MB\n",
|
|
"gpu reserved : 1470MB\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"torch.cuda.empty_cache()\n",
|
|
"print(f\"gpu allocated : {torch.cuda.memory_allocated() // 1024**2} MB\")\n",
|
|
"print(f\"gpu reserved : {torch.cuda.memory_reserved() // 1024 ** 2}MB\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 24,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"%matplotlib inline"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 25,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import numpy as np\n",
|
|
"import matplotlib.pyplot as plt"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 26,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAZEAAAD4CAYAAAAtrdtxAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAAsTAAALEwEAmpwYAAA8bUlEQVR4nO3dd5hU1fnA8e/LLktZqiKCgAoIiooiIqJGg0YUiYnYIpiosSEqsUQssQGW2CJWFFEQ9BchFlQCWAgWUHrvyIIIC0ivC+zu7L6/P87MTtnpW5l9P89zn7nl3HPP3J2dd845954rqooxxhiTjGoVXQBjjDGHLgsixhhjkmZBxBhjTNIsiBhjjEmaBRFjjDFJS6/oAoRTrVo1rVWrVkUXwxhjDhn79+9XVS33ikGlDCK1atUiJyenoothjDGHDBE5UBHHteYsY4wxSbMgYowxJmkWRIwxxiQtZp+IiIwALgW2qOrJ3nX/AY73JmkA7FLVDmH2XQvsBQoAj6p2KpVSG2OMqRTi6VgfCbwOvOdboarX+OZF5EVgd5T9z1fVbckW0BhjTOUVszlLVacAO8JtExEB/gSMLuVyGWOMKWUiMkJEtojIkgjbRUReFZEsEVkkIh1j5VnSPpFzgc2quirCdgW+FpG5ItInWkYi0kdE5ojIHI/HU8JiGWOMCWMk0D3K9kuANt6pD/BmrAxLep9Ib6LXQs5R1Y0i0hiYJCIrvDWbYlR1GDAMIDMzM6nx6Z/8/kk6N+vMxcddnMzuld+kSdCqFbRuDYWFsGIFbNoEv/tdyfNetQrWrfPn5XtEgEhwuk2bYPp0uOIKKCiAvDyoVQu++gratoWWLf3pZs2Cyy6DffsgM9PltX+/Sx+Y7+LFsHMnnHeeW96wAebNgz/8weWfkeG2Z2ZC9epu36+/dueiSROoU8edj5Ur3Tm59FK3nJHhXrduhZ9+gjVroGlT6NYNqnl/P33zjcujRQv46CM4+2yYMwd69YJ0779HYSF4PC6/ggIYNgwuuQSOPhpGjIC//AVq1oQDB9w+1av7z+F778FVV7kyi7j3Di7tzp3w3/9Cnz6wezd8/jlcf71Ll5Pj9q9Tx3+e3nwTXnkF/vpXd8xhw1w+2dnuXDz5JPz4I6SlwfLlcPXV7j3VrQudOkGzZu691akD7du7czB+vDvuKafAnXfCwIFw8CD06OH+Hjt2wIwZsHChO0crV0K/fvD9966c/fu79zBnjjsfGRnw29/Cu+/Crl3uczJ2LEyb5tJfdx00aODKM2QIfPstNG4MXbrAli0wYAA895wr45//DDNnunOxY4crkwg0bAiLFrl169bBE0+4c9C0qfs7HXGEO1d5edCokTvnCxa4srz0kvt8LV3qjpmX5z6zr77q3veYMe7z+Le/wfPPu3Kcey4MGgRZWVC7NuzZ485tz57ufU6YAI8+Cscc48779u2ubCNHwosvuvTXXefWf/SRy//aa915ql4dhg6FJUvgpJNg71544IES/jNHpqpTROTYKEkuA95T94yQGSLSQESaquqmaJnGnIBjgSUh69KBzUDzOPMYCPSPJ23t2rU1GbWfrq39v+qf1L5latUq1by8xPa59lrVzEw3v3ev6u7dqu5rpfi0bp1/vwMH3Lrf/MbNd+umOnSo6rx5qtu3hz/Wxo3+vHx8yxMnqv76q+rw4apdu/rXn3GGf37yZP98QYHb//jj3fKnn7rXZ55RnT3bzR92mCvf2rXBx7r9dtUePfzLw4YVf6+PPhq8D6j+8Y+Rz0206bvvYqe55Rb//HnnBW+rVi34vIFqy5aqP/+sevfd/m2NGwfvV7Nm5ON98YXq9df7lw8eVP3gA9WZM5N7jxUxBZ4zmxKfkgTkAnMCpj6q8X2fB2wbD/wmYHky0Clc2qI00TZGOyiuSvR9lH0ygboB89OA7vEcL9kgkvl0pv79y78ntW9C9u1THTzY/4Xp4/G4bYGys91pvvde/7rvvlO96SbVwsLw+e/f7/9A/fRT/B++996LneaSS9zrPfeorl6tet11xdNcdFHF/yPZ5KYrr6z4MthUvlOSgBzV5L7PA7ZNCBNETo+aXxwHHA1sAvKBbOBm7/qRQN+QtEcBE73zrYCF3mkp8Eg8b1A1+SBS95919d4v742dMBm7d6v26qV6wgmq7du7U/fpp8FpbrvNrV+7VvWJJ1TXrw/+cGzY4NK1aOGWv/rK1RBCNW3q32fgwIr/UNtk06EyHXVU6eQTWnsszylJpRRE3gJ6ByyvBJpGzS+eg5b3lGwQqfdMPb37i7uT2jemcH/sMWNUP/rIBY0NG4pvP/LI4uvGj1c97rjiHxrf/JNPVtyH16ZDa/rrXyu+DLGmv/zFNVOW5TE6dFBdsUJ1wQL3vzR4cPE0gc2k8Uyqqt98419+7TX//BlnqB5xRPx5tW2revjh8aV94okSfEWVShD5PfAFIEAXYFbM/OI5aHlPyQaRBs820Lsm3pXUvqqq+q9/qS5dGn5buD94t25l+89hU/lMZ55Z8WWIZxo9WvXBB/3LkT6Xvikjw+1z882x887J8c8HfmGCalpaYuWcN8/f3+XTu7d/+333ub6tkpyLvDzV3Nzg/sBACxeqvvSSP/24ceHzCdc/1b27P5+xY/19ibfeqvrWW/5tdepEL+PVV/vTHjyoWqNG+HS/+517/c1vEvq6Kv4VFTuIhGtZAvr6WpW8wWMIsBpYHKs/RL2fxEo3JRtEGj7bUPtN6JfUvpqf705HvXpueckS1Z073fzrr5fsA29T8tPJJ0feNmlS7P1nzIidprAw8rZdu9xro0aqtWu7+TZtVB97rHjawC/K0On77/3zoR304aaCguDlgQPdZzGwrKrhj9+9u3tt186lCe3ofvVV1SuuUL3wwuC8fF9yqqpHH+3mFy1yZRk/3i2/9prqaaf597vjDv/8DTe4L8RImjRx6UaNcvn69rviisjnIbB/MPRvFg9f+gULwufz97+rrlnj+hNvuSX+fFVV9+xx3xFDhrhWh88/D877r38tvs8nnwSnOXDArS8oKN7HmqB4ayKlPZX7AeOZkg0ihz93uN454c6k9i36JZae7pZBtWFD1R9+iP0PX9mmdu3KNv9ITQPxfDkmMtWsqbp1q3/Zd5ECqP7zn+7v9OyzwfusWeNe+wdcpRftgoM2bVyakSP96z780D+/f7/qL7+obtumunmzu9DBZ+tW/9Vw4P/chJtmzXKvHTu6dDt3uvMY7sKG0Lx69Qr+gglMs2+fq2n07u2Wd+92P4heftlfG/B9ufn6CyZMiJzX3r1ufvNm99n3KSxUnTvX/yWbkeH2y81Vbd7czUeqFQSerzvvdL/K58/3H3vIkPDNuIMGBQfN4cNdmf7xj+jHCeTbN/B44H4kZma6QFCaJkzwN9+98ELx7bm5/jJceWWpHtqCSMCUbBBp9HwjvX387Ynv+OWX/j9sRoZbV5ZfwmU9rV1bNpdZTpvmmiL27Am//fnn3cUGM2e6ZgDf+gED/PP9+8d/PN+vtMBlcMHK43HbQpspVFWXLXNfpIF8gaFDB9Ubb/Sn9wUjVfeFtWWLm2/Z0m0PzSecwGNv2aL6wAPF34vv8mZfEPHZtat4Debyy4Pzfe65yMeL18GDLsBed52bL0leqq7J6uGH3Tm76iqXx7Zt8e8/bZr/2MuWuXVr1qg+/rgW+4L1/b3eeSfxcq5Z465CXLfOf7wuXRLPJ1E//hi+ZhFYwxw8uFQPaUEkYEo2iDR+obH2/W/fxHcM/AeuWbP4ukNh8jXHBX4hrF2r2qpVYvn07Bl5m09eXvD6Cy5wry++WPzcbtrkXn3t7PPmuWaojz8Of4wlS1Rr1XLBKvTvU1BQvLnB9wPgkkui/419zQg9e7pf1OGCSKD1613giceAAe5XcqB4g4hq8aB8003BefgCZmDeDRvGV7ZYkg0igXJyVOfMSWyfX391xx05Mnj9Rx9psSCyapUL/okEqXBWr3YBNPR8ljffOZ87t5SzrZggUimfbJgsQSjUwsR2+uWXkEzE3Slb0e68093RG2/a9DB/ymOOcXdwp6f779D2eeQRd3dyerr/7vEff3R3bPuE3q3uk5YWvHz99e6u9KuvLp62SRN/GS+9FI491r+tQQN3V3OgjAx3V3s4oe8B4IIL3B3fDz8cfh8f3/kpKChe9nCaNw//fsIZODD88dq2hWXL3LLvnNWsWTxt6Pn03fEeafv06e7u8NLw00/ujv6SqF0bTj89sX2OPNJ9lYY67DD32qKFf91xx8H8+cmXz6dVq5LnUVqeeAI6xhyW6pCQUkGkmlRDCfPBjKZPyJBeBw7ABx+UXqESsXMnDB8Oo0fD66+74Tl69oTbboMLL4z8pda1a+Q8Q7+QfJo29X+x/vCDG14iMIAE2rfPDQ/hE/plfuqpbv9oRIIDCIT/EgkXKJYuDb8e3Pt7663oxwY4+WT3esUV7svdp1mz2PsmY/9+954XLHDl79DBBe7bbiuetnZteO01+PlnGDw4/A+CQF26lF4527RxU2Vx/vnwn//AH/9Y0SUpO+E+94ewlAoiIknURCL92i5LM2aE/yJo0ADuu89N4Madyspyv8o+/jhyfsl8ETZu7J8/5xw3hVq/3r1mZropktBfyvEqDPO3CrfuxBOTyz9Qq1ZubKQaNUqeVzx8wbtTJzcBPPVU5PT9+sH777v5pk3LtmyVmQj86U8VXQqTgJR6smE1qYYmGuWT/QKM5fzzI28788z482nd2jXxHHOMW37sseJpzjordj5nn+0GmfO56qrY+zRv7qZYkj2HvprPY4/BjTe6+bp1k8srHuUVQJL15z/D228HD8BXr17FlceYOKRWTQShkARrIqUdRO691wWJq692o4nuCPsolsSdc44bubR5czdaq8/Chf75LVvC/5IH198BbjTVrVtLXgPr3ds1u0Hs5pdIPvrIjR7csSPk5sI99/j7UKqiatXgllv8y0uXulFojanEUiqIlGlN5MEH3RDVd90V/Ive5/HHXZratf3ropWlenXIz4fZs13nclZW7DK0b++GxfYpKAjuKzjiiNh5jI42cn8CPvjANc+MGJF8m3rduv7OxRo13HDk5WX+fNcHVZmVRjOeMWUspYJIwn0id90Fn30WX9pnnnHPTjjsMNcU9PLL7jkJPrVrBwcQiB5ElixxAcTXXn7hhfGVIyPDPx+ps7m8tGoVvZ2/MuvQoaJLYExKSL0+kXivzrriCndFTLxEXNNCtWruITWhl4qGE9q0FHilVNu2yV1KnGzTkTHGlIGUCiIJ3Sfy6aclO5gviBx/vPfgYfoYfE/5mz3bXbq7Y4d7yllJ+I5TlfsOjDGVRkoFkbj7RCJ1Pifi9tvd65gx7lGpgR2iPl995S7N7dQJbrrJPWqzNK4+mjjRBSZjjKlgKdU2EnefyMqVJT9Yjx7+Po+JE8OnOfJIuPLKkh8r1CWXlH6exhiThNSricTTJ2JXvRhjTKlIqSCS1NhZ4QwYUPI8jDGmCkipIJLUfSLhDBwY313gxhhzCBGR7iKyUkSyROShMNsbisinIrJIRGaJyMmx8owZRERkhIhsEZElAesGisgGEVngnXokU+DSltTYWZHEcwmvMcYcIkQkDffo20uAE4HeIhLatv8wsEBVTwGuB16JlW88NZGRQPcw619S1Q7eqVjPcpwFLlVx9YncfHN8mfXv75/v1i35QhljTOXQGchS1TWqmgeMAS4LSXMiMBlAVVcAx4rIkdEyjRlEVHUKkMwAUPEUuFSF7RPZvTt4qJARI4rvePPNxe8uv/pqt66gwF2qa4wxlVu6iMwJmEKec0EzYH3AcrZ3XaCFwBUAItIZOAaIOgprSS7x7Sci1wNzgPtUNXQgonAFTmD42sSF7RNp0MCNKbVlS+wMZs0Kfm4GVPzQIsYYEx+PqnaKsj3cqKuhTTfPAq+IyAJgMTAf8EQ7aLLfkG8CrYEOwCbgxTBp4imwP7FIH18E9XiiljmiiH0iW7fC4sXuCWnRnHFG+OdqGGPMoS8bCHhkJM2BjYEJVHWPqt6oqh1wfSJHAD9HyzSpIKKqm1W1QFULgbdxTVcJFzgkz2Gq2klVO6UnOT5U1D6RJ5+E1auTytcYY1LAbKCNiLQUkQygFzAuMIGINPBuA7gFmKKqUcdqSiqIiEjgo9cuB5aESRazwKUt6n0ipTHUiTHGHKJU1QP0A74ClgMfqupSEekrIn29ydoBS0VkBe6iqLtj5RvzJ7+IjAa6Ao1EJBsYAHQVkQ645qm1wG3etEcB76hqD1X1iIivwGnACFVdGv9bTlzU+0QWLAhevvtumDDBPcejIh6Ra4wx5cx7Je3EkHVDA+anAwk9IChmEFHV3mFWD4+QdiPQI2C5WIHLUtT7RAKbsmrWdM8DOekk6NMn+nM/jDHGRJRSlx7FPXbWF1+UfWGMMaYKSKkgEvfYWQ0auNeLL3avfUIvpzbGGBOPlBoKPuGxs44+2pqyjDGmBFKrJiJC7X25/jvUZ84Mn9CGgjfGmFKRUkGkmlRjfL9p/iDRpUvxROefDxkZxdcbY4xJWEoFEfHdJL9uXeRE1auXT2GMMaYKSKkgUk3ieDv16pV9QYwxpopIqSAi8dw0eNddZV8QY4ypIlIqiATVRF6J8CyVc88tn8IYY0wVkFJBRAIHDr7nngorhzHGVBUpFUQO35UXO5ExxphSk1JB5KEhC6MnOO+88imIMcZUESkVROrk5EdP8N575VMQY4ypIlIqiMS8Nssu7zXGmFKVUkGk1sGC6AkaNiyfghhjTCUkIt1FZKWIZInIQ2G21xeR/4rIQhFZKiI3xswzoQELy0lmZqbm+Ma/SkSs+0Qq4Xs1xpjSICL7VTUzyvY04CegG+7x5bOB3qq6LCDNw0B9VX1QRI4AVgJNVDXiVUspVRMxxhgTUWcgS1XXeIPCGOCykDQK1BV353YdYAfgiZapBRFjjEkN6SIyJ2AKfVBSM2B9wHK2d12g13HPWd8ILAbuVo3+kKaUep6IMcZUYR5V7RRle7j2/tA2/ouBBcAFQGtgkohMVdU9kTJNqZrI4vaNi688//zyL4gxxlQ+2UCLgOXmuBpHoBuBsepkAT8DJ0TLNGYQEZERIrJFRJYErHtBRFaIyCIR+VREGkTYd62ILBaRBSIyJ9axSupgzTDDvNetW9aHNcaYQ8FsoI2ItBSRDKAXMC4kzTrgdwAiciRwPLAmWqbx1ERGAt1D1k0CTlbVU3C9/f+Isv/5qtohRjWrVIS9NsseQGWMMaiqB+gHfAUsBz5U1aUi0ldE+nqTPQmcLSKLgcnAg6q6LVq+MftEVHWKiBwbsu7rgMUZwFVxv5MyJOHCiAURY4wBQFUnAhND1g0NmN8IXJRInqXRJ3IT8EWEbQp8LSJzw1wpEERE+viuKvB4ol5RlhgLIsYYU2ZKdHWWiDyCu4b43xGSnKOqG0WkMa6Xf4WqTgmXUFWHAcPA3WyYVHnCrczIgGrV4MILk8nSGGNMFEkHERG5AbgU+J1GuO3dWzVCVbeIyKe4m13CBpHSEBRE0tKgoAA6dnSvxhhjSl1SzVki0h14EPijqu6PkCZTROr65nHtbEvCpS09AWHk4oth/nzoE7UVzRhjTAnEc4nvaGA6cLyIZIvIzbi7GuvimqgWiMhQb9qjRMTXaXMk8IOILARmARNU9csyeRe+sgbWh0SgQ4fY42kZY4xJWjxXZ/UOs3p4hLQbgR7e+TXAqSUqXYKC4kWdOuV5aGOMqZJS6o51UVh8VDo89hgMGVLRxTHGmJSXYmNnCYXVgCeeqOiCGGNMlZBaNRHskSHGGFOeUi+IWD+6McaUm5QLIsVHNjbGGFNWUiqIgBD16SnGGGNKVUoFEbFKiDHGlKvUCiICas1ZxhhTblIriKj1iBhjTHlKqSACYldnGWNMOUqpICJgN4oYY0wEItJdRFaKSJaIPBRm+/3e8RAXiMgSESkQkcOi5ZlyQcRqIsYYU5yIpAFDgEuAE4HeInJiYBpVfcH7OPMOuMeef6+qO6Llm1JBBOsTMcaYSDoDWaq6RlXzgDHAZVHS9wZGx8o0pYKIuzrLGGOqpHTfI8a9U+jDlJoB6wOWs73rihGR2kB34JOYB022tJWRu0/EwogxpkryqGqnKNvDNfZH+sL8A/BjrKYsSLGaiN2xbowxEWUDLQKWmwMbI6TtRRxNWZBiQcT61I0xJqLZQBsRaSkiGbhAMS40kYjUB34LfB5PpqnVnIWrm6kqYo/FNcaYIqrqEZF+wFdAGjBCVZeKSF/v9qHepJcDX6tqTjz5ilbC+yoyMzM1Jyeu8gdZ07EVG7f9zNm/FFBNUqqSZYwxUYnIflXNLO/jptQ3ra/yURkDozHGpKKYQURERojIFhFZErDuMBGZJCKrvK8NI+wb9e7I0ueGPSlU6143xpjyEE9NZCTueuFADwGTVbUNMNm7HCSeuyNLm28oeBvJ1xhjykfMIKKqU4DQa4UvA0Z550cBPcPsmujdkaVCsZqIMcaUl2T7RI5U1U0A3tfGYdLEfXckgIj08d1p6fF4kiqU73os6xMxxpjyUZYd64ncHYmqDlPVTqraKT09uSuPfQMwWk3EGGPKR7JBZLOINAXwvm4JkyaRuyNLRVFNxPpEjDGmXCQbRMYBN3jnbyD8nY1x3R1Z2qxPxBhjyk88l/iOBqYDx4tItojcDDwLdBORVUA37zIicpSITAR3dyTguztyOfChqi4tm7fhLavv6izrEzHGmHIRs/NBVXtH2PS7MGk3Aj0ClicCE5MuXcLsPhFjjClPqXXHurcvxPpEjDGmfKRUEPGFEauJGGNM+UipICLevhDrEzHGmPKRUkHE+kSMMaZ8pVQQ8d0nYkHEGGPKR0oFkfRq6Siw/cD2ii6KMcZUOvGMrC4iXUVkgYgsFZHvY+WZUk82rC7u7ew4EPPZ8sYYU6UEjKzeDTeiyGwRGaeqywLSNADeALqr6joRCTcuYpCUqomIuLGzrGPdGGOKiWdk9WuBsaq6DkBVww1pFSSlgojv9hDrEzHGmGLiGVm9LdBQRL4Tkbkicn2sTFMqiAgujgyfP5ycvMSf0W6MMYewdN/jNLxTn5Dt8Yysng6cDvweuBh4TETaRj1o0sWtjLyn49+L/03t6rUZ9odhFVseY4wpPx5V7RRlezwjq2cD21Q1B8gRkSnAqcBPkTJNrZqIt08EYNO+TRVbGGOMqVziGVn9c+BcEUkXkdrAmbgBdCNKqZqIBFTMrF/EGGP8VNUjIr6R1dOAEaq6VET6ercPVdXlIvIlsAgoBN5R1SXR8k2pIOIfgtGu0DLGmFDhRlZX1aEhyy8AL8SbZ2o1ZwX0EdlIvsYYU/ZSKoj4xs4Cq4kYY0x5SLEg4md9IsYYU/ZSKogEdqxbc5YxxpS9lAoi4L9zxmoixhhT9pIOIiJyvHekR9+0R0TuCUnTVUR2B6R5vMQljlamgHnrEzHGmLKX9CW+qroS6ABFo0NuAD4Nk3Sqql6a7HESLpc3klhNxBhjyl5pNWf9Dlitqr+UUn5JsT4RY4wpX6UVRHoBoyNsO0tEForIFyJyUqQMRKSPb+Awj8eTdEF8oSNrR1bSeRhjjIlPiYOIdwyWPwIfhdk8DzhGVU8FXgM+i5SPqg5T1U6q2ik9PblWtsCbDfMK8pLKwxhjTPxKoyZyCTBPVTeHblDVPaq6zzs/EaguIo1K4ZhhiVQr6hPJL8gvq8MYY4zxKo0g0psITVki0kRExDvf2Xu8MnsAelrA2/EUJt8kZowxJj4lGoDRO1RwN+C2gHVFI0ICVwG3i4gHOAD00jK89raaVCtq0MrJt4dSGWNMWStREFHV/cDhIeuGBsy/DrxekmMkQkLi066Du2hQs0F5Hd4YY6qc1LtjPeCOQ+tcN8aYspVaQSSkJpLrya2gghhjTNWQWkGE4KfOH/AcqLByGGNMZSMi3UVkpYhkichDYbYnPFRVaj3ZMKQmciDfgogxxkDR8FRDcBdDZQOzRWScqi4LSZrQUFWpVxMJ6BM56DlYcQUxxpjKpTOQpaprVDUPGANcVtJMUyuIhNZErDnLGFN1pPuGjvJOfUK2NwPWByxne9eFimuoqqKDlqDAlVJQn4g1Zxljqg6PqnaKsl3CrAu9b883VNU+EemBG6qqTbSDpnRNZOv+rRVUEGOMqXSygRYBy82BjYEJkhmqKrWCCMF9Ihv2bKi4ghhjTOUyG2gjIi29A+f2AsYFJkhmqKrUas4KvU+kwO4TMcYYAFX1iEg/4CsgDRihqktLOlSVVMbHyGZmZmpOThJjX518Mh8XLuXqa/yrFty2gFObnFp6hTPGmEpIRParamZ5Hze1mrO8AfGI2kcUrXp15qvM3zSftbvWVlChjDEmdaVWEAEuPf4PZN3lf6rh3ry9dBzWkZavtKzAUhljTGpKrT4RoGZ6DWrWqFe0XKAFFVgaY4xJbalVE6mE/TvGGJPKUiuIAEjw/TSV8cIBY4xJFakVRMIEjE9XfFoBBTHGmKohtYIIFKuJGGOMKTupFUSs6coYY8pVagURsJqIMcaUoxIFERFZKyKLvU/AmhNmu4jIq96naC0SkY4lOV5MMWoi9rhcY4wpXaVREzlfVTtEGIL4Etwwwm2APsCbpXC86KLURM5999wyP7wxxlQlZd2cdRnwnjozgAYi0rTMjhajJjJ74+wyO7QxxlRFJQ0iCnwtInPDPEUL4n+SFiLSx/dELo/Hk3yJvDWRj67+KPk8jDHGxKWkw56co6obRaQxMElEVqjqlIDt8TxJy61UHQYMAzeKb1KlCaiJXHXiVUllYYwxJn4lqomo6kbv6xbgU9yD4APFfJJWqbOrs4wxptwkHUREJFNE6vrmgYuAJSHJxgHXe6/S6gLsVtVNSZc2FrtPxBhjylVJaiJHAj+IyEJgFjBBVb8Ukb6+J2UBE4E1QBbwNnBHiUobD6uJGGNMWCLSXURWem+7eChKujNEpEBEYvYLJN0noqprgGKPDPQ+YtE3r8CdyR4jiUKV26GMMeZQIiJpwBCgG66rYbaIjFPVZWHSPYd7jG5MKX3H+hNdn6jAghhjTKXSGchS1TWqmgeMwd2GEepvwCfAlngyTa0gElITeey3jxVLsmjzovIqjTHGlKd0320S3in0touYt1yISDPgcmAocUqtIAIx+0RemPZCORXEGGPKlUdVOwVMw0K2x3PLxcvAg6rxPxI2tR6PG0efSH5BfjkUxBhjKp14brnoBIwR92O8EdBDRDyq+lmkTFMriEDMmkh+oQURY0yVNBtoIyItgQ1AL+DawASq2tI3LyIjgfHRAgikWhCJsyaSV5BHRlpGORTIGGMqB1X1iEg/3FVXacAIVV3quyUj8MraREhlfAZ5Zmam5uTkJL7jscdC164wcmTRKhkUvmay8e8baVq37MaCNMaY8iQi+1U1s7yPm1od6wkExLW71pZdOYwxpopIrSACcd+xruHHgTTGGJOA1Asicdq0t+yG8DLGmKoitYJIAs1ZV33khoTZm7u3rEpjjDEpL7WCCCQ0AOOM7BnUe7YeHy/7uAwLZIwxqSu1gkiCV5qdNfwsAL5e/XVZlMYYY1JeagURKFYTGfunsZxy5CkVVBhjjEltqRVEwtRELm93Oee0OCfqbhJ2SBljjDGxpFYQgbB9IpXxhkpjjEkFqRVEIgSL0486HYCnL3g67HaxpyEaY0xSUiuIQNiayM2n3cyKO1fw8LkPh93l7XlvMzN7ZlmXzBhjUk5qBZEINRER4fhGx0fcrVAL6TK8C6u2ryqrkhljTEpKOoiISAsR+VZElovIUhG5O0yariKyW0QWeKfHS1bcuAqW9K7bD2wvxYIYY0zqK8lQ8B7gPlWdJyJ1gbkiMin0oe/AVFW9tATHiV8JO9CtA94YYxKTdE1EVTep6jzv/F5gOSHP660QJaiJFGphKRbEGGNSX6n0iYjIscBpQLje6bNEZKGIfCEiJ5XG8SKKoyax/t71EbdN+WUKk9dMDlp31vCz+Mf//lHiohljTEUTke4islJEskTkoTDbLxORRd7uhzki8puYeZa0CUdE6gDfA0+r6tiQbfWAQlXdJyI9gFdUtU2EfPoAfQAyMjJOz83NTbwwTZpAz54wNPoDuiI9qMpHB/jPiS9t4DpjjKlsYj2USkTSgJ+Abrjnrc8Gegd2QXi/z3NUVUXkFOBDVT0h2nFLVBMRkerAJ8C/QwMIgKruUdV93vmJQHURaRQuL1UdpqqdVLVTenqSXTXWp2GMMZF0BrJUdY2q5gFjgMsCE6jqPvXXLDIh9oOXSnJ1lgDDgeWqOjhCmibedIhIZ+/xyvYSKLtx0BhTNaV7m6B8U5+Q7c2AwPb8bML0Y4vI5SKyApgA3BTroCWpiZwDXAdcEHAJbw8R6et78DtwFbBERBYCrwK9tCwvgYoz650P7oy6fV/ePmZtmMXZw8+Omdf63es5fdjpbN63OWq68949jw8WfxBX+YwxJgkeX2uOdxoWsj3cL+xiX5qq+qm3Casn8GSsgyZ9ia+q/hChUIFpXgdeT/YYSYmjJtKgZoOo26evn84D/3uABb8uiJnXqzNfZd6meby38D3uP+f+iOmmrpvK1HVTubb9tTHzNMaYMpANtAhYbg5sjJRYVaeISGsRaaSq2yKlK8l9IpVPKVVyLvq/i0olH2OMqURmA21EpCWwAegFBP2qFZHjgNXejvWOQAYxuiBSa9gTiLtP5IVuLzD099Gv4orHv6b/CwCN0v8UrgVvX94+snZkxcx/T+4e8gvyky+gMcYAquoB+gFf4e7r+1BVl4Z0QVyJ64JYAAwBronVBZFaQSSBmkj/s/vzp5P+VGqHHjY3tPkR8gvyWfDrAvbl7Su27cL3LqTNa+5q539O/Sfd/6972HzrP1uf3p/0LrVyGmOqLlWdqKptVbW1qj7tXTdUVYd6559T1ZNUtYOqnuXttogqtYIIJHR1VsNaDWnZoGVSh9m8bzMPTHqgaHn1ztXF0tw2/jZOe+s0Og7rWGzbzA3++zIf+eYRvlr9VcRjfbL8k+B9s2cig4Q1O9ckU3QDzN80n29+/qaii2HMIa/K94lMuXEKLV5qETPdhJ8m8OP6HznvmPNoWqcpT0x5grHLi90aE+TzlZ8DRG22evSbRxMrMNBleBcApq2fRquGrRLev7TkF+Szbvc6Wh/WusLKkCxfYLebSI0pmdQKIpDwfSLN6zWPK92lo90Yks/88AwAF7e+OOY+GWkZMdM8PTX8g7Ig9lhedTPqxsw/1NyNcznoOcg5R0d/ZHA87vnyHt6Y8wab+2+mcWbjEuWlquQW5FIzvWaJy2WMKT+p15xVTsI1Pz0z9Rn25u7l1nG3krUji+rVqhdLE6v2EqigsKBoPicvp9i6ZJ7I2OntTvzm3ZjD4cTlfz//D4AdB3aUOK9nf3iWWk/XYueB4vfw7Mvbx6LNi0p8DB8brbly8BR67G+RAlIriFTwB/Lhbx7m5Rkv8878d2jzWhvW7yk+2OOVH14Zd36eQk/RfJ1n6vBl1pfc8+U9ResuG3MZy7cuL1q+5uNr6PHvHskVPglpkgYEB7Zkvb/ofQA27dtUbNvl/7mcU4eeGvUqtRXbVvBVVuR+pUBLtixJrpCm1GTvyab6k9V5e97bFV0UU0KpFUQgqWFPZtw8I2h5wG8HJH34x7+L/dytWANA+rw689Wg5Uv+fQmvzw6+d/PEN04smv9w6Yd8kfVFXHkH+mXXL3y39ruE96sm7uNToAXsOrgr4f0DpVWLHJD+t8bVeKI177Ub0o7u/w5/hVuowOBcGRRqIReMuoAvViX+tztU+foJbRSHQ19qBZEkayJnNj8zaIj4gV0HllKBEjd2+Vg+WPwBoxeP5qHJxUZqDmvWhlkJH2fltpVF8+2GtOP8UecDsGjzImSQsPDXhVH3l0HC0q1LAXd5c8PnGrJi24qi7dv3b4+rhjJqwShavdKqqHYQ7Qu+QEte44nXqu2ryq2pJScvh2/XfsvVH11dLserTKLdX2UODakVRCDpARgb1mwI+JtoKsLOAzu58sMr+fPYP3Pt2PiHR1m7a23MNB3f6sig7wYVLU/+2f/clAOeA4BrEvL12QT23Wzbv431uyM/h2XiqomAv5koJy+HRi804t6v7o24z4SfJnDMy8fw18//ys+7fi5aHy1QlEazWahwgWJG9gzavt6WoXOSvxn1l12/sGxr6EM+w/P1beXk5yCDpKj/p8s7XZBBEnfN1cR275f38vZca0IrTakVRErwyzG9mrtQzffLKOtvWXRu1rlUihWvw54/LKn9rvn4mphfePN/nc/A7wcWLR/IP8CGPRuC0rQb0o7h84cD8MSUJ7hsjBsl+qgXj+Lol48uShf6xes7d7ke9wyYg56DALw267WieXCXBO84sIO35rxFn/F9WLd7XbFyRgsU/b/uH/U9Btq2f1vEPpTACxI+XvZxse2rd7h7fn5c/2Ncx3r+x+eZkR3cJHrsK8dy0hvxPYMttJnu3fnvAsH3EkVzz5f3BN2zdCgQ77B7uw/uLtPjrNm5JugS+5dnvkyf8aGD25atPbl7+Pbnb4uW277WFhkkUX+YHUpSK4hA0jWR6mnuSqoLW10IQOvDWjPtpmnMvCW+f+SKdvuE24ute3f+u6zavips+v6T+tP8peaMWzkuaH32nuyi+XErxzHg2wHkF7ov49kbZgMULfv4lnMLXBAJ/FL8evXXgOuvyXgqg7OHn03fCX3ZuDf8uG9dhnfh6JeO5p1573Duu+dy7MvHFm0bNq/4qADhqCpHvHAEN3x2AwD9JvbjtyN/y5qda4p9YYfew6OqvLvAfYnH23fy4P8e5KzhZ8WVNpzQwJloE88rM1/hhWkvJH38irRw80JmZkf/H/MUevhsxWdJNS+2frV10cgQFeXaT67lgvcuYEvOFgBW7XD/k6E3ER+qUiuIlKAmUk2qsfzO5Yz9k78ZJ61aWrnXRkqLqnLTuJto+3rbqM0hvtpGJE9MeaJovvM77lzkFeQFpfE1p+UV5KGq/Pen/xZt86X1/eJfuX0lsazfs55b/3srP6z7gV92/xIzfaDJaybT4LkGAIxeMhqAIbOHMOWXKbR+tTX/mvavoPShl0lP/nlyUVNfPEEkNAD8d+V/g8731R9dHVQbC5tHSBNesn0xS7YsYcT8EUntG4un0MOQWUOKand5BXlFX4rJCDzvczbOiZr2n1P/yeX/uZzxP41P+njhlEXzaDiBzbypeElzagURKNFDqU5odAKZGcWfLjm3z9ySlKjcnTX8LB76X3yd8skIDSI++QX5/HPqP7l53M1F60bMHxExfTLC1WACa1v3T7qfPbl7Iu4/dd3UsOt3H9zNos2L2Ju7t2jdhFUTgtIUaiF3fXEXP23/iVXbV3HHhDvYnBP8HJk/jvlj0PLHyz6m1tO1gtYVFBYUNeN8v/Z7vl/7fdB2T6Gn6L6gRLR/s33QuQ/10vSXkEESVNsMJ78gn7kbgz/zb815i35f9OOVma8AcOPnN3Lkv46MeUNsJBLwFIlCLSwKtBv3bkQGSdEVeeD/kbJ1/9akjhXOtPXTSH8ynUmrJ5Uon417N/LewveKlgdPH8zA7wYGpfE19xZoQdD5KkkQrkxSK4iUUZTv2LT42FeV2YzsGTw/7fkyyVsGCYc/f3jYbfmF+Tz6bfAwLl9kfUG7Ie2S/rIJ1WxwM+ZunEtBYQG/7vuVuRvn0vb1tkXb5/86Pyj9pR9cGrQ8ff30oGXfl1mD5xpw6tBTg5qSDnoOBtVGnpryFK/Neo1Th55K29fb8uacN2k2uNiD4cI6Z8Q59B3ftyj4NHiuAYs2L6LrqK5c9dFVQWnfmPMG54yIPKJA57c7U/Op6Hf2b9izoajWkOvJ5d357/L3r/8OwDc/f0ODZxsw9RcXUOdtmoeqsuvgLnYe2Mmj3zxKp7c7Bd1P47uh1NdPNHqxq+UF9jt1/7/ucV8EEFgTufvLu6n1dC3emP0GF73vHsMQ2MdXGldwhf6w8J1f3w+FIbOGIIMkocu/7/riLpoNbsYNn91QdJPsfV/fx6DvBwWl812+7in0BNU6faNfHOqq/LAnpvTc9/V9Ydev2bmmVAeL7PR2p7jThtYmth/YHvTr86HJD/HYt48VLYfeDDpv0zw6NOlARloGA75z9w9Fap6K1mw1bf00pq2fxvTs6UVXX/20/aeI6RdujnyJ9eyNrm/KU+gJe8zv135P11FdufyEyxl7zVienvo0T055Mmj77tzdPD31ae48eCd/HPNH3r3sXW78/EbA3y/Y/s32dGnehVYNW3HC4ScA8NmKz/hl1y9FX+yeQg9vzngz6Eq8tbvWMmL+CAZ1HRTXqAq+vO6ceGfRuk+Wf8J5757Hte2vZeSCkYAL+E98/wQDvhsQ15hngVct7svbR70a9YqlyS/IZ9v+bfT7oh/gLjipWyP8cEK7Du6iZnrNoqF5Xpv1WtG24fOH07ph+DHkfFd87s/fH1TTDbTw14XUr1mfYxscG/N9VTapFUTKsL3xmPrHcNNpNxV9kQDkPJzDgl8X0K6Ru6pp1MJRSd0N3aFJh7ieomhKxwP/C76SKfRCgUBnvnMmAC9e9GLMfEObrcIJHNYlkftCZJDwzh/e4fyW5xet25O7J2yt0DfO26crPuXJ75/k132/Bm3PK3TNi9WkGv0nuSveAoeV8TW/gKvVhrvyzCe/ML9YP1O397uRtSOL8489n5Man0TjzMYMmzuM28bfxr+6/Yt7utwT1JwVydR1U4NGMLhpnP9x3/kF+aRXS48YpP4y9i/8e/G/i5Zz8nK4fXzxi0/yCvI44oUjipY/XvYx1dOq85dT/lIsbcPnGtK+cXsW3b6oKLD53D8p+KmmG/Zs4Ki6RyEiRTWR04edXizPpi825ZqTrilqJvzwqg+5+qRD634hqYwdPZmZmZqTk3ibMHXrQp8+8GLsf/iS2HlgJzsP7iw2gu62/dtYvHkxbQ5vw/sL3+eUI08p+of26daqG5PWBLfD6gDlhs9u4JTGp9C3U196/qdnUJuwMeF8c/03XPDeBaWS13GHHVd0pdrJjU+O+8fQmCvH0OuTXlHT5D2aR8ZT/sFInzr/KXILcoNqRyUx+frJHFbrMPbn7+fsFmdz7SfXFl1U4TOq56iiq/UCNa/XPGwfkQ5QXp7xMvd+dS+5j+aSkZZR1FTnecxD+pOxf3+3PbwtK+5cQcdhHRP6kZjsyNIisl9Vi3fqlrHUCiJ16sBtt5V5EImXqtL/6/4MnjGYExqdQI20GozsOZLT3joNgNV3rUZVww6l7vvARvrwG2OKW3L7Ek5+8+QS57P7od3Uf7Z+0fIV7a5IaPBUn3u73MuohaMSGqS0LIOIiHQHXgHSgHdU9dmQ7X8GHvQu7gNuV9Wow1eUqGNdRLqLyEoRyRKRYpcDifOqd/si7zN7y1Yl6hMREe4/x1Vz/3Tin1jQdwEdmnRgSI8h1KtRj1YNW8V8Fsd1p1zHmCvHlEn5rjnpmjLJ15iKUhoBBAgKIJDY6NuBXprxUqmMcl0aRCQN98jbS4ATgd4icmJIsp+B36rqKcCTQMybs5IOInEW6BKgjXfqA7yZ7PHiUglrVU3qNGFL/y0M6OrvS7njjDvY/VD0O3WX3bGMSddNQkS45uRr8DzmrhoRhH3/2IcO0KB225m3zGTFnSuCOg8nXjuRp85/ilm3zOL6U6/nynZXMvT3QxnXaxxZf8tizFVj0AEa1Cw3pMcQZt86u2i5R5seQcuJuqPTHRG3fd7rc96//P2I289ucXbY9e0bt0+6PGXl9k7+9vbAPoWSCtc2b0ySOgNZqrpGVfOAMUDQjWKqOk1VfR13M4CYD1xKujlLRM4CBqrqxd7lf3gL8UxAmreA71R1tHd5JdBVVYuP9x0g6easzEy44w544dC8ezeWGdkzaFGvBc3qRb+sdG/u3ohXmESyce9Gft33a9HlzOt3r2f7ge10aNIBgB/X/cgBzwFmbZjFI988Qv5j+YxaMIo2h7fhpCNOYsySMXQ/rjutD2tNXkEeE1dNZPD0wfzv+v9x2/jbePy8x9mXt4+rPrqKtbvWcnrT05l28zTADTNSM70mj377KD2P78mEVRN4e97bFD5eyLsL3mX4/OGcfMTJLN26lDFXjaF5vebcOeFOzmh2Bs3rNeeg5yB/GP2HhN7v0xc8zcPnPhzzktR+Z/QrNnJyODpAef7H5zn+8OP5fdvf8/yPz/PIN48AUL9GfQZ1HUT1tOr8Y/I/ot7HMv3m6UV3v19/6vWM6jmK/yz5T8x+h2Tc2vHWoKHY2zduz+Iti0ucb9vD20a98sxEV4LmrDwg8A84TFWHBWy/Cuiuqrd4l68DzlTVfhHy6w+c4Esf8bglCCIxCyQi44FnfQ97F5HJwIOqWuwWVRHpg6utkJGRcXpubm7ihbruOrjoIvdqDlmqSoEWJPyLPntPNtl7sul0VCcW/OqaDtMkjV0Hd9GwVkO25mxld+5ujq5/dNFTJ1dtX0WDmg04IvOIomODu+y0mlSjUAvZmrO16MmN+YX5fLr8U7of153MjEye//F57jrzLupk1ClWnlxPLgVaQO3qtYPem6fQw5acLTSo2YAf1v3AxcddjKqydb87Tq4nl0ItpFb14Ku9DuQfYOSCkRRoAd2P605GWgYfLP6ApnWasmHvBh445wGWblnKCY1OYPj84dzY4UZ+3vUznkIP7Rq1o3padSaumkj9GvWLnmz59eqvGbt8LNe2v5bOzTpz/9f3U6iF3Hr6rdTJqEN6tXQa1GxA7eq1WbtrLbWr10ZVmf/rfPIK8hi3chx3nXkXjTMbs273OlSVs1uczePfPk6f0/uQkZZBRloG90+6nzoZdeh5Qk9WbV9FuyPaMXj6YFrUa0Gt6rXoemxX9ufvZ0b2DAZ2Hciug7toVrcZIsKanWv4fMXnXHbCZdRIq0GTOk3YnLOZLTlbGL14NDM2zGDKL1NoVLsR1atVR1Geu/A5urXqxrxN8+jcrDOZGZnkF+TTZ3wfpq+fTsemHel/dn/mbZpHkzpNePSbRzm6/tFc0e4Ksvdk88wPz9Dr5F7c1OEmtu3fxptz3kREWLZ1Gfvz9zPwtwNp1bAVr89+nfo16jOq5ygOeA6waPMipv4yle0HtjNmyRgeOfcRalWvxZ/b/5ka6TX4Yd0PfLLsE3q06cF5x5xH/Zr1+WTZJ/xn6X/ofXJvjqxzZMQaeCyx+kRE5Grg4pDv7M6q+rcwac8H3gB+o6rbox63BEEkZoFEZALwTEgQeUBVo94CnnRNxBhjqqg4gkjM1iPv+lOAT4FLVDVmlbIkHevZQIuA5eZA6JgU8aQxxhhT9mYDbUSkpYhkAL2AoBFYReRoYCxwXTwBBEoWRGIWyLt8vfcqrS7A7lj9IcYYY0qfqnqAfsBXwHLgQ1VdKiJ9RaSvN9njwOHAGyKyQESij45JCe8TEZEewMu4a45HqOrTvsKo6lBxt5O+DnQH9gM3husPCWXNWcYYkxi72TCABRFjjElMRQWR1BrF1xhjTLmyIGKMMSZpFkSMMcYkzYKIMcaYpFXKjnURKQQOJLl7OhD/48mqBjsn4dl5Kc7OSXGHyjmpparlXjGolEGkJERkjqrG/+i7KsDOSXh2Xoqzc1KcnZPorDnLGGNM0iyIGGOMSVoqBpGYD1GpguychGfnpTg7J8XZOYki5fpEjDHGlJ9UrIkYY4wpJxZEjDHGJC1lgoiIdBeRlSKSJSIPVXR5ypOIrBWRxYFDN4vIYSIySURWeV8bBqT/h/c8rRSRiyuu5KVLREaIyBYRWRKwLuHzICKne89nloi86h2N+pAU4ZwMFJEN3s/LAu9o3L5tVeGctBCRb0VkuYgsFZG7veur9Gclaap6yE+4oehXA62ADGAhcGJFl6sc3/9aoFHIuueBh7zzDwHPeedP9J6fGkBL73lLq+j3UErn4TygI7CkJOcBmAWcBQjwBe4JbxX+/krxnAwE+odJW1XOSVOgo3e+LvCT971X6c9KslOq1EQ6A1mqukZV84AxwGUVXKaKdhkwyjs/CugZsH6Mquaq6s9AFu78HfJUdQqwI2R1QudBRJoC9VR1urpvifcC9jnkRDgnkVSVc7JJVed55/fiHtDUjCr+WUlWqgSRZsD6gOVs77qqQoGvRWSuiPTxrjtSvU+R9L429q6vaucq0fPQzDsfuj7V9BORRd7mLl+zTZU7JyJyLHAaMBP7rCQlVYJIuHbIqnTt8jmq2hG4BLhTRM6LkraqnyufSOehKpyfN4HWQAdgE/Cid32VOiciUgf4BLhHVfdESxpmXcqel0SlShDJBloELDcHNlZQWcqdqm70vm4BPsU1T232Vrfxvm7xJq9q5yrR85DtnQ9dnzJUdbOqFqhqIfA2/ubMKnNORKQ6LoD8W1XHelfbZyUJqRJEZgNtRKSliGQAvYBxFVymciEimSJS1zcPXAQswb3/G7zJbgA+986PA3qJSA0RaQm0wXUOpqqEzoO3GWOviHTxXmlzfcA+KcH3Rel1Oe7zAlXknHjfw3BguaoODthkn5VkVHTPfmlNQA/cVRargUcqujzl+L5b4a4cWQgs9b134HBgMrDK+3pYwD6PeM/TSlLoahJgNK55Jh/3K/HmZM4D0An3xboaeB3vyA6H4hThnLwPLAYW4b4gm1axc/IbXLPTImCBd+pR1T8ryU427IkxxpikpUpzljHGmApgQcQYY0zSLIgYY4xJmgURY4wxSbMgYowxJmkWRIwxxiTNgogxxpik/T94aIIbxb6KAQAAAABJRU5ErkJggg==\n",
|
|
"text/plain": [
|
|
"<Figure size 432x288 with 2 Axes>"
|
|
]
|
|
},
|
|
"metadata": {
|
|
"needs_background": "light"
|
|
},
|
|
"output_type": "display_data"
|
|
}
|
|
],
|
|
"source": [
|
|
"iters = [item[\"iter\"] for item in result]\n",
|
|
"fig, ax1 = plt.subplots()\n",
|
|
"ax1.plot(iters,[item[\"loss\"] for item in result],'g')\n",
|
|
"ax2 = ax1.twinx()\n",
|
|
"ax2.plot(iters,[item[\"accuracy\"] for item in result],'r')\n",
|
|
"plt.xlabel(\"iter\")\n",
|
|
"plt.show()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 27,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"100%|█████████████████████████████████████████████████████████████████████████████| 864/864 [00:10<00:00, 83.69batch/s]\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"model.eval()\n",
|
|
"collect_list = []\n",
|
|
"with torch.no_grad():\n",
|
|
" with tqdm(test_loader, unit=\"batch\") as tepoch:\n",
|
|
" for batch_i,batch_l in tepoch:\n",
|
|
" batch_inputs = {k: v.cuda(device) for k, v in list(batch_i.items())}\n",
|
|
" batch_labels = batch_l.cuda(device)\n",
|
|
" output = model(**batch_inputs)\n",
|
|
" loss = CELoss(output.view(-1, output.size(-1)), batch_labels.view(-1))\n",
|
|
" \n",
|
|
" prediction = output.view(-1, output.size(-1)).argmax(dim=-1)\n",
|
|
" correct = (prediction == batch_labels.view(-1)).sum().item()\n",
|
|
" accuracy = correct / batch_inputs[\"attention_mask\"].view(-1).sum()\n",
|
|
" \n",
|
|
" collect_list.append({\"loss\":loss.item(),\"accuracy\":accuracy, \"batch_size\":batch_labels.size(0),\n",
|
|
" \"predict\":output.argmax(dim=-1).cpu(),\n",
|
|
" \"actual\":batch_labels.cpu(),\n",
|
|
" \"attention_mask\":batch_inputs[\"attention_mask\"].cpu()})"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 31,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def getConfusionMatrix(predict,actual,attention_mask):\n",
|
|
" ret = torch.zeros((tagIdConverter.size,tagIdConverter.size),dtype=torch.long)\n",
|
|
" for i,(p_s,a_s) in enumerate(zip(predict,actual)):\n",
|
|
" for j,(p,a) in enumerate(zip(p_s,a_s)):\n",
|
|
" ret[p,a] += attention_mask[i,j]\n",
|
|
" return ret"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 33,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"average_loss : 0.13179491325355058, average_accuracy : 0.9739284515380859, size :3453\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"total_loss = 0\n",
|
|
"total_accuracy = 0\n",
|
|
"total_size = 0\n",
|
|
"confusion = torch.zeros((tagIdConverter.size,tagIdConverter.size),dtype=torch.long)\n",
|
|
"\n",
|
|
"for item in collect_list:\n",
|
|
" batch_size = item[\"batch_size\"]\n",
|
|
" total_loss += batch_size * item[\"loss\"]\n",
|
|
" total_accuracy += batch_size * item[\"accuracy\"]\n",
|
|
" total_size += batch_size\n",
|
|
" confusion += getConfusionMatrix(item[\"predict\"],item[\"actual\"],item[\"attention_mask\"])\n",
|
|
"print(f\"\"\"average_loss : {total_loss/total_size}, average_accuracy : {total_accuracy/total_size}, size :{total_size}\"\"\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 34,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"tensor([[ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
|
|
" [ 0, 1566, 19, 78, 19, 1, 3, 1, 0, 22],\n",
|
|
" [ 0, 30, 599, 53, 3, 1, 11, 1, 1, 76],\n",
|
|
" [ 0, 48, 32, 1479, 21, 2, 1, 3, 0, 53],\n",
|
|
" [ 0, 7, 14, 17, 1557, 0, 6, 1, 2, 27],\n",
|
|
" [ 0, 1, 0, 0, 0, 1510, 47, 101, 26, 45],\n",
|
|
" [ 0, 2, 8, 0, 0, 39, 583, 54, 4, 298],\n",
|
|
" [ 0, 7, 2, 9, 0, 39, 63, 2560, 32, 122],\n",
|
|
" [ 0, 0, 0, 1, 8, 11, 38, 29, 3355, 33],\n",
|
|
" [ 0, 7, 28, 24, 9, 25, 86, 45, 15, 55141]])"
|
|
]
|
|
},
|
|
"execution_count": 34,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"confusion"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 35,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"confusion = confusion[0:(tagIdConverter.size - 1)][0:(tagIdConverter.size - 1)]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 36,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAUkAAAEICAYAAADSjgZhAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAAsTAAALEwEAmpwYAABO9UlEQVR4nO2ddVxW1x/H319AsKdOVAQ7QDFo7Nmzu2fNmv5Wrjvc5ow5NzvmQqezpptdsxuxZ7eiGBhTQKXO74/nAVGJB3gu5Xn7ui/uc+653++5l8cvpz+ilEKj0Wg0CWOT0QXQaDSazIwOkhqNRpMEOkhqNBpNEuggqdFoNEmgg6RGo9EkgQ6SGo1GkwQ6SGYDxMSvInJbRALSYKeuiJywZtkyChEpKSKhImKb0WXRZG1Ez5PM+ohIXWAu4KqUCsvo8hiNiJwHBiil/snosmiyP7ommT0oBZx/FgKkJYiIXUaXQZN90EEynRGREiKyWERuiMhNEZloTrcRkU9F5IKIXBeRWSLynPlaaRFRItJHRC6KSIiIfGK+1h+YAdQ0Ny+HiUhfEdn2hF8lIuXN5y1E5KiI3BORyyLyrjm9vogExbunkohsEpE7InJERNrEu/abiEwSkRVmO7tFpFwizxxb/pdF5JK5W2CwiPiKyCGz/Ynx8pcTkQ3m9xMiInNEpID52u9ASWCZ+Xnfj2e/v4hcBDbES7MTkUIiEiQirc028orIaRHpndbfp+YZQCmlj3Q6AFvgIPADkAfICdQxX+sHnAbKAnmBxcDv5mulAQX8BOQCqgMPgUrm632BbfH8PPbZnKaA8ubzYKCu+bwg4GU+rw8Emc9zmMvzMWAPNATuYWrSA/wG3AL8ADtgDjAvkeeOLf9U8zM3BR4AfwNFAGfgOvCCOX95oAngADgCW4Af49k7DzROwP4s83vNFS/NzpynKXDV7O8n4M+M/j7oI2scuiaZvvgBxYH3lFJhSqkHSqnYGt9LwFil1FmlVCjwEdDtiabjMKXUfaXUQUzBtnoqyxEJVBaR/Eqp20qpfQnkqYEpWI9USkUopTYAy4Hu8fIsVkoFKKWiMAVJj2T8fm1+5rVAGDBXKXVdKXUZ2Ap4AiilTiul1imlHiqlbgBjgRcseK4vze/1/pMXzD4XAuuBlsArFtjTaHSQTGdKABfMQeVJigMX4n2+gKmGVjRe2tV45+GYglhq6Ai0AC6IyGYRqZlIeS4ppWKeKJNzGspzLd75/QQ+5wUQkSIiMs/cFXAXmA0UTsY2wKVkrk8HqgC/KqVuWmBPo9FBMp25BJRMZGDhCqYBmFhKAlE8HkgsJQzIHftBRIrFv6iU2qOUaoup6fk3sCCR8pQQkfjfkZLA5VSUJ6WMwNRUrqaUyg/0BCTe9cSmZCQ6VcM8FWgapib5kNj+WY0mOXSQTF8CMPUHjhSRPCKSU0Rqm6/NBd4SkTIikhf4FpifSK0zOQ4C7iLiISI5gS9jL4iIvYi8JCLPKaUigbtAdAI2dmMKtu+LSA4RqQ+0BualojwpJR8QCtwREWfgvSeuX8PUd5sSPjb/7AeMAWbpOZQaS9BBMh1RSkVjCjTlgYtAENDVfPkX4HdMgxTnMA1svJ5KPyeBr4B/gFPAtiey9ALOm5uygzHV1J60EQG0AZoDIcBkoLdS6nhqypRChgFewH/ACkyDWPEZAXxqHhV/NzljIuINvI2p/NHAKEy1zg+tWmpNtkRPJtdoNJok0DVJjUajSQIdJDUajSYJdJDUaDSaJNBBUqPRaJLAkI0AChcurEqVKm2EaY1Gkw5cuHCekJAQST5n4tjmL6VU1FOLnxJE3b+xRinVLC3+jMKQIFmqVGm27w40wrRGo0kHavv7pNmGinqAg1s3i/I+2D/BkhVVGYLeUkqj0RiDAJKmymimQAdJjUZjHJL1hz10kNRoNMaRDWqSGRrm165ZTTV3V9zdyvPd6JHafgb40Pazt/308pEwAja2lh2ZGSM2qfTy8lb3I1WSR+iDKFWmbFl19MQZ9V/YQ1W1ajW17+CRZO+z9Mjq9rPDM2j7Wfc75OXlrdIaByRPUZXT/32LDiAwozfXzXSb7u4JCKBcufKUKVsWe3t7OnftxvJlS7T9dPSh7Wdv++nlI3HE1Ny25MjEZFiQvHLlMi4uJeI+Ozu7cPmy9bYqzOr208OHtp+97aeXjyQRG8uOTIxFpRORZiJywiyeZJXtpRLafUis+Bclq9tPDx/afva2n14+kuRZqEmaNyadhGlfwcpAdxGpnFbHzs4uBAU92m3/8uUgihcvnlaz2cZ+evjQ9rO3/fTykTjyzNQk/YDTZoGqCEw7U7dNq2MfX19Onz7F+XPniIiIYOH8ebRs1Sb5G58R++nhQ9vP3vbTy0eiCNlidNuSeZLOPC6wFAT4P5lJRAYBgwBKlCyZvGM7O34YN5HWLV8kOjqaPn37Udnd3bJSW0BWt58ePrT97G0/vXwkjmT6WqIlJLszuYh0Bl5USg0wf+4F+CmlEpUW8Pb2UXrttkaTdant78PevYFp6iy0yeesHHwGW5T3wabP9yql0r5g3AAsqUkGYZJCjcUFk5KeRqPRJI6QLWqSljzBHqCCWcXPHugGLDW2WBqNJluQDUa3k61JKqWiROQ1YA1gC/yilDpieMk0Gk0WRzL9oIwlWLTBhVJqJbDS4LJoNJrsRjZobutdgDQajTFkgaa0JeggqdFojCMb1CSz/hNoNJrMi5UGbkQkp4gEiMhBETkiIsPM6YVEZJ2InDL/LBjvno/MS6lPiMiL8dK9ReSw+dp4SWadpg6SGo3GIKy6LPEh0FApVR3wAJqJSA3gQ2C9UqoCsN78GfPS6W6AO9AMmGxeYg0wBdPClwrmI0kBMh0kNRqNMVhxWaIyEWr+mMN8KExLpGea02cC7cznbYF5SqmHSqlzwGnAT0ScgPxKqZ3KtJJmVrx7EkQHSY1GYxApqkkWFpHAeMegp6yJ2IrIAeA6sE4ptRsoqpQKBjD/LGLOntByamfzEZRAeqJkyYGbYWtPGGr/8yYVDbUPEB2T9HLQtGJrY+yoYrput6VJkOSWFKfJtrUMWf49CUluWaJSKhrwEJECwF8iUiUpzwmZSCI9UXRNUqPRGIcBW6Uppe4AmzD1JV4zN6Ex/7xuzpbYcuog8/mT6Ymig6RGozEO641uO5prkIhILqAxcBzTEuk+5mx9gFhtiqVANxFxEJEymAZoAsxN8nsiUsM8qt073j0JkiWb2xqNJgsgVt0qzQmYaR6htgEWKKWWi8hOYIGI9AcuAp0BlFJHRGQBcBSIAl41N9cBhgC/AbmAVeYjUXSQ1Gg0hiE21gmSSqlDgGcC6TeBRoncMxwYnkB6IJBUf+ZjZEnd7RU/fsz4HrWY8b/WcWlb50xgYu96/PJaO355rR1n9myOu3b93AlmvdOVGUNa8fP/WhMV8RCA6MgIVo3/jGkDX2T6K805vn1Nsr5fGdiPUs5F8fGoGpd26OBB6tetha9nNTq2a8Pdu3ctfpYnOXnyBLX8vOKO4o4FmDRhHIcOHqBBvVrU8vOiXi0/AvcEpMq+0eV/zNeAfpQsXgRvD4u/jyniwYMH1Knph59Xdbyqu/P1sC+s7sPoZ7C2JnbQpUs0a9IQz6qV8a5ehUkTxgHw8Yfv4VGlEn5e1enaqQN37txJs6/kEEwDfJYcmZkMC5LR0dEMfeNVlixbxf5DR1k4by7Hjh616N6qjdvT5aufnkr3bduHfhP/pt/Evynn+wIAMdFRLBvzHi++OowBU5bTY+QsbGxNFegd86eSp8DzvPLTGgZOWUHJKn7J+u7Vuy9/L3+8dv6/wQP5evgI9uw/RJt27fjh++8seo6EqFjRlR0B+9gRsI+tO/eQK3duWrdpx2cff8BHn3zGjoB9fPL5l3z2cer02Iwu/2O++vRlyfLVVrGVEA4ODqxet4GAfQfZHXiAtWtWs3vXLqv6MPIZ0vJ/IDFs7ewYMXoM+w8fZdO2nUybMpljR4/SsFETAg8cJmDfQSpUqMCYUSOs9BRJICk4MjFZUne7ZBVfcuZ7zqK85/Ztp0hpV4qWdQMgV/6C2NiaJq8eWreYGl1M07HExobczxVM1E4sderWo1DBQo+lnTp5gjp16wHQqFETlvy12KKyJcemDespU6YcJUuVQkS4Z67h3f3vP5ycnFJlMz3LX6duPQoVKpR8xlQiIuTNmxeAyMhIoiIjrV4rMfIZjNDEdnJywtPTC4B8+fLh6laJK1cu07hJU+zsTJUDX/8a6SQra1ktUtckE8EIPeC9y+fw86ttWPHjxzy49x8Aty6fBxHmf9afX9/owK4/ZwDwINQUcLb+Po5f3+jAX9++SdjtkFT5rexeheXLTPsQL1608DF1urTw58L5dO7aDYCRY37g048+wK1cKT756H2+/Ppbq/gA48qfHkRHR+Pv7UHJ4kVo2LgJfv5PyS9lWozWxL5w/jwHD+7H1+/xdzLrt19p+mKSK/GsxjMRJEXkFxG5LiL/WtOxtfWAvVp0Z/CMdfSb8Dd5Czqy/udRgKm5HXR0L63fHUPP0XM4uXMd5w/sJCY6mnshV3Gu7MXL4xfjXMmDDT+PTpXvqdN/ZvrUydTy9+HevXvY29un+jliiYiIYOWKZbTv0AmAn6dPZeR333P8zAVGjv6eVwcPTLOPWIwof3pha2vL7r0HOH0+iMA9ARz516pfU0MxUhM7NDSU7l07MXrMD+TPnz8ufdSI4djZ2dGtx0tW8ZMcNjY2Fh2ZGUtK9xvJLABPDdbWA85TsDA2traIjQ3Vm3Um+ORhAPIVLkaJKr7kfq4gOXLmopzPC1w7c5Rc+QuQwyEXrjWbAOBWpxnXzqSuP8jVzY1lK9ewY3cgXbp2p0zZcql+jljWrlmFh4cnRYoWBeCP2bNo064DAO07dmZvYOoGbhLCiPKnNwUKFKDeC/VZu9a4PlBrY5QmdmRkJD26dqJb9x60a98hLn32rJmsWrmCX2fNTp/a27PSJ6mU2gLcsrZja+sBh966Hnd+csc/OJaqAEBZrzrcOH+SyAf3iYmO4uLhPTxfohwiQnn/Blw4bAo25w/s5PkSqQsO16+bfMfExDBqxHAGDHol1c8Ry58L5tGpS7e4z8WcirNti2nEfvPGDZQrXyHNPmIxovzpwY0bN+JGae/fv8+G9f/g6uqWsYVKAUZoYiulGDJoAK5ubrwx9O249LVrVjN2zGgWLl5C7ty501p0i5Bs0ieZYfMk06IHvGTU21w8vIf7d28zqfcL1HnpdS4eDuD62WMgwnNFnGn2+jAAcuZ7Dt92fZn5VmcQoZxPPcr71Qeg/svvsGzMB6yf/i25nytEi6HJ9/P16dmDLVs2cTMkhPJlSvDp518SFhrKtCmTAWjbrj29+7ycupdiJjw8nA3r/2HcxKlxaRMmT+ODd98iKiqKnDlzMn7S1CQsZGz5Y+ndsztbN28iJCSEcqVd+OzzYfTt198qtgGuBgczsF8foqOjiVExdOzUhRYtW1nNPhj7DEZoYu/csZ0/5vxOlSpV8fcxTSsc9vVw3n37TR4+fEir5k0B8PP3Z0Iqv0MpIbMHQEtIVncbQERKA8uVUolOFjPv2jEIoETJkt4nz1ywVhmfQm9wkTx6g4vsj5EbXNSu4cu+NOpu2z1fVuVv8Y1FeW/PfinT6m5brcdUKTVdKeWjlPJxLOxoLbMajSYLo5vbGo1GkxgCYnCLJj2wZArQXGAn4CoiQeaF5BqNRpMkz8zAjVKqe3oURKPRZD8yewC0BN3c1mg0xpH1Y6QOkhqNxiBE1yQ1Go0mSXSQ1Gg0mkQQJNOvy7YEHSQ1Go1xZP2KpA6SGo3GIHSfZMbxWWNjlw2evR5mqH2AUoWN3WQgO3w5NUlj9NJWa5AdvodZMkhqNJqsgQ6SGo1GkwTZYVmiDpIajcYQssKSQ0vQQVKj0RhGdgiSWX8Sk0ajybRYa4MLESkhIhtF5JiIHBGRN83pX4rIZRE5YD5axLvnIxE5LSInROTFeOneInLYfG28JFOADA2S1hZmf/DgAfVq++Pv44GPRxW++cokVn/r1i1aNW9KtcoVadW8Kbdv306R3aY13GnfyJ+OTWvRpYVJevX40cO81KYh7Rv582rfzoTeM6kvRkZE8Onbg2nfyJ8OTWoSsGNrip/DvWJZ/L2rU8vPi3q1TFrgX3/5OTV8PKjl50Xbli8SfOVKiu0+yYMHD6hT0w8/r+p4VXfn62FfpNlmfF4Z0I+SxYvg7ZHoXs1Wsbnoz4V4VXcnt70NewMDreYrMX+Z3f7Jkyeo5ecVdxR3LMCkCeMAmDp5Ip5VK+HrWZVPP/7Aaj4TxXoaN1HAO0qpSkAN4FURqWy+9oNSysN8rAQwX+sGuGPS6JosIrbm/FMwbRBewXwkqeGVYUHSCGF2BwcHVq5Zz+7AA+zcs591a9cQsHsX3383kvoNG3Lo6EnqN2zI99+lPCD/snAFi9buYMHKLQB88d5rDP3oK/5av5tGzVrz61TTl/DPP34D4K/1u/lp7lLGfP0xMTExKfa3Ys16dgTsY8sOkwbPm2+/y67AA+wI2EezFq0Y+e3XKbb5JA4ODqxet4GAfQfZHXiAtWtWs3vXrjTbjaVXn74sWW5dYa6EbLq7V2HegsVx2uFG+8vs9itWdGVHwD52BOxj68495Mqdm9Zt2rFl00ZWLFvKrsAD7Nl/mDeHvmNVvwlhrZqkUipYKbXPfH4POAY4J3FLW2CeUuqhUuoccBrwExEnIL9Saqcybe0+C2iXlO8MC5JGCLM/KVYfaRarX7FsKS/17APASz37sHxp2vwAnD9zCp8atQGoWa8h61aabJ45dRz/2vUBeL6wI/nyP8eRg/vS7C++LGhYWJhV+nqefF9R5vdlLerUrUehQoWsZi8xm26VKlHR1dWqfpLyl5Xsb9qwnjJlylGyVClm/DSVt999HwcHBwAcixQxzC+ACNjYiEUHUFhEAuMdgxK3K6UBT2C3Oek1ETlklr8uaE5zBuILyAeZ05zN50+mJ0qGBUmjhNmjo6Op4etJaZeiNGzUGF8/f65fv4aTkxMATk5O3LhxPRkrjyMiDOrRji7N67Jw9i8AlHetxMa1KwBYu/wvrl4xld21UhU2rl1BVFQUQRfPc/TwgbhrKfHXrlUz6tb05ZcZ0+PSh33+KW7lSrFg3h988vmwFNlMjOjoaPy9PShZvAgNGzfBz98/+Zs0WYY/F86nc1eT6ubpU6fYsX0bDerWpFnjBuwN3GOw9xRtuhsSK/9iPqYnaFEkL7AIGKqUuoup6VwO8ACCge/jnD+NSiI9USzZmTzBDtO0YpQwu62tLbv27Ofk2UvsDdzDkSNpF6v//a91LFy9jSm/L2buzJ8I3LWNr7+fzNyZP9GleV3CQkPJkSMHAO279aaokzNdW9Rj1Jcf4OHtj62dbTIeHmfdxq1s2xXI4iUr+GnaFLZtNTfxv/qG42cu0KVbD6ZPmZTm5wLT+9q99wCnzwcRuCeAI/+m/X1pMgcRERGsXLGM9h06ARAVFcWdO7fZsGUH34wYRZ+XuhkqJgam2qQlh2W2JAemADlHKbUYQCl1TSkVrZSKAX4C/MzZg4AS8W53Aa6Y010SSE8US2qSSXWYphqjhNljKVCgAHXrvcC6NaspUqQowcHBAAQHB+PomLJmRpFiplro84UdadSsNYcP7KVseVd++mMJC1ZtpUW7TpQoVRYwyYR+8OVIFq3dwYRf5nP37h1KlSmfIn9O5vfgWKQIrdu0e+ovfpeu3Vny9+IU2UyOAgUKUO+F+qxda1z/myZ9WbtmFR4enhQpWhQAZ2dn2rRtj4jg4+uHjY0NISEhhpbBiqPbAvwMHFNKjY2X7hQvW3sg9q/8UqCbiDiISBlMAzQBSqlg4J6I1DDb7A0k2f+WbJBMRYepRRghzP6kWP3GDetxdXWjRavWzJk9E4A5s2fSsrXlfsLDwwgLvRd3vmPLeiq4VuZmyA0AYmJimDbuO7r06mf2G054uGnt944tG7Czs6NcRTeL/YWFhXHv3r248/Xr11HZ3Z3Tp0/F5Vm5YplV+uCefF8b1v+Dq6vlZdVkbv5cMI9OXbrFfW7Vpi2bN20E4NSpk0RERFC4cGHjCmBhLdLCmmRtoBfQ8InpPqPN03kOAQ2AtwCUUkeABcBRYDXwqlIq2mxrCDAD02DOGWBVUo5TNJk8gQ7T+Nfi624na8sIYfarV4MZ1L+vSaw+JoaOnTrTvGUr/GrUpFePrsz69RdcSpRk9twFFtu8eeM6bw7oAUB0dBQt2nWhToMm/D5jMvNmmrpNGjdvQ/uuvQC4FXKDV15qh9jYULRYcUaM+ylFz3D92jV6dO0ImJpHXbp2p0nTZrzUrROnTp7ExsaGEiVLMm7ClBTZTYirwcEM7NfH9L5UDB07daFFy1ZpthtL757d2bp5EyEhIZQr7cJnnw+jb7+06cglZLNgoUK8PfR1Qm7coEPbllSr7sGylWsy7TOkh/3w8HA2rP+HcROnxqX16tOP/w3qj59XNezt7Zk241dDJ3sLxA7KpBml1DYS7k9cmcQ9w4HhCaQHAhbPuRJL+yTMHaabgeGx/QGJ4e3to7bvtu58tfjEGLz7ybkbWX8XIDtbvU4guxMVnfKpZZZSr5Yf+/YGpinC5XKqqMr1t6zv/MjwpnuVUj5p8WcUFtUkE+ow1Wg0miRJwaBMZibZIJlYh6lGo9EkhfDsrN1OrMNUo9FokiBF8yQzLcnWJJPoMNVoNJokyeTxzyL0VmkajcYYxHqj2xmJDpIajcYQskufpA6SGo3GMLJBjNRBUqPRGIeuSWo0Gk0SZIMYqYOkRqMxCNE1yQzDaEn2Mo55DPYAY7ecMdT+Oy+UM9R+dvjya4xFED26rdFoNEmRHf6W6iCp0WgMIzu0OHSQ1Gg0xvCsbHCh0Wg0qSG7TCbPVrrbsURHR1PLz4tO7VoDcOjgARrUrUlNX0/q1vQlcE9Aquwmpuv98Yfv4Vm1En7e1enWuUPcbt9J8deYDxnZ2Z8JA5/eK2Tbwhl81qQCYf/dAuDg+iVMeqV13PF504oEnzbJ7x7etIKJg1oxfkBz1vw0Klm/QZcu0axJQzyrVsa7epU4PebFfy7Eu3oV8jjYsnevdfYCvXTpEi82boBH1Up4VXdn4vhxVrEbHyN1sY3WJQfj/g+kl3Z7cmSHDS6yle52LJMnjMPVrVLc508/+oCPPvmcnXv28+nnw1Ityp6YrnfDRk3Ys/8wAXsPUr5CBcaMHpGsLc+mHej97S9Ppf93PZgze7fzXJFHej/VG7Xl1WnLeHXaMjp+OIYCRZ1xKl+Z8Lu3WTN9FC+PnskbM1YRevsmZ/btSNKvrZ0dI0aPYf/ho2zatpNpUyZz7OhRKrtXYe6CRVbVrrazs2Pk6O85cPgYm7ftYtrUSVb7HcdipC620brkRv4fgPTRbk+OFEjKZlqyle42wOWgIFavWkmflx9tgS8i3L13F4D/7v6Hk1PqBMcS0/Vu3KQpdnamngs//xoWSeOWruZHrnzPPZW+cupwmg58P9G/roc3LKdqA1MN+VbwJZ53KUOeAs8DUM6zFke3JS1b4OTkhKenFwD58uXD1a0SV65cNkS72snJCU+vR77czL6siZG61Ubrkhv1fyAxjNBuTxLratxkGNlOd/v9d9/imxGjsLF59GijxvzApx+9j2u5knzy4XsM+/rbVNtPSNc7PrN++5WmLzZLle1jO9aT//miOJWrlGiew5tXUK2BSYfm+eKlCLl0httXg4iOjuLYjnX8dyPYYn8Xzp/n4MH9Tz2DEVw4f54DB9LHlzUxUpfcqP8DkL7a7YmWIZvsJ2mJ7nZOEQkQkYNm3W2rvFkjdLdXrViOo6Mjnl7ej6XPmD6Fkd+N5cSZi4z8biz/e2VAqn0kpes9euRw7Ozs6Nb9pRTbjXhwny1zJ9Oo79BE81w6doAcDrkoWqYiALnyPUfrN4axYPib/PxWdwoUdcHG1rKxuNDQULp37cToMT88VsMwgtDQULp36ch33/9ouC9rY6QuuVHa85C+2u1J8azUJB8CDZVS1QEPoJmI1EirYyN0t3ft3M7KFcuoXLEMfXt1Z/OmDfTv24s/Zs+ibbsOAHTo2Jm9gakbuIlPfF1vgNm/z2TVyhX8MnN2qr7ot4IvcvtqEJNeac33Petz98ZVpgxpx71bN+LyHN70qBYZi1vNRrwyYRGDxi+kcIkyPO9cKllfkZGR9OjaiW7de9CufYcUlzUlREZG0r1LR7p2f8lwX0ZihC65kdrzGaHdnhA2IhYdmRlLdLeVUirU/DGH+UjzykAjdLeHfTOCk2cvcfTkOX77fS4v1G/Iz7/9TjGn4mzdshmATRs3UK58hVTZT0zXe+2a1fwwZjQLFi0hd+7UqSAWK+PKhwt3887sTbwzexP5HYsxZMrf5CvkCJj0vY9sWUXVBi0fuy/09k1Tee79R8DSOXg375KkH6UUQwYNwNXNjTeGvp2qslqKUorBA/vj6laJN98y1pcRGK1LbsT/AUhf7fakEMkeAzeWqiXaAnuB8sAkpVSm1N1OjIlTpvP+O0OJiooiZ86cTJg8LVV2EtP1rlqpAg8jHtK6RVMA/Pz8GT9papK2FgwfyrlDAYT/d5vvutehYe838W7eOdH8Fw7vIX/hYhRyevzdrpz8NVfPHgegfs/XKOxSJkm/O3ds5485v1OlSlX8fTwBGPb1cB4+fMg7b71ByI0bdGzbimrVPVi6Im21ph3b4/ny9jD5+uZbmjW3nkSSkbrYRuuSG/V/ID2125PDWvFPREoAs4BiQAwwXSk1TkQKAfOB0sB5oItS6rb5no+A/kA08IZSao053Rv4DciFSbf7TZWEtrbFuttm4wWAv4DXlVKJds4YrbsdbbDudnr8XdMbXGjSSmbX3X6uVCVV+6OZFuVdNcQ/Sd1tEXECnJRS+0QkH6ZKWzugL3BLKTVSRD4ECiqlPhCRysBcwA8oDvwDVFRKRYtIAPAmsAtTkByvlFqVmO8UjW4rpe4Am4DUDd9qNJpnCmsN3CilgpVS+8zn94BjgDPQFoiNxDMxBU7M6fOUUg+VUueA04CfOdjmV0rtNNceZ8W7J0EsGd12NNcgEZFcQGPgePKPpdFonmUE8zQgC/4BhUUkMN4xKFG7IqUBT2A3UFQpFQymQAoUMWdzBi7Fuy3InOZsPn8yPVEs6ZN0Amaa+yVtgAVKqeUW3KfRaJ5xUtAnGZJUczsWEckLLAKGKqXuJtHtk9AFlUR6oliiu30IU9TWaDQayxHrjlyLSA5MAXKOUip2/tI1EXFSSgWbm9LXzelBQIl4t7sAV8zpLgmkJ0qGbnCh0WiyL4L15kmKqcr4M3BMKTU23qWlQB/zeR9gSbz0biLiICJlgApAgLlJfk9Eapht9o53T4LordI0Go1hWHESRG2gF3BYRA6Y0z4GRgILRKQ/cBHoDKCUOiIiC4CjQBTwqlIq2nzfEB5NAVplPhJFB0mNRmMY1poqppTaRuKz8xolcs9wYHgC6YGAxXvr6SCp0WgMISusy7YEHSQ1Go1h2GaDKKmDpEajMYzssDIrSwZJW4MXxKdkqWZqMXrZ4Df/nEo+Uxr4rElFQ+1rkiez755jGt3O6FKknSwZJDUaTRYgC2yoawk6SGo0GsPIBjFSB0mNRmMcuiap0Wg0iSAYP36QHmRL3W2AkydO4O/tEXcUKZSfCeN+TLPdVwb2o5RzUXw8qj517cexY8htb0NISIhV7ffq0Q1/H0/8fTxxq1AmbrPcpFj+w0f82L0m04c82iR2y+wJjO9VlxmvtWXGa205vce0W/uda0GMblctLn3VhM8BeBgeGpc247W2/NDNn3XTnpqbmyRG/o4B7ty5Q/eunahexQ2PqpXYtXOnVe0bqetthC550KVLNG/aEK9qlfHxeKSr3vulbtTw9aSGryeVKpahhm/6bMcgFh6ZmQyrScZqDq9YtQ5nFxfq1PClVas2VKpc2Sr2K7q6snvvgThf5Uo506Zd+zTb7dW7L4P/9xoDX+7zWHrQpUtsWP+PRbuyp9T+73/Mizv/8P13yJ//aSnaJ6nWuAM+rXuy9PvHNcb92vWlRsend+4u4FSSARMfX8LqkDvvY2m/vNEB11pNLX4Wo3/HAO++9SZNmzZj7vw/iYiIIDw83Gq2waTrPfh/rzGgX2+r2oVHuuSeXl7cu3ePWv7eNGrcJE3vx9bOjm9HjcHT02SzTg0fGjZuwqw5j3+Hnnsu+e9QWhHJ/CPwlpDtdLcTYuOG9ZQpW45SpZIXyUqOOnXrUajg0zrP77/7Nt98OyrNfTCJ2QfT1KRFfy6kS9fuydopWdWXnAnoeqeWW5fPE3bnJiWqJLubVRxG/47v3r3Ltm1b4uQa7O3tKVCggNXsg7G63kbokieoqx5PplYpxeJFC+ncJfnvkDV4VtQSDcFIzeEnWTh/nkWBJbUsX7aU4s7FqVa9umE+ALZv20qRIkUpXyF1QmYAe5fN4af/tWb5Dx9x/95/cen/XQ3i59fa8fv7Pbn479PSG0c2L6dyvRYp+iNg9O/43NmzFC7syKD+L1PDx5MhgwYQFhZmNfvpiRG65AnpqlvjO5QSngnd7VhExFZE9ouIVTbcNVJzOD4RERGsWL6UDp0SF9lKC+Hh4Ywe+S2fffGVIfbjs2D+XLp07Zbq+71admfIz+sYMHEJeQsVYf0MUx9h3kJFeHXmRvpP/JvGAz9kyeh3eBge+ti9RzevpPILLRMymyhG/46joqI4sH8fA18Zwq7A/eTOk4cxBvR7Go0RuuShoaH06Pa0rvrC+XPp3CX136GU8qzVJN/EpCthFYzUHI7PmtWr8PD0omjRola3DXD2zBkunD+Hv48HbhXKcDkoiFr+3ly9etWqfqKiolj691907Nw11TbyFiyMja0tYmODR7POXDl5GAC7HPbkzl8QAKcKVSjoVJJbQefi7rt29jgqOhqnCikbvDD6d+zs4oKziwt+/qaaUvuOnTiwf5/V7KcHRuiSx+qqd+3WI05vHkzfoSVL/qJTGr5DKUFEsLWx7MjMWBQkRcQFaAnMsJZjozSHn8RU+zKuqV2lalUuXL7G8VPnOH7qHM4uLuzYvZdixYpZ1c+G9f9Q0dUNFxeX5DMnQuit63HnJ3f8g2MpU5Mr7L9bxESbttq7HXyJW1fOU8DpUTP56OblVK6fslokGP87LlasGC4uJTh54gQAmzasx62S9QaFjMYIXXKlFENeSVhXPVY33DkN36GUkh2a25aObv8IvA/kSyxDZtTdDg8PZ8M/65iYSp3thOjTswdbtmziZkgI5cuU4NPPv6Tvy9bReU7K/p8L5tM5BU3tv0e9zYVDAdy/e5sJvepRt+frXDwUwLWzx0GgQFFnmr9u6iK4dHgPW2aPN9cybWn+2jBy5SsQZ+vY1lV0GTY9xc+SHr/jsT9O4OXeLxEREUHpsmWZPuNXq9o3UtfbCF3ynTu2M3fO77hXqRo3zefLr4bTrHkL/lw4P12b2pA9pA+S1d0WkVZAC6XU/0SkPvCuUipJhXajdbeNJj02uDAavcFF9ifGQP35OjV906y7XbR8FdV1zJ8W5Z3QvlKSutsZiSU1ydpAGxFpAeQE8ovIbKVUT2OLptFosjqZvLvRIpKtDSulPlJKuSilSgPdgA06QGo0muQQIVsM3Oi12xqNxjAyefyziBQFSaXUJmCTISXRaDTZjkw+cG0Ruiap0WgMIVZ3O6ujg6RGozGM7DAFKDs8g0ajyaRYa1miiPwiItdF5N94aV+KyGUROWA+WsS79pGInBaREyLyYrx0bxE5bL42XiyYya6DpEajMQQrL0v8DWiWQPoPSikP87HS7Lcyppk47uZ7JouIrTn/FEyLXiqYj4RsPoYOkhqNxjBsxLIjOZRSW4BbFrptC8xTSj1USp0DTgN+IuIE5FdK7VSmFSOzgHbJPoOFTjUajSZFxA7cWHKkgddE5JC5OV7QnOYMXIqXJ8ic5mw+fzI9SfTATQJk9gX3lmD0ssGR641d9vheg/KG2o82cElfLDlsDdaHN9S6dUjBf6XCIhJ/LfN0pVRyGwZMAb7G9Cq+Br4H+pGwIoRKIj1JdJDUaDTGYGFT2kxIStduK6WuxbkS+QmI3es2CCgRL6sLcMWc7pJAepLo5rZGozEMsfBfqmyb+hhjaQ/EjnwvBbqJiIOIlME0QBOglAoG7olIDfOodm8gWT0RXZPUaDSGIICdlaphIjIXqI+pWR4EfAHUFxEPTE3m88ArAEqpIyKyADgKRAGvKqWizaaGYBopzwWsMh9JooOkRqMxDGv17yulEto5++ck8g8HntI/VkoFAinaYj/b6m6DsZrJYHz508NHau0vGfsR33WtweRXHu1Yvun38Yx9qQ5T/9eGqf9rw6mATQCE373NzPd78W07D1ZOGvaYnSun/mXK4FaMf7kxqyZ/bfFentHR0dTy86JTu9aASVe6pq8nNX09qVyxDDVToCv96iv9KVeyGDW8q8Wl3bp1i7Ytm+JZxZW2LZty+/ZtU/rNm7R6sRHFC+fn3aGvW+wjlqBLl2jWpCGeVSvjXf2RLjbAlEkTqO7uhnf1Knzy4fspth2fJ9/P8K+/pEIZl7h3tGbVyjTZtwTT6LZ1pgBlJBkWJGM1mZcsW8X+Q0dZOG8ux44etaqPXn36smT5aqvajCU9ym+0j7TY92jSgZ7fPP2HvEb7lxk8eSmDJy+lgl99AOzsHWjQ+02aDvzgqfwrJnxBqze+5vVf1nHrynlOB26xyP/kCeNwdasU93nWnHns3LOfnXv207ZdhxRprPfo1YdFSx4PGj+MGcUL9Rux/98TvFC/ET+MGQWAQ86cfPL5ML4eMdpi+/GxtbNjxOgx7D98lE3bdjJtymSOHT3K5k0bWb5sKQH7DrL34L+8+fa7qbIfy5PvB+C114fGvaMX07D7ucVYuNoms08myda620ZqJqdH+Y32kRb7par6kstCXW/7nLkpWcUHuxwOj6Xfu3mdh+GhlKjsiYhQrVF7ju/4J1l7l4OCWL1qJX0SkM1Ija507Tr1KPjE92Tl8qX06NkbgB49e7PC/F7y5MlDzdp1yJkzp8X245OgLvaVy/w0bSrvvPcBDg6md1SkSJFU2Yek3096kw7zJA3nmdDdNoL0KL/RPoywH7B0NlMGt2bJ2Md1vRPi3s1r5C/8SDAtv2NR7t28lsQdJt5/9y2+GTEKG5unv77W0pW+cf0axZxMg6fFnJy4ceN6MneknPi62KdOnWT7tq3Uq12Dpo3qExi4J9V2E3s/06ZOwt+7OkMG9YvrPjASAWxtLDsyM5aqJZ43Lwo/8MSEz1STXrrbRpEe5Tfah7Xt+7TqwRu//sPgyUvIW8iRtT8l3ceZGv+rVizH0dERTy/vBK+nt650agkNDaV710e62NFRUdy5c5vN23YyfORoevXomiqtpcTez4BBQzh87DQ79+ynaDEnPv7gHWs9ShIINhYemZmUxPAG5kXkVhHrSS/dbaNIj/IbrlttZfvxdb29m3Xh8olDSebPX7gYd0Me6ZPfvXGNvIWSbmbu2rmdlSuWUbliGfr26s7mTRvo37cXYNYmX5I2bfJYHIsU5WpwMABXg4NxdEx98/dJYnWxu3XvEae1XdzFhbbtOiAi+Pr6YWNjQ0hISIptJ/Z+ihYtiq2tLTY2NrzcbyCBe1JfU7UUQfdJpon00t02ivQov9E+rG3/3s1HTdJjO9ZRpHTSTd58zxfBIVcego4dQCnFofV/4VazUZL3DPtmBCfPXuLoyXP89vtcXqjfkJ9/+x2AjWZtcmvoSjdv2Zo/Zs8C4I/Zs2hhpfeulGLIoKd1sVu3acumjRsAOHXyJBERERQuXDjF9hN7P7EBH2DZkr+o7G7MjI/HsHBkO7OPbls6T1IBa0VEAdMSWlOZGXW3jdRMTo/yG+0jLfYXjXiL84cCCL97m7E961K/5xtcOLSbq2ePA0KBos60euOruPw/9m7Aw/BQoqMiOb7zH3oN/xXHUuVp+fow/v7+Q6IiHlDepx7lfV9I9fOkVle6X+8ebNu6mZshIVQqV5KPPvuCt9/9gD49u/H7zF9wKVGSmXPmx+Wv6lqWu/fuEhkRwYplS/hr+WrcKlW2yNfOHfG0tn1M05SGfT2cPn37MXhgf3w8qpLD3p6ffv7Nql0rn378AYcOHkBEKFWqNOMnTbWa7aTI7IMylpCs7jaAiBRXSl0RkSLAOuB189ZFCZLVdbc1yaM3uEgeoze4MPIR6lpBd7t0pWrqk9+WWZR3UI3SmVZ326LmtlLqivnndeAvwM/IQmk0muxBdpCUTTZIikgeEckXew405dFCco1Go0kQwRRgLDkyM5b0SRYF/jL3j9gBfyiljFnGotFosg+Stab1JUayQVIpdRaong5l0Wg02YysHyL1LkAajcYgtO62RqPRJEPWD5E6SGo0GsMQbDL5yLUl6CCp0WgMIXZ0O6ujg6RGozGMZ2J0W6PRaFJL1g+ROkhqUsn7Bi8bPHcjzFD7pQrnNtQ+GF+L+i8swjDbVlm2+azMk9RoNJrUIICtDpIajUaTOFk/ROogqdFoDCQbVCR1kNRoNMZgmgKU9aNkttXdvnTpEi82boBH1Up4VXdn4vhxyd+UQsb/+ANe1d3x9qhC757defDggdV9ZFbd7cR48OAB9Wr74+/jgY9HFb756gsADh06SIN6tfD1qkan9m24e/duiuw2reFO+0b+dGxaiy4t6gFw/MgherRuEJd2eL9pD9PD+wPp2LQWHZvWokOTmvyzammKfLlXLIu/d3Vq+XlRr9bjuwKO++F78uW0TZW0QmJER0dTw8eTDm1bper+y0GX6NS6KS/4V6NBTQ9mTJ0AwJHDh2jdtB6NannRp1t77pnfeWRkJG8O6U+jWl684F+NCWNTJ49rCdaSbxCRX0Tkuoj8Gy+tkIisE5FT5p8F4137SEROi8gJEXkxXrq3Wa/rtIiMFwtGlrKt7radnR0jR3/PgcPH2LxtF9OmTrKq/cuXLzN50ni27wpk74F/iY6OZuH8eVazD5lbdzsxHBwcWLlmPbsDD7Bzz37WrV1DwO5dvDp4IF99M4I9+w7Rum07fhz7XYpt/7JwBYvW7mDBStN+z98P/4whb33EorU7eO2dT/h++GcAlHerzPyVW1i0dgfTZv/FVx++SVRUVIp8rViznh0B+9iyIyAuLejSJTauX0eJEsnvvJ8SJo4fh2ulSslnTAQ7Ozu++GYUm3cfYtnarfw2Yyonjx/jvTcH8/EX37B+xz6at2rLlAljAVj+9yIiHj5k/Y59rN64i9m/zeDSxfNWepr4iMX/LOA3oNkTaR8C65VSFYD15s+ISGWgG+BuvmeyiNia75mCSUGhgvl40uZTZFvdbScnJzy9Hukbu5n1ja1JVFQU9+/fN/0MD8fJykJgmVl3OzFEhLx58wKmGktkZCQiwqmTJ6hT11QDbNSoCUv+Wpzm8osIoaH3AAi9d5ciRU0SsLly5cbOztST9PDhA6t1jH34/tt8/e0oq05rCQoKYvWqFbzcb0CqbRQt5kTV6iYpiLz58lGhohtXgy9z5vRJatSqC0Dd+o1YuewvwPTewsPDTN/bB/fJYZ+DvPnyp/1hniB2dNuSIznMSgi3nkhuC8w0n88E2sVLn6eUeqiUOgecBvxExAnIr5TaqUySDLPi3ZMoz4Tu9oXz5zlwwKRvbC2cnZ0Z+ta7VCxbkjIlnMif/zkaN2lqNfuQNXW3wdx89PWktEtRGjZqjK+fP5Xdq7BimanZu3jRwsdUGi1BRBjUox1dmtdl4exfAPjgy5F8/82nNPJ1Y8zXnzD0oy/j8h/at4e2DX1p37gGn4/4MS5oWuqrXatm1K3pyy8zTHJOK5YvpXhxZ6pWs+6uge+9M5ThI0YnqCGeGi5dPM+/hw7i6e2Hq5s7a1eZ5BOWL1nElctBALRs24HcufPg6VYKv6rlGfzaWxQsWMgq/h/Dwqa2OUYWFpHAeMcgCzwUVUoFA5h/xkpaOgPxv2BB5jRn8/mT6Uliqe52ARH5U0SOi8gxEalpyX1JkV6626GhoXTv0pHvvv+R/Pmt99fy9u3bLF+2hGOnznH24hXCwsOYO2e21exD1tPdjsXW1pZde/Zz8uwl9gbu4ciRf5ky7WemTZ1M7Ro+hIbew97ePkU2f/9rHQtXb2PK74uZO/MnAndtY/6sn/ngi5Gs33Oc978cyefvvhqXv5qXL0s27GHeik3MmDiWhynoL163cSvbdgWyeMkKfpo2hW1btzBm1Ag++XxYisqcHCtXLKeIYxG8vBPWEE8pYaGhDOzdjWEjxpAvf37GTpzGbzOm0qx+DcJCQ8mRw/TOD+zdg62tLfuOnWfXgRNMm/QjF86ftUoZniQFQTJEKeUT73hKbDAlbhNIU0mkJ4mlf77GAauVUm6YNuA9ZuF9iZIeutWRkZF079KRrt1fitM3thYb1v9D6dJlcHR0JEeOHLRr14FdO3dY1UdW091+kgIFClC33gusW7MaVzc3lq1cw/ZdgXTu0p0yZculyFaRYqam9POFHWnUrDWHD+xl6Z9/0LiFSer1xVbtOXxg71P3lavgRq7cuTl1wvK+1thuE8ciRWjdph3bt27h/Plz1PL1xL1iWS5fDqJuDR+uXb2ajKWk2bljO8uXL8W1fGl6v9SNTRs38HLvnqmyFRkZycA+XWnfuRstWrcDoHxFN+YuXsnqTbto27ELpcuUBeCvP+dRv1FTcuTIQWHHIvj61+Lg/n1pepbEsGKfZEJcMzehMf+M1TQOAkrEy+cCXDGnuySQniSWaNzkB+oBPwMopSKUUneSL3/SGK0prZRi8MD+uLpV4s233k7+hhRSokRJAgJ2ER4ejlKKjRvW4+qW+s73hMhqutsAN27c4M6dOwDcv3/f9F5c3bh+3fT9jYmJYdTI4fQf+IrFNsPDwwgz9z2Gh4exY8t6KrhWxrFoMfbs3AbA7u2bKVXGFHiDLp6PG6i5EnSR82dP4WzhYEtYWBj37t2LO1+/fh1ePj6cu3SVIyfPcuTkWZydXdi6K5CixYpZ/AwJ8fXwEZw5H8SJ0+eZNWce9Rs05NdZKW+NKKV45/VXKF/RjVdeHRqXHnLj0TsfN2YkvV4eCICzS0m2b92EUorwsDD2Be6mfAXXND1LQpg23TVUd3sp0Md83gdYEi+9m4g4iEgZTAM0AeYm+T0RqWEe1e4d755EsaSjpixwA/hVRKoDe4E3lVKPLa7NbLrbO7bH0zf29gBg2Dff0qx5C6vY9/P3p32HTtT088LOzo7q1T3pP9CSbhTLycy624lx9Wowg/r3JTo6mpiYGDp26kzzlq2YNGEc06dOBqBNu/b07vOyxTZv3rjOmwN6ABAdHUWLdl2o06AJufPkYeQXHxAVFYWDQ06+GDUegH0BO/l58ljs7HJgY2PDp8PHUrBQYYt8Xb92jR5dOwKmgbkuXbvTpGmyA6AZyp5dO1g0fw6VKlehSV1fAD787CvOnT3NbzNM+totWrWj60umeNJ3wGDeem0gDWt5opSia4/eVK5S1ZCyWWtnchGZC9TH1HcZBHwBjAQWiEh/4CLQGUApdUREFgBHgSjgVaVUtNnUEEwj5bmAVeYjad/J6W6LiA+wC6itlNotIuOAu0qpzxK7R+tuZ39iDNatzg4bXNjZGjsueivUuA0umjeoycH9e9MU4VyreKhpizZYlLeB2/NZWnc7CAhSSu02f/4T8DKuSBqNJjuQDs3tdCHZIKmUugpcEpHYTotGmKqxGo1GkwRWnUyeYVg6eex1YI6I2ANnAcs7lDQazbOJhUsOMzsWBUml1AEgU/YXaDSazEs2iJF6FyCNRmMMetNdjUajSY6sHyN1kNRoNMaR2QdlLEEHSY1GYxjZoLWtg6RGozGObBAjdZDUaDQGkg2ipA6SGo3GEESst3Y7I9FBUpMpMXpttft7Kwy1D3BsTOo0aywlt4Nt8plSidU2prCKlYxFB0mNRmMc2SBK6iCp0WgMIvOvy7YEHSQ1Go1hZIMuyeyru33yxAn8vT3ijiKF8jNh3I9W9ZEe2t4Tx4/D26MKXtXdrV5+yDq62/G5c+cOPbt3xqtaZbyru7N7106+/vJzavh4UMvPi7YtXyT4StK78kfdu8G1xZ9wZfb/CJ7zKncPmETK7uz+g8u/9CV47psEz32T++cf7YsaEXKOqwvfI3jOqwT/8ToqyrSfY8T10wT/8TpXZg3i1ubpCWoHJfd+Pv7wPTyrVsLPuzrdOneI293dUl57ZQAVSjlR0+eRUNlnH7+Pn4c7tf086dm1I/+ZbW5cv476tfyo5etB/Vp+bNlk2Z6PKUWwnu52RpLsprupwZJNd6Ojo6lauSIrVq3D2cWFOjV8mTl7LpUqV7Z6eaKjoylXypnN23dTqlQpq9kNDg7manAwnl5e3Lt3j1r+3iz482+rPcORf/+ld89ubN0RgL29PW1aNmP8xCmUr1DBKvbT8jtIbNNdpRRhYWHkzZuXyMhIGjeoy3ff/8g7b73BtyO/o269F5j52y9cOH+Oz7/8OnH7SXwvB/XvS63adejbbwARERGEh4djY2MTJ/Q2ZdIEjh87yriJUxK14fa/34kOu419kXLERIRzdf7bOLb8mLBT27DJkYv8Xu0ff66YaK7OG8rzTd7G3rEM0ffvYuOQB7Gx5er8dyhYbyD2xVy5sXQY+aq3Jldp7wQHbhJ7P3fv3qV+g4bY2dnx6ccfAPDNt6MSLT9ARHRM3Pn2bVvImycvgwe+zM7AgwBs+Gct9eqbbH7x6YcADPtmJIcO7MexSFGcihfn6JF/6dSmBUfPXHzMdoPa/uzfF5im8OVezUvNXbHForzVS+bL0pvuGoLRmtLx2bhhPWXKlrNqgATjtb2PHz+Gn18Ncuc26UjXrfcCS5b8ZTX7WVF3++7du+zYtpU+L/cHwN7engIFCjymhBkWFpas6qNtnkLYFzFp4tjY5yZHQReiQm8mmv/Bxf3kKFwae8cypvtz5UdsbIkOu0VMRDgOTm6ICHkqNeD+2V2J2kns/TRu0jRO9tbPv0aKpX1r16lHwUKPy8I2bPzIpq9vDa6YbVbz8IwTO6tU2Z0HDx/w8OHDFPmzlOxQk3wmdLcXzp9Hl67dDbEdixHa3u7uVdi2bQs3b94kPDyc1atWEnQpZXrVSZGVdLdjOX/uLIUdHRk8sB+1/b15dfBAwsJMUg/DPv8Ut3KlWDDvjxTJv0bdvUbEjbM4FDPtK33v0AqC/3idm/+MI+ZBKACRdy4DwvUlXxA8byh39y4y3Rt6E9u8j/RzbPMUJios8WALCb+f+Mz67VeavmhdXZ3Zs36lcQJaPUv/Xky16h44ODhY1V8sYuGRmbFELdFVRA7EO+6KyNC0Ok4v3e2IiAhWLF9Kh06drW47FqO0vd0qVeKddz+gVbMmtGnZjGrVqsfVDKxBVtLdjiUqKooD+/cxYNBgtu/eS548eRj7nalZ+sVX33D8zAW6dOvB9CmTLLIXE3GfGytHUrDuAGzsc5OvanOK955Gse7jsM1TiNvbfjZnjOFh8FGeb/oORTuOIvzsLh5cOpiI1WRqsQm8n1hGjxyOnZ0d3bq/ZFH5LWHMqG+xs7OjS7cej6UfO3qELz/9iB8mJN4tkSYsjZCZPEpaIt9wQinloZTyALyBcCDNbb700N0GWLN6FR6eXhQtWtTqtsFYbW+Avv36s3PPPv7ZuIWChQpRvrx1+iMha+lux+Ls7IKzs0tc7att+44cOPC4ZnSXrt1Z8nfyzXkVHUXIqpHkcX2B3OVrAWCbuyBiY4uIDXndmxJx7ZQpPe/z5CxeBdtc+bHJ4UCuUt5E3DiDXd7niQ4NibMZHRaCXZ5CCfp7kvjvB2D27zNZtXIFv8ycbbUKw9zZs1i7agXTf/39MZuXg4Lo1a0TU2b8murfhSVkB/mGlDa3GwFnlFIX0urYaE3pWBbMn2tYU9tobW8gTq/64sWLLPl7MV26We9ZsorudnyKFiuGs0sJTp48AcDmjRtwq1SZ06dPxeVZuWIZFV2T1pFWSnFz/QRyFHQhv2e7uPTosFtx5+FndpHjeVM/dq6SXkTcPE9M5ENUTDQPLh8hR8ES2OYphI19Lh5ePW4alDm2kVxlE+9ySez9rF2zmh/GjGbBoiXkzm2d1Ub/rF3NuLHf8cfCvx+z+d+dO3Tt2IbPvxpOjZq1reIrIbKLEFhK227dgLlWcWywpjRAeHg4G/5Zx8TJ06xqNxajtb0BunfpyK1bN8lhl4Mfx0+iYMGCVrOdVXS3n2TMD+MY0LcXERERlC5ThinTf+G1IQM5dfIkNjY2lChZknHJNCEfBh8j/MRGcjxfiuC5bwJQoGYvwk5uITLkHAB2+YtSqMH/ALDJmZf8Hm25tuBtQMhZ2ptcZUwa1wXrD+HWP+NQURHkLOVFzlLeKX4/VStV4GHEQ1q3aAqAn58/4ydNtfid9O/zEtu3bObmzRDcy5fiw0+/4Icxo3j48CHtW5n6In38/PlhwmR+mjqJc2dO892I4Xw3YjgAi5etwrFIEYv9WUwmD4CWYPEUILMI2BXAXSl1LYHrg4BBACVKlvQ+eSbNlU1NJsZo3e2kpgBZg+ywdjv+FCBrY40pQFWqe6k/V2+zKG+l4nmSnQIkIueBe0A0EKWU8hGRQsB8oDRwHuiilLptzv8R0N+c/w2l1JrUPEdKmtvNgX0JBUgApdR0pZSPUsrHsbBjasqi0WiyGQZMAWpgHiOJDagfAuuVUhWA9ebPiEhlTC1fd6AZMFlEUrUjSEqCZHes1NTWaDTPBukwuN0WmGk+nwm0i5c+Tyn1UCl1DjgN+KXGgUVBUkRyA02A1M0A1mg0zyaWR8nCIhIY7xiUgDUFrBWRvfGuF1VKBQOYf8Z2rDoD8SfjBpnTUoylutvhwPOpcaDRaJ5NUrjpbogFyxJrK6WuiEgRYJ2IHE/KfQJpqeroztANLjQaTfbGms1tpdQV88/rmOZq+wHXRMQJwPzzujl7EFAi3u0umAaeU4wOkhqNxjisFCVFJI+I5Is9B5oC/wJLgT7mbH2A2M0HlgLdRMRBRMoAFYCA1DyC3k9So9EYhFVX0xQF/jKvGrID/lBKrRaRPcACEekPXAQ6AyiljojIAuAoEAW8qpSKTo1jHSQ1Go1hWGs7BqXUWaB6Auk3Ma0ETOie4cDwtPrWQVKj0RiCkPm3QbMEHSQ1Go1hZPbNKyxBB0mNRmMYuiapeWaxMXrrFuOWJQPGr6sGeN7/dUPt39w9wTDb1gpu2SBG6iCp0WgMIgtIM1iCDpIajcZAsn6U1EFSo9EYQuymu1kdHSQ1Go1hZIfmdoYuS1y7ZjXV3F1xdyvPd6NHavtP8MqAfpQsXgRvjypWtx3L+B9/wKu6O94eVejdszsPHjxIk72Eyrzoz4V4VXcnt70NewOT1mNPjgcPHlCvtj/+Ph74eFThm6++AODgwQPUr1uTGr6e1KnpS+CeVK1AI+jSJZo3bYhXtcr4eFRh0oRxABw6dJAG9Wrh61WNTu3bcPfu3WTL+fDkQh4en8fD438QGbwbgMjg3ea0eUScWYqKNCk9xjy8y4ODU+OuRV7aFGfr4am/eHhsTtw1FRme4vfz1Zef4eddnRq+nrRu8SLBV1K1jDnFPIsaN1YjOjqaoW+8ypJlq9h/6CgL583l2NGj2n48evXpy5Llq61qMz6XL19m8qTxbN8VyN4D/xIdHc3C+fPSZDOhMru7V2HegsVxuttpwcHBgZVr1rM78AA79+xn3do1BOzexacffcBHn3zOrj37+fTzYXz68Qepsm9rZ8e3o8aw79BRNm7dyfSpkzl27CivDh7IV9+MYM++Q7Ru244fx36XbDnty7XFwa0b9q5dibl3kZiwq9gV8cTBrRsObt2wyV+KqKt74u4Rh+firuUoUf8xezlKNYm7JjkS18BJ7P0Mffs9AvYeZNee/TRv0ZIRw79K1ftJMc+CWqJR7AkIoFy58pQpWxZ7e3s6d+3G8mVLkr/xGbEPUKduPQoVskx5L7VERUVx//5908/w8DjR+tSSUJndKlVKVpjLUkSEvHnzAialysjISEQEEeHePVPt7u7d/yjmlLrncHJywtPTC4B8+fLh6laJK5cvc+rkibgg36hRE5b8lfTWqiKC2Jplc1WM6YBHaQAxUakqY3J+E3o/8aWOw8LDDJFvTrA8Fh6ZmQzrk7xy5TIuLo92MnJ2diEgYLe2n444Ozsz9K13qVi2JLly5aJR46Y0btI0o4uVLNHR0dSu4cPZM6cZNPh/+Pr5M3rMD7Rt3YyPP3yPmJgYNmzanmY/F86f5+DB/fj6+VPZvQorli2lVZu2LF608DEp3sRQKoaIEwtQEf9hW7gqNnmKARAZvIvoWycQW3vsy7d7lD/iLg9PzAcbe3I4+WOT91Ggj7y4HhBsC5TDtqhPkkEuofcD8OXnn/DHnN/Jn/85Vq3dkLqXkgJSIc2QKbF0Z/K3ROSIiPwrInNFJGdaHSckQGbNv25Z3X56cPv2bZYvW8KxU+c4e/EKYeFhzJ0zO6OLlSy2trbs2rOfk2cvsTdwD0eO/MuM6VMY9d1YTp65yKjvxjLklQFp8hEaGkqPbp0YPeYH8ufPz5RpPzNt6mRq1/AhNPQe9vb2ydoQsTE1kSv3RYVfJ+b+TQByONUgp3sfbAtWJOrGIVPeHHlwqNwHB9eu5HCuTcSFdajoCADsSzXBwa079hU6EBMWTMztEyl+PwBffjWck2cu0rV7D6ZNmZiW12MxsbX85I7MTLJBUkScgTcAH6VUFcAWk8BOmnB2dnnsr/Hly0EUT2NTLzvZTw82rP+H0qXL4OjoSI4cOWjXrgO7du7I6GJZTIECBahb7wXWrVnNnNmzaNuuAwAdOnZmb2DqBm7A1Ezt0bUTXbv1iLPp6ubGspVr2L4rkM5dulOmbDmL7YmdAzZ5ixNz7+Jj6bYFKxDz31lTHhtbxM5U97DJXQSxz496eMd0zd7UfBZbe2wKVCAm/DqWEP/9xKdr1x78nUx3gbXIDs1tS/sk7YBcImIH5CaVO/zGx8fXl9OnT3H+3DkiIiJYOH8eLVu1SavZbGM/PShRoiQBAbsIDw9HKcXGDetxdauU0cVKkhs3bnDnzh0A7t+/byqzqxtOTsXZumUzAJs2bqBc+Qqpsq+UYsgrA3B1c+ONoW/HpV+/bgpMMTExjBo5nP4DX0m2nCrqoclmTBTR94IQh4LEmAMfQPR/5xEHk466irqPMvdbxjz8DxXxnylQqhhU1H1z2aKJuXsByZl4P3Vi7+f0qVNxeVYsX4qrq5uFbyRtGKCWmO4k2yeplLosImMwbWh5H1irlFr7ZL4ndLeTd2xnxw/jJtK65YtER0fTp28/Kru7p/gBsqt9gN49u7N18yZCQkIoV9qFzz4fRt9+/a1m38/fn/YdOlHTzws7OzuqV/ek/8CE9JcsJ6EyFyxUiLeHvk7IjRt0aNuSatU9WLYyVRLIXL0azKD+fYmOjiYmJoaOnTrTvGUrnitQgPfeGUpUVBQ5c+Zk4uRpqbK/c8d25s75HfcqVanh6wmYmqlnTp9i+tTJALRp157efV5OtpwRZ/4GpQCFbYHy2D5Xmohzq8w1REHs85HD5QUAYkKvEHV1N2ADIuRweQGxy4mKjiTizDLzwE8MNnlLYPt85RS/nx5dO3Hy5AlsbGwoWbIU4ydOSdX7SRmZf3qPJUhCfWuPZRApCCwCugJ3gIXAn0qpRDuvvL191PbdaZsPp3m2iYlJlWZTpiIrb3BRp6Yv+/YGpinCeXr5qA3bLBvMLJTHbq8FQmAZgiXN7cbAOaXUDaVUJCZZ2VrGFkuj0WQHnonmNqZmdg2z9vZ9TFul62qiRqNJluzQ3LakT3K3iPwJ7MMkqLMfmG50wTQaTRYnC9QSLcGiyeRKqS+ALwwui0ajyUZkhek9lqB3AdJoNMaRDaKkDpIajcYwnok+SY1Go0kt2WHT3QzdT1Kj0WRzrLQuUUSaicgJETktIh8aVt4E0EFSo9EYhjU23RURW2AS0ByoDHQXkcSXHVkZHSQ1Go0hCFabTO4HnFZKnVVKRQDzgLYGFz+OZJclpsqoyA3gQgpuKQyEWL0g6Ycuf8aT1Z8hs5W/lFLKMS0GRGQ1pueyhJxAfO2Q6Uqp6WY7nYBmSqkB5s+9AH+l1GtpKZ+lGDJwk9KXKyKBmXXdpiXo8mc8Wf0Zsnr5E0Ip1cxKphKqa6bb4n7d3NZoNJmdIKBEvM8uWGG7RkvRQVKj0WR29gAVRKSMiNhj2vR7aXo5zyzzJLP6WnBd/ownqz9DVi+/YSilokTkNWANJmWEX5RSR9LLvyEDNxqNRpNd0M1tjUajSQIdJDUajSYJMjRIZuRSI2sgIiVEZKOIHDNL7r6Z0WVKDSJiKyL7RWR5RpclpYhIARH5U0SOm38PNTO6TCnBCLlmjXXJsCCZ0UuNrEQU8I5SqhJQA3g1Cz4DwJvAsYwuRCoZB6xWSrkB1clCz2GUXLPGumRkTTJDlxpZA6VUsFJqn/n8Hqb/oM4ZW6qUISIuQEtgRkaXJaWISH6gHvAzgFIqQil1J0MLlXKsLtessS4ZGSSdgUvxPgeRxQJMfESkNOAJWCYPl3n4EXgfiMngcqSGssAN4Fdzd8EMEcmT0YWyFKXUZSBWrjkY+C8huWZNxpKRQTJDlxpZExHJi0l2d6hS6m5Gl8dSRKQVcF0ptTejy5JK7AAvYIpSyhMIA7JM37ZZrrktUAYoDuQRkZ4ZWyrNk2RkkMzQpUbWQkRyYAqQc5RSizO6PCmkNtBGRM5j6u5oKCKJ6qlnQoKAIKVUbO39T0xBM6ug5ZqzABkZJDN0qZE1EBHB1B92TCk1NqPLk1KUUh8ppVyUUqUxvf8NSqksU5NRSl0FLomIqzmpEXA0A4uUUuLkms3fpUZkoYGnZ4UMW5aY0UuNrERtoBdwWEQOmNM+VkqtzLgiPXO8Dswx/6E9C7ycweWxGC3XnDXQyxI1Go0mCfSKG41Go0kCHSQ1Go0mCXSQ1Gg0miTQQVKj0WiSQAdJjUajSQIdJDUajSYJdJDUaDSaJPg/2IbJtvNe8NkAAAAASUVORK5CYII=\n",
|
|
"text/plain": [
|
|
"<Figure size 432x288 with 2 Axes>"
|
|
]
|
|
},
|
|
"metadata": {
|
|
"needs_background": "light"
|
|
},
|
|
"output_type": "display_data"
|
|
}
|
|
],
|
|
"source": [
|
|
"import itertools\n",
|
|
"\n",
|
|
"plt.title(\"confusion matrix\")\n",
|
|
"plt.imshow(confusion,cmap='Blues')\n",
|
|
"\n",
|
|
"plt.colorbar()\n",
|
|
"for i,j in itertools.product(range(confusion.shape[0]),range(confusion.shape[1])):\n",
|
|
" plt.text(j,i,\"{:}\".format(confusion[i,j]),horizontalalignment=\"center\",color=\"black\" if i == j else \"black\")\n",
|
|
"plt.show()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 37,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def getF1Score(confusion,c):\n",
|
|
" TP = confusion[c,c]\n",
|
|
" FP = confusion[c].sum() - TP\n",
|
|
" FN = confusion[:,c].sum() - TP\n",
|
|
" precision = TP / (TP + FP)\n",
|
|
" recall = TP / (TP + FN)\n",
|
|
"\n",
|
|
" f1Score = (2*precision*recall)/(precision + recall)\n",
|
|
" return f1Score"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 39,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"class 0 f1 score : nan\n",
|
|
"class 1 f1 score : 0.9293768405914307\n",
|
|
"class 2 f1 score : 0.8267770409584045\n",
|
|
"class 3 f1 score : 0.9029303789138794\n",
|
|
"class 4 f1 score : 0.9614078402519226\n",
|
|
"class 5 f1 score : 0.9060906171798706\n",
|
|
"class 6 f1 score : 0.6701149344444275\n",
|
|
"class 7 f1 score : 0.9169055223464966\n",
|
|
"class 8 f1 score : 0.9731689691543579\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"for i in range(tagIdConverter.size - 1):\n",
|
|
" f1 = getF1Score(confusion,i)\n",
|
|
" print(f\"class {i} f1 score : {f1}\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 40,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import collections"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 41,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"20744it [00:00, 170034.81it/s]"
|
|
]
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"token \t count frequency%\n",
|
|
"O(9) \t 314867 77.841%\n",
|
|
"I-PER(8) \t 21118 5.221%\n",
|
|
"I-ORG(7) \t 16329 4.037%\n",
|
|
"I-LOC(5) \t 10922 2.700%\n",
|
|
"B-LOC(1) \t 10645 2.632%\n",
|
|
"B-PER(4) \t 10059 2.487%\n",
|
|
"B-ORG(3) \t 9322 2.305%\n",
|
|
"I-MISC(6) \t 6176 1.527%\n",
|
|
"B-MISC(2) \t 5062 1.251%\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"counter = collections.Counter()\n",
|
|
"total_l = 0\n",
|
|
"\n",
|
|
"for item in tqdm(itertools.chain(datasetTrain,datasetDev,datasetTest)):\n",
|
|
" entities = item[\"entity\"]\n",
|
|
" for entity in entities:\n",
|
|
" counter[entity] += 1\n",
|
|
" total_l += len(entities)\n",
|
|
"print(f\"{'token':<12}\\t{'count':>12} {'frequency%':>12}\")\n",
|
|
"for token,count in counter.most_common():\n",
|
|
" tid = tagIdConverter.convert_tokens_to_ids([token])[0]\n",
|
|
" print(f\"{f'{token}({tid})':<12}\\t{count:>12}{count*100/total_l:>12.3f}%\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3 (ipykernel)",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.7.11"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 2
|
|
}
|