Bert Model -Final.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import libraries and dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 206
    },
    "id": "srdNnfLmDFyV",
    "outputId": "4e959e93-82ce-4f63-8fe6-d89c2cc09ccf"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-dba690d0-ece6-4e29-a7d6-af0d26ce6a21\">\n",
       "    <div class=\"colab-df-container\">\n",
       "      <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>title</th>\n",
       "      <th>author</th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>House Dem Aide: We Didn’t Even See Comey’s Let...</td>\n",
       "      <td>Darrell Lucus</td>\n",
       "      <td>House Dem Aide: We Didn’t Even See Comey’s Let...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>FLYNN: Hillary Clinton, Big Woman on Campus - ...</td>\n",
       "      <td>Daniel J. Flynn</td>\n",
       "      <td>Ever get the feeling your life circles the rou...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Why the Truth Might Get You Fired</td>\n",
       "      <td>Consortiumnews.com</td>\n",
       "      <td>Why the Truth Might Get You Fired October 29, ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>15 Civilians Killed In Single US Airstrike Hav...</td>\n",
       "      <td>Jessica Purkiss</td>\n",
       "      <td>Videos 15 Civilians Killed In Single US Airstr...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Iranian woman jailed for fictional unpublished...</td>\n",
       "      <td>Howard Portnoy</td>\n",
       "      <td>Print \\nAn Iranian woman has been sentenced to...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-dba690d0-ece6-4e29-a7d6-af0d26ce6a21')\"\n",
       "              title=\"Convert this dataframe to an interactive table.\"\n",
       "              style=\"display:none;\">\n",
       "        \n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
       "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
       "  </svg>\n",
       "      </button>\n",
       "      \n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      flex-wrap:wrap;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "      <script>\n",
       "        const buttonEl =\n",
       "          document.querySelector('#df-dba690d0-ece6-4e29-a7d6-af0d26ce6a21 button.colab-df-convert');\n",
       "        buttonEl.style.display =\n",
       "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "        async function convertToInteractive(key) {\n",
       "          const element = document.querySelector('#df-dba690d0-ece6-4e29-a7d6-af0d26ce6a21');\n",
       "          const dataTable =\n",
       "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                     [key], {});\n",
       "          if (!dataTable) return;\n",
       "\n",
       "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "            + ' to learn more about interactive tables.';\n",
       "          element.innerHTML = '';\n",
       "          dataTable['output_type'] = 'display_data';\n",
       "          await google.colab.output.renderOutput(dataTable, element);\n",
       "          const docLink = document.createElement('div');\n",
       "          docLink.innerHTML = docLinkHtml;\n",
       "          element.appendChild(docLink);\n",
       "        }\n",
       "      </script>\n",
       "    </div>\n",
       "  </div>\n",
       "  "
      ],
      "text/plain": [
       "   id                                              title              author  \\\n",
       "0   0  House Dem Aide: We Didn’t Even See Comey’s Let...       Darrell Lucus   \n",
       "1   1  FLYNN: Hillary Clinton, Big Woman on Campus - ...     Daniel J. Flynn   \n",
       "2   2                  Why the Truth Might Get You Fired  Consortiumnews.com   \n",
       "3   3  15 Civilians Killed In Single US Airstrike Hav...     Jessica Purkiss   \n",
       "4   4  Iranian woman jailed for fictional unpublished...      Howard Portnoy   \n",
       "\n",
       "                                                text  label  \n",
       "0  House Dem Aide: We Didn’t Even See Comey’s Let...      1  \n",
       "1  Ever get the feeling your life circles the rou...      0  \n",
       "2  Why the Truth Might Get You Fired October 29, ...      1  \n",
       "3  Videos 15 Civilians Killed In Single US Airstr...      1  \n",
       "4  Print \\nAn Iranian woman has been sentenced to...      1  "
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np \n",
    "import pandas as pd\n",
    "!conda install -y pytorch torchvision cudatoolkit=10.1 -c pytorch\n",
    "!pip install transformers\n",
    "from transformers import BertTokenizer, BertForSequenceClassification\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "import matplotlib.pyplot as plt\n",
    "from transformers import BertTokenizer, BertForSequenceClassification\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd \n",
    "\n",
    "\n",
    "data = pd.read_csv('train.csv')\n",
    "\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "PB6MjBOGDFyY",
    "outputId": "0dbfc982-8d7d-4ed8-e3d1-ba4948b995c9"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20800\n"
     ]
    }
   ],
   "source": [
    "print(len(data))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "kJ6ciRadDFyY"
   },
   "source": [
    "## Split the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "UqmcSk65DFyk",
    "outputId": "ac524b8d-7ccd-4d71-a3bc-b5e19508813c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Size of training set: 12480\n",
      "Size of validation set: 4160\n",
      "Size of testing set: 4160\n"
     ]
    }
   ],
   "source": [
    "train_data, validate_data, test_data = np.split(data.sample(frac=1), [int(.6*len(data)), int(.8*len(data))])\n",
    "\n",
    "train_data = train_data.reset_index(drop=True)\n",
    "validate_data = validate_data.reset_index(drop=True)\n",
    "test_data = test_data.reset_index(drop=True)\n",
    "\n",
    "print(\"Size of training set: {}\".format(len(train_data)))\n",
    "print(\"Size of validation set: {}\".format(len(validate_data)))\n",
    "print(\"Size of testing set: {}\".format(len(test_data)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "azcrcZNPDFyr",
    "outputId": "ab203e82-7395-4ad2-eaca-17b09bf8b8d0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/bin/bash: conda: command not found\n",
      "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
      "Collecting transformers\n",
      "  Downloading transformers-4.28.1-py3-none-any.whl (7.0 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.0/7.0 MB\u001b[0m \u001b[31m90.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.27.1)\n",
      "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (23.1)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2022.10.31)\n",
      "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.22.4)\n",
      "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.65.0)\n",
      "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.12.0)\n",
      "Collecting huggingface-hub<1.0,>=0.11.0\n",
      "  Downloading huggingface_hub-0.14.1-py3-none-any.whl (224 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m224.5/224.5 kB\u001b[0m \u001b[31m31.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting tokenizers!=0.11.3,<0.14,>=0.11.1\n",
      "  Downloading tokenizers-0.13.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.8 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m120.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0)\n",
      "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.11.0->transformers) (2023.4.0)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.11.0->transformers) (4.5.0)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (1.26.15)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2022.12.7)\n",
      "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.0.12)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.4)\n",
      "Installing collected packages: tokenizers, huggingface-hub, transformers\n",
      "Successfully installed huggingface-hub-0.14.1 tokenizers-0.13.3 transformers-4.28.1\n"
     ]
    }
   ],
   "source": [
    "!conda install -y pytorch torchvision cudatoolkit=10.1 -c pytorch\n",
    "!pip install transformers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load the Pretrained model and add 3 layers to customize it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 252,
     "referenced_widgets": [
      "3dd0e03a3e3e48d99c1f8f5014cfa489",
      "bab4328b642a4dd99c30320267b25bef",
      "2030babffefa490e978b2821379db31e",
      "1f8817ce2adc41f3b4551593bca5a7cb",
      "1afe0d53ff7a491289188d006854bb51",
      "a891f9ad215e4ad9b52ef55029d91642",
      "a6f322562ddc481fa10495bc791d3877",
      "3432d85ffbe042b1b704a80dd32a4344",
      "22a8f852ea1d4916800b7a2e871a1d5d",
      "6ee4b55cbd3543898d1784a644a7709c",
      "a2e89bfe27b24215b52e6c07f228708d",
      "510b7278236b47648152af204c6bf814",
      "67e7897e39bc4bb199624d64946aa5aa",
      "c0df9cf96bdb4fc59e819e5e4d091b3c",
      "baf9fcf55c5e41b990703851f7885c08",
      "c46a91dd682a4d068307130e386e5fda",
      "b67cc694d4174d44a061982a8f6ec257",
      "83263151fcea4d05813186cef90a9195",
      "7f7f10cf3b4f41218fedc1738eb19d4d",
      "ef75d6657b35415781c0e749192ab48b",
      "7dadf6731b2c4fba922be12ec00117d8",
      "23ff8d463e0141169dac6ff90f1c5b51",
      "c3c353aa1d124509afebe084c07d7092",
      "c733eac88cc34337b6b4b694eac5026d",
      "2fea4c3af4e14fb890e8c134de63e155",
      "bbbc3b7bb5194588a6625bdb64d127c5",
      "2ca0b99eab7948bd9d47e902ce8e24b1",
      "d0dc819b2b9247d185d07eb47d406201",
      "1c4802f3bc1d49668f64e12760fdc507",
      "0ef749989aea417d85312c9cd35fae92",
      "f1fc1da606844656bb4f4bc942494efc",
      "2afee066d10144079c8c5f62abd687aa",
      "8df05f505d584e4a9c226f7f7dc61ccd",
      "1f81e5915f874b5587dabed20015cb44",
      "4c936325266b48f18169c318997b28cc",
      "9b527fd28ec04babb148c619acc41812",
      "b5df15a216b24e8b8ae7bf1c273d6d80",
      "13a152edcd754d548380a76eef3b2e93",
      "c11bc5cc20c24d03b6bbd7da89443c2f",
      "e4ef2cda58c14f8bb5c2c32fe76a45ef",
      "8d10a7496eae433998d009b206963194",
      "5b11bcff43ed410c85eaeda499954dcf",
      "d80f8795d39043259e7b448f640b5532",
      "5eb0cb0e927b4275988f0eb6b27f55fc"
     ]
    },
    "id": "gwI1hlX7DFyr",
    "outputId": "877f541b-de54-4b6a-e84b-574468988878"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3dd0e03a3e3e48d99c1f8f5014cfa489",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)solve/main/vocab.txt:   0%|          | 0.00/232k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "510b7278236b47648152af204c6bf814",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)okenizer_config.json:   0%|          | 0.00/28.0 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c3c353aa1d124509afebe084c07d7092",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)lve/main/config.json:   0%|          | 0.00/570 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1f81e5915f874b5587dabed20015cb44",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading pytorch_model.bin:   0%|          | 0.00/440M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.dense.bias', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.decoder.weight']\n",
      "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n",
    "\n",
    "tokenizer = BertTokenizer.from_pretrained(\"bert-base-uncased\")\n",
    "model = BertForSequenceClassification.from_pretrained(\"bert-base-uncased\")\n",
    "model.config.num_labels = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "iOlB3UPIDFys"
   },
   "outputs": [],
   "source": [
    "# Freeze the pre trained parameters\n",
    "for parameter in model.parameters():\n",
    "    parameter.requires_grad = False\n",
    "\n",
    "# Add 3 new layers at the end of the network\n",
    "model.classifier = nn.Sequential(nn.Linear(768, 256), nn.ReLU(), nn.Linear(256, 64), \n",
    "                                 nn.ReLU(), nn.Linear(64, 2),nn.Softmax(dim=1))\n",
    "\n",
    "model = model.to(device)\n",
    "\n",
    "criteria = nn.MSELoss().to(device)\n",
    "model_optimizer = optim.SGD(model.classifier.parameters(), lr=0.01)\n",
    "\n",
    "criterion = nn.MSELoss().to(device)\n",
    "optimizer = optim.SGD(model.classifier.parameters(), lr=0.01)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocess Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "P7AmTvgLDFyt"
   },
   "outputs": [],
   "source": [
    "def preprocess_text(text):\n",
    "    parts = []\n",
    "\n",
    "    text_len = len(text.split(' '))\n",
    "    delta = 300\n",
    "    max_parts = 5\n",
    "    nb_cuts = int(text_len / delta)\n",
    "    nb_cuts = min(nb_cuts, max_parts)\n",
    "    \n",
    "    \n",
    "    for i in range(nb_cuts + 1):\n",
    "        text_part = ' '.join(text.split(' ')[i * delta: (i + 1) * delta])\n",
    "        parts.append(tokenizer.encode(text_part, return_tensors=\"pt\", max_length=500).to(device))\n",
    "\n",
    "    return parts"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "hy8elG3bDFyt",
    "outputId": "e7d73725-4406-4561-ab54-c632c10b85ad"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1000/12480. Average loss: 0.2394493551105261\n",
      "2000/12480. Average loss: 0.22700477093574592\n",
      "3000/12480. Average loss: 0.21732943483191774\n",
      "4000/12480. Average loss: 0.20885038024974348\n",
      "5000/12480. Average loss: 0.19958012448185763\n",
      "6000/12480. Average loss: 0.1974432807917474\n",
      "7000/12480. Average loss: 0.1943973993192776\n",
      "8000/12480. Average loss: 0.19904647997135907\n",
      "9000/12480. Average loss: 0.18181000678936834\n",
      "10000/12480. Average loss: 0.18721263656649534\n",
      "11000/12480. Average loss: 0.17344970407818164\n",
      "12000/12480. Average loss: 0.18195950743490538\n"
     ]
    }
   ],
   "source": [
    "iter_size = 1000\n",
    "\n",
    "total_loss = 0\n",
    "all_losses = []\n",
    "\n",
    "CUDA_LAUNCH_BLOCKING=1\n",
    "\n",
    "model.train()\n",
    "\n",
    "for idx, row in train_data.iterrows():\n",
    "    text_parts = preprocess_text(str(row['text']))\n",
    "    label = torch.tensor([row['label']]).long().to(device)\n",
    "\n",
    "    optimizer.zero_grad()\n",
    "\n",
    "    overall_output = torch.zeros((1, 2)).float().to(device)\n",
    "    for part in text_parts:\n",
    "        if len(part) > 0:\n",
    "            try:\n",
    "                input = part.reshape(-1)[:512].reshape(1, -1)\n",
    "                # print(input.shape)\n",
    "                overall_output += model(input, labels=label)[1].float().to(device)\n",
    "            except Exception as e:\n",
    "                print(str(e))\n",
    "\n",
    "#     overall_output /= len(text_parts)\n",
    "    overall_output = F.softmax(overall_output[0], dim=-1)\n",
    "\n",
    "    if label == 0:\n",
    "        label = torch.tensor([1.0, 0.0]).float().to(device)\n",
    "    elif label == 1:\n",
    "        label = torch.tensor([0.0, 1.0]).float().to(device)\n",
    "\n",
    "    # print(overall_output, label)\n",
    "\n",
    "    loss = criterion(overall_output, label)\n",
    "    total_loss += loss.item()\n",
    "    \n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "\n",
    "    if idx % iter_size == 0 and idx > 0:\n",
    "        average_loss = total_loss / iter_size\n",
    "        print(\"{}/{}. Average loss: {}\".format(idx, len(train_data), average_loss))\n",
    "        all_losses.append(average_loss)\n",
    "        total_loss = 0\n",
    "        \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 504
    },
    "id": "c1_3u6CrDFyu",
    "outputId": "4d6c3706-1c18-4f4c-9ab8-01bb5399f9a5"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:matplotlib.legend:No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7f1c77905e70>]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkAAAAGwCAYAAABB4NqyAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABWWUlEQVR4nO3deVhU9eLH8ffMsCMgiLIoCm65o4niklnJ1cprWZrlNTXbbta1jOqq3au2XpfMuqZXy7Jsu9pm2aYZaaWRKIiau7khCojKriwz8/uD4v7IDWQ5M/B5Pc88T545c+YzVMzHc77n+zXZ7XY7IiIiIvWI2egAIiIiIrVNBUhERETqHRUgERERqXdUgERERKTeUQESERGRekcFSEREROodFSARERGpd1yMDuCIbDYbx44dw8fHB5PJZHQcERERqQC73U5ubi6hoaGYzRc/x6MCdB7Hjh0jLCzM6BgiIiJyGVJSUmjWrNlF91EBOg8fHx+g9Afo6+trcBoRERGpiJycHMLCwsq+xy9GBeg8fr/s5evrqwIkIiLiZCoyfEWDoEVERKTeUQESERGRekcFSEREROodjQESERERh2G1WikuLj7vc66urlgslmp5HxUgERERMZzdbictLY2srKyL7tewYUOCg4OrPE+fCpCIiIgY7vfy06RJE7y8vM4pOHa7nYKCAjIyMgAICQmp0vupAImIiIihrFZrWflp1KjRBffz9PQEICMjgyZNmlTpcphDDIJesGAB4eHheHh4EB0dTUJCwgX3Xbx4Mf369cPf3x9/f39iYmIuuv8DDzyAyWTi5ZdfroHkIiIiUlW/j/nx8vK65L6/73OhcUIVZXgBWr58ObGxsUyfPp2kpCQiIyMZNGhQ2SmuP1q3bh0jR45k7dq1xMfHExYWxsCBA0lNTT1n3xUrVvDzzz8TGhpa0x9DREREqqgi43qqa41OwwvQ3Llzue+++xg3bhwdOnRg0aJFeHl5sWTJkvPu/9577/Hggw/StWtX2rVrx+uvv47NZiMuLq7cfqmpqUyYMIH33nsPV1fX2vgoIiIi4iQMLUBFRUUkJiYSExNTts1sNhMTE0N8fHyFjlFQUEBxcTEBAQFl22w2G6NHj+aJJ56gY8eOlzxGYWEhOTk55R4iIiJSdxlagDIzM7FarQQFBZXbHhQURFpaWoWOMWnSJEJDQ8uVqFmzZuHi4sLDDz9coWPMmDEDPz+/sodWghcREanbDL8EVhUzZ85k2bJlrFixAg8PDwASExP597//zVtvvVXh64RTpkwhOzu77JGSklJjmbemZJGRe7bGji8iIuKs7HZ7texTEYYWoMDAQCwWC+np6eW2p6enExwcfNHXzpkzh5kzZ/LNN9/QpUuXsu0//vgjGRkZNG/eHBcXF1xcXDh8+DCPPfYY4eHh5z2Wu7t72crvNbkC/NKfDnHLfzbw1ModNXJ8ERERZ/T7WN2CgoJL7vv7PlUd32toAXJzc6N79+7lBjD/PqC5d+/eF3zd7NmzefbZZ1m1ahVRUVHlnhs9ejTbtm0jOTm57BEaGsoTTzzB6tWra+yzVERUuD8mk4mvtqex6peKXeITERGp6ywWCw0bNiQjI4OTJ09y5swZzp49W+5x5swZTp48SUZGBg0bNqzykhiGT4QYGxvL2LFjiYqKomfPnrz88svk5+czbtw4AMaMGUPTpk2ZMWMGUDq+Z9q0abz//vuEh4eXjRVq0KABDRo0oFGjRudMouTq6kpwcDBXXHFF7X64P+gY6sdfr27Jf9b9yrTPfqF3q0b4eeoONRERkd+v/FxoGpzf/b4URlUZXoBuv/12Tpw4wbRp00hLS6Nr166sWrWqbGD0kSNHMJv/d6Jq4cKFFBUVMXz48HLHmT59Ok899VRtRr8sDw9ow6pf0jiQmc+Mr3Yxc1iXS79IRESkjjOZTISEhNCkSZNaWQzVZK+u0UR1SE5ODn5+fmRnZ9fIeKCEg6cY8Wrpbf7v3RtN39aB1f4eIiIi9U1lvr+d+i4wZ9UzIoDRvVoAMOWT7ZwpshqcSEREpH5RATLI36+/ghA/D46cKmDumj1GxxEREalXVIAM4uPhyvO3dALgjfUHSU7JMjaQiIhIPaICZKDr2gUxtGsoNjtM+mgbRSU2oyOJiIjUCypABps2pCMB3m7sSc9l0fe/Gh1HRESkXlABMliAtxvTh3QA4JXv9rEvPdfgRCIiInWfCpADuCkylAHtmlBstTPp421YbZqZQEREpCapADkAk8nEc7d0ooG7C0lHsng7/pDRkUREROo0FSAHEeLnyeQb2gEwe9UeUk5dekE4ERERuTwqQA7kLz2b0zMigDPFVp5csR1N0i0iIlIzVIAciNlsYuatnXFzMfPjvkw+Tko1OpKIiEidpALkYFo2bsCjMW0BePaLnZzILTQ4kYiISN2jAuSA7usXQcdQX7LPFPPUyh1GxxEREalzVIAckIvFzKxhXbCYTXy5/Tird6QZHUlERKROUQFyUJ2a+vHXq1sCMPXTX8g+U2xwIhERkbpDBciBPTygDS0DvcnILWTGV7uMjiMiIlJnqAA5MA9XCzOHdQFg2aYUftqfaXAiERGRukEFyMH1jAhgdK8WAEz+ZDtniqwGJxIREXF+KkBO4O/XX0GInwdHThUwd80eo+OIiIg4PRUgJ+Dj4crzt3QC4I31B9makmVsIBERESenAuQkrmsXxNCuodjsMOnjbRSV2IyOJCIi4rRUgJzItCEdCfB2Y3daLou+/9XoOCIiIk5LBciJBHi7MX1IBwBe+W4f+9JzDU4kIiLinFSAnMxNkaFc164JxVY7kz7ehtWmFeNFREQqSwXIyZhMJp4b2okG7i4kHcni7fhDRkcSERFxOipATii0oSeTb2gHwAur95ByqsDgRCIiIs5FBchJ/aVnc3pGBFBQZOXJFdux23UpTEREpKJUgJyU2Wxi5q2dcXMx8+O+TD5OSjU6koiIiNNQAXJiLRs34NGYtgA8+8VOTuQWGpxIRETEOagAObn7+kXQMdSX7DPFPLVyh9FxREREnIIKkJNzsZiZNawLFrOJL7cfZ/WONKMjiYiIODwVoDqgU1M//np1SwCmfvoL2WeKDU4kIiLi2FSA6oiHB7ShZaA3GbmFzPhql9FxREREHJoKUB3h4Wph5rAuACzblMJP+zMNTiQiIuK4VIDqkJ4RAdzZqzkAkz/Zzpkiq8GJREREHJMKUB0z6fp2hPh5cORUAXPX7DE6joiIiENSAapjfDxcef6WTgC8sf4gW1OyjA0kIiLigFSA6qDr2gVxc9dQbHaY9PE2ikpsRkcSERFxKCpAddS0P3cgwNuN3Wm5LPr+V6PjiIiIOBQVoDqqUQN3pg/pAMD87/azLz3X4EQiIiKOwyEK0IIFCwgPD8fDw4Po6GgSEhIuuO/ixYvp168f/v7++Pv7ExMTc87+Tz31FO3atcPb27tsn40bN9b0x3A4N0WGcl27JhRZbUz6eBtWm1aMFxERAQcoQMuXLyc2Npbp06eTlJREZGQkgwYNIiMj47z7r1u3jpEjR7J27Vri4+MJCwtj4MCBpKb+bzX0tm3bMn/+fLZv38769esJDw9n4MCBnDhxorY+lkMwmUw8N7QTDdxdSDqSxTvxh4yOJCIi4hBMdrvd0NMC0dHR9OjRg/nz5wNgs9kICwtjwoQJTJ48+ZKvt1qt+Pv7M3/+fMaMGXPefXJycvDz8+Pbb79lwIABlzzm7/tnZ2fj6+tbuQ/kgN79+TD//PQXvNwsrJ54NWEBXkZHEhERqXaV+f429AxQUVERiYmJxMTElG0zm83ExMQQHx9foWMUFBRQXFxMQEDABd/jtddew8/Pj8jIyPPuU1hYSE5OTrlHXfKXns3pGRFAQZGVJ1dsx+DOKyIiYjhDC1BmZiZWq5WgoKBy24OCgkhLq9iq5pMmTSI0NLRciQL44osvaNCgAR4eHrz00kusWbOGwMDA8x5jxowZ+Pn5lT3CwsIu7wM5KLPZxMxbO+PmYubHfZl8kpR66ReJiIjUYYaPAaqKmTNnsmzZMlasWIGHh0e556699lqSk5P56aefuP766xkxYsQFxxVNmTKF7OzsskdKSkptxK9VLRs3YGJMGwCe+WInJ3ILDU4kIiJiHEMLUGBgIBaLhfT09HLb09PTCQ4Ovuhr58yZw8yZM/nmm2/o0qXLOc97e3vTunVrevXqxRtvvIGLiwtvvPHGeY/l7u6Or69vuUdddF+/lnQM9SX7TDFPrdxhdBwRERHDGFqA3Nzc6N69O3FxcWXbbDYbcXFx9O7d+4Kvmz17Ns8++yyrVq0iKiqqQu9ls9koLKzfZz1cLWZmDeuCxWziy+3HWb2jYpcZRURE6hrDL4HFxsayePFili5dyq5duxg/fjz5+fmMGzcOgDFjxjBlypSy/WfNmsXUqVNZsmQJ4eHhpKWlkZaWRl5eHgD5+fk8+eST/Pzzzxw+fJjExETuvvtuUlNTue222wz5jI6kU1M/7r+6JQBTP/2F7DPFBicSERGpfYYXoNtvv505c+Ywbdo0unbtSnJyMqtWrSobGH3kyBGOHz9etv/ChQspKipi+PDhhISElD3mzJkDgMViYffu3QwbNoy2bdsyZMgQTp48yY8//kjHjh0N+YyO5pEBbWgZ6E1GbiEzvtpldBwREZFaZ/g8QI6ors0DdD4JB08x4tXSqQbevzeaPq3Pf4eciIiIs3CaeYDEOD0jArizV3MAJn+ynTNFVoMTiYiI1B4VoHps0vXtCPHz4MipAl76dq/RcURERGqNClA95uPhyvO3dALg9R8PsDUly9hAIiIitUQFqJ67rl0QN3cNxWaHSR9vo6jEZnQkERGRGqcCJEz7cwcCvN3YnZbLq9//anQcERGRGqcCJDRq4M70IR0AeOW7/exLzzU4kYiISM1SARIAbooM5bp2TSiy2pj08TasNs2OICIidZcKkABgMpl4bmgnGri7kHQki7d+OmR0JBERkRqjAiRlQht6MvmGdgDM+GoXP/2aaXAiERGRmqECJOWMim7OzV1DKbHZefC9JA5l5hsdSUREpNqpAEk5JpOJWcO6EBnWkKyCYu59ezM5Z7VgqoiI1C0qQHIOD1cLi0d3J8TPg/0ZeUx4fwslVs0PJCIidYcKkJxXE18PFo+JwsPVzPd7T/Cvr3YbHUlERKTaqADJBXVq6sdLI7oCsGTDQZYlHDE2kIiISDVRAZKLuqFzCLF/agvAPz/9hZ8PnDQ4kYiISNWpAMklTbiuNUMiS+8MG/9uIkdOFhgdSUREpEpUgOSSTCYTLwzvQmQzP04XFHPP0k3k6s4wERFxYipAUiEerhZeGxNFsK8H+zLyePi/W7RchoiIOC0VIKmwoP93Z9jaPSeY8dUuoyOJiIhcFhUgqZTOzfx48bauALy+/iDLN+nOMBERcT4qQFJpg7uEMDGmDVB6Z9hG3RkmIiJORgVILssjA9owuEsIxVY7D+jOMBERcTIqQHJZTCYTc4ZH0kV3homIiBNSAZLL5ulm4bXRUTTxcdedYSIi4lRUgKRKgv1K7wxzdym9M2zm17ozTEREHJ8KkFRZZFhD5twWCcDiHw/yweYUgxOJiIhcnAqQVIshkaE8PKD0zrB/rNhOwsFTBicSERG5MBUgqTYTB7Thxs7BZXeGpZzSnWEiIuKYVICk2pjNJl68rSudmvpyKr+Ie5duJq+wxOhYIiIi51ABkmrl6WZh8ZjSO8P2pOfyiO4MExERB6QCJNUuxM+z7M6wuN0ZzF612+hIIiIi5agASY2IDGvIC7/dGfbqDwf4UHeGiYiIA1EBkhpzU2QoD1/XGoB/rPiFzYd0Z5iIiDgGFSCpURNj2nJDp2CKrDb++o7uDBMREcegAiQ1ymw28eKISDqG+nIyv4j73tadYSIiYjwVIKlxXm4uvD42isY+7uxOy2XismRsujNMREQMpAIktSLEz5PXRnfHzcXMt7vSmb16j9GRRESkHlMBklrTrbk/LwzvAsCi73/lo8SjBicSEZH6SgVIatXNXZvyt2tL7wx78pPtJB7WnWEiIlL7VICk1sX+qS2DOgZRZLVx/9uJHD2tO8NERKR2OUQBWrBgAeHh4Xh4eBAdHU1CQsIF9128eDH9+vXD398ff39/YmJiyu1fXFzMpEmT6Ny5M97e3oSGhjJmzBiOHTtWGx9FKsBsNvHS7V3pEFJ6Z9i9SzeTrzvDRESkFhlegJYvX05sbCzTp08nKSmJyMhIBg0aREZGxnn3X7duHSNHjmTt2rXEx8cTFhbGwIEDSU1NBaCgoICkpCSmTp1KUlISn3zyCXv27OGmm26qzY8ll/D7nWGBDX67M2y57gwTEZHaY7Lb7YZ+60RHR9OjRw/mz58PgM1mIywsjAkTJjB58uRLvt5qteLv78/8+fMZM2bMeffZtGkTPXv25PDhwzRv3vySx8zJycHPz4/s7Gx8fX0r94GkUpKOnOaO136mqMTG+GtaMen6dkZHEhERJ1WZ729DzwAVFRWRmJhITExM2Taz2UxMTAzx8fEVOkZBQQHFxcUEBARccJ/s7GxMJhMNGzY87/OFhYXk5OSUe0jtuLK5P7OHld4ZtnDdr3ySpDvDRESk5hlagDIzM7FarQQFBZXbHhQURFpaWoWOMWnSJEJDQ8uVqP/v7NmzTJo0iZEjR16wDc6YMQM/P7+yR1hYWOU+iFTJ0G5NefCaVgBM/ng7iYdPG5xIRETqOsPHAFXFzJkzWbZsGStWrMDDw+Oc54uLixkxYgR2u52FCxde8DhTpkwhOzu77JGSopXLa9vjA69gYIeg39YM20xq1hmjI4mISB1maAEKDAzEYrGQnp5ebnt6ejrBwcEXfe2cOXOYOXMm33zzDV26dDnn+d/Lz+HDh1mzZs1FrwW6u7vj6+tb7iG16/c7w9qH+JKZpzvDRESkZhlagNzc3OjevTtxcXFl22w2G3FxcfTu3fuCr5s9ezbPPvssq1atIioq6pznfy8/+/bt49tvv6VRo0Y1kl+ql7f773eGubHreA6P6s4wERGpIYZfAouNjWXx4sUsXbqUXbt2MX78ePLz8xk3bhwAY8aMYcqUKWX7z5o1i6lTp7JkyRLCw8NJS0sjLS2NvLw8oLT8DB8+nM2bN/Pee+9htVrL9ikqKjLkM0rFNW3oyaujo3CzmPlmZzovrtGaYSIiUv1cjA5w++23c+LECaZNm0ZaWhpdu3Zl1apVZQOjjxw5gtn8v562cOFCioqKGD58eLnjTJ8+naeeeorU1FRWrlwJQNeuXcvts3btWq655poa/TxSdd1b+DNzWGdiP9jKgrW/0qaJD0O7NTU6loiI1CGGzwPkiDQPkGOY+fVuFn3/K24uZpbf34tuzf2NjiQiIg7MaeYBErmYvw+6gpj2QRSV2Ljv7USO6c4wERGpJipA4rDMZhMv39GVdsE+ZOYVcu/SzRQU6c4wERGpOhUgcWgNfrszrJG3GzuP5xC7fKvuDBMRkSpTARKH18zfi1dHd8fNYmbVjjRe+nav0ZFERMTJqQCJU4gKD+Bft3YG4JXv9vNZcqrBiURExJmpAInTGN69GX+9uiUAT3y0jS1HtGaYiIhcHhUgcSp/v74dA9o1oajExt1vbWJveq7RkURExAmpAIlTsZhN/HtkNyKb+XG6oJhRr2/kUGa+0bFERMTJqACJ02ng7sLSu3vSLtiHE7mFjHp9o+YIEhGRSlEBEqfU0MuNd+6JJiLQm9SsM9z5+kZO5BYaHUtERJyECpA4rcY+7rx7bzRNG3pyIDOf0W9sJKtAC96KiMilqQCJU2va0JP37o2msY87u9NyGfvmJvIKNVu0iIhcnAqQOL3wQG/evSeahl6ubE3J4p63NnGmyGp0LBERcWAqQFInXBHsw9t396SBuwsbD55i/HuJFJXYjI4lIiIOSgVI6owuzRqy5K4eeLiaWbfnBI8s20KJVSVIRETOpQIkdUrPiABeGx2Fm8XM17+k8fePt2nxVBEROYcKkNQ5V7dtzPy/dMNiNvFJUirTV+7AblcJEhGR/1EBkjppYMdg5o6IxGSCd34+zMxVu1WCRESkjAqQ1Fk3d23K80NLV5B/9fsDzP9uv8GJRETEUagASZ32l+jm/HNwewBeXLOXN9YfNDiRiIg4AhUgqfPu7deSiTFtAHj2i50s33TE4EQiImI0FSCpFx4Z0Ib7+kUAMPmT7azceszgRCIiYiQVIKkXTCYTT97Ynr9EN8duh9jlyXy7M93oWCIiYhAVIKk3TCYTz93ciaFdQymx2Xnw/SQ27M80OpaIiBhABUjqFbPZxJzbIhnUMYiiEhv3Lt1M4uFTRscSEZFapgIk9Y6Lxcy8kd24um1jzhRbuevNTfySmm10LBERqUUqQFIvubtYePXO7vQMDyD3bAljliSwLz3X6FgiIlJLVICk3vJ0s/DGXVF0aebHqfwiRr2+kcMn842OJSIitUAFSOo1Hw9Xlo7ryRVBPmTkFvKXxRs5nn3G6FgiIlLDVICk3vP3duOde3sS3siL1KwzjHp9I5l5hUbHEhGRGlTpAnTmzBkKCgrK/nz48GFefvllvvnmm2oNJlKbmvh48O690YT6eXDgRD6j30ggu6DY6FgiIlJDKl2Abr75Zt5++20AsrKyiI6O5sUXX+Tmm29m4cKF1R5QpLY08/fivft6EdjAnV3Hcxj7ZgJ5hSVGxxIRkRpQ6QKUlJREv379APjoo48ICgri8OHDvP3228ybN6/aA4rUpohAb969tycNvVxJTsni3qWbOFtsNTqWiIhUs0oXoIKCAnx8fAD45ptvuPXWWzGbzfTq1YvDhw9Xe0CR2tYu2Je37+5JA3cXfj5wivHvJlJUYjM6loiIVKNKF6DWrVvz6aefkpKSwurVqxk4cCAAGRkZ+Pr6VntAESN0adaQJXf1wMPVzNo9J3h0eTIlVpUgEZG6otIFaNq0aTz++OOEh4cTHR1N7969gdKzQd26dav2gCJG6RkRwKujo3CzmPly+3Emf7Idm81udCwREakGJrvdXunf6GlpaRw/fpzIyEjM5tIOlZCQgK+vL+3atav2kLUtJycHPz8/srOzdVZLWPVLGg+9n4TVZmds7xY8dVNHTCaT0bFEROQPKvP9fVnzAAUHB9OtWzfMZjM5OTl8+umn+Pj41InyI/JH13cKZs5tXTCZYGn8YV5YvcfoSCIiUkWVLkAjRoxg/vz5QOmcQFFRUYwYMYIuXbrw8ccfV3tAEUdwS7dmPDe0EwD/WfcrC9buNziRiIhURaUL0A8//FB2G/yKFSuw2+1kZWUxb948nnvuuWoPKOIoRkW34MkbS89yvrB6D29uOGhwIhERuVyVLkDZ2dkEBAQAsGrVKoYNG4aXlxeDBw9m3759lxViwYIFhIeH4+HhQXR0NAkJCRfcd/HixfTr1w9/f3/8/f2JiYk5Z/9PPvmEgQMH0qhRI0wmE8nJyZeVS+SP7r+6FY8MaAPA05/v5INNKQYnEhGRy1HpAhQWFkZ8fDz5+fmsWrWq7Db406dP4+HhUekAy5cvJzY2lunTp5OUlERkZCSDBg0iIyPjvPuvW7eOkSNHsnbtWuLj4wkLC2PgwIGkpqaW7ZOfn89VV13FrFmzKp1H5FImxrTh3qsiAJj8yTa+2HbM4EQiIlJZlb4L7D//+Q+PPPIIDRo0oEWLFiQlJWE2m3nllVf45JNPWLt2baUCREdH06NHj7JxRTabjbCwMCZMmMDkyZMv+Xqr1Yq/vz/z589nzJgx5Z47dOgQERERbNmyha5du1Y4k+4Ck0ux2+08ueIX/ptwBBeziVdHd2dA+yCjY4mI1Gs1ehfYgw8+SHx8PEuWLGH9+vVlt8G3bNmy0mOAioqKSExMJCYm5n+BzGZiYmKIj4+v0DEKCgooLi4uuyx3OQoLC8nJySn3ELkYk8nEc0M7cXPXUEpsdsa/l8RP+zONjiUiIhV0WbfBR0VFccstt+Dt7c3vJ5AGDx5M3759K3WczMxMrFYrQUHl/+YcFBREWlpahY4xadIkQkNDy5WoypoxYwZ+fn5lj7CwsMs+ltQfFrOJObdF8qcOQRSV2Lj37c0kHj5tdCwREamAyypAb7/9Np07d8bT0xNPT0+6dOnCO++8U93ZLmnmzJksW7aMFStWXNb4o99NmTKF7OzsskdKiga2SsW4WszM/0s3+rUJpKDIyl1vJrDjWLbRsURE5BIqXYDmzp3L+PHjufHGG/nggw/44IMPuP7663nggQd46aWXKnWswMBALBYL6enp5banp6cTHBx80dfOmTOHmTNn8s0339ClS5fKfoxy3N3d8fX1LfcQqSh3Fwuvju5Oj3B/cs+WMOaNBPZn5BkdS0RELqLSBeiVV15h4cKFzJo1i5tuuombbrqJ2bNn85///Id58+ZV6lhubm50796duLi4sm02m424uLiyNcbOZ/bs2Tz77LOsWrWKqKioyn4EkWrn5ebCG3f1oHNTP07mF3Hn6xtJOVVgdCwREbmASheg48eP06dPn3O29+nTh+PHj1c6QGxsLIsXL2bp0qXs2rWL8ePHk5+fz7hx4wAYM2YMU6ZMKdt/1qxZTJ06lSVLlhAeHk5aWhppaWnk5f3vb9ynTp0iOTmZnTt3ArBnzx6Sk5MrPK5I5HL4eriy9O6etA1qQFrOWf7y+s+kZZ81OpaIiJxHpQtQ69at+eCDD87Zvnz5ctq0aVPpALfffjtz5sxh2rRpdO3aleTkZFatWlU2MPrIkSPlitXChQspKipi+PDhhISElD3mzJlTts/KlSvp1q0bgwcPBuCOO+6gW7duLFq0qNL5RCojwNuNd++JJryRFymnzjB2SQI5Z4uNjiUiIn9Q6XmAPv74Y26//XZiYmLK7vrasGEDcXFxfPDBB9xyyy01ErQ2aR4gqaqjpwu49T8/kZFbyFWtA3lzXA9cLZd1z4GIiFRQjc4DNGzYMDZu3EhgYCCffvopn376KYGBgSQkJNSJ8iNSHZr5e7Hkrh54uVlYvz+TJz/ZTiX/riEiIjWo0meALiQjI4PXX3+dJ598sjoOZyidAZLqsnZ3Bvcs3YTNDo/9qS0TBlT+MrGIiFRMjZ4BupDjx48zderU6jqcSJ1wbbsmPHNzJwBeXLOXFVuOGpxIRESgGguQiJzfnb1a8NerWwLw94+2Ef/rSYMTiYiICpBILZh0fTsGdw6h2Grnr+9sZn9GrtGRRETqNRUgkVpgNpt4cUQk3Vv4k3O2hLve3MSJ3EKjY4mI1FsuFd0xNjb2os+fOHGiymFE6jIPVwuLx0Rx6382cOhkAfcu3cR/7++Fl1uF/zcUEZFqUuHfvFu2bLnkPldffXWVwojUdQHebrw5rie3/mcDW49m88iyZBbd2R2L2WR0NBGReqXaboOvS3QbvNS0zYdO8ZfXN1JUYmNc33CmD+lodCQREadnyG3wIlJxUeEBvDSiKwBvbjjEkvUHjQ0kIlLPqACJGGRwlxCm3NAOgGe/3MnqHVqsV0SktqgAiRjo/qtbMiq6OXY7PLJsC8kpWUZHEhGpF1SARAxkMpl4+qaOXHtFY84W27jnrU0cOVlgdCwRkTpPBUjEYC4WM/P/ciUdQ305mV/EXW8lkFVQZHQsEZE67bLuAsvKyiIhIYGMjAxsNlu558aMGVNt4Yyiu8DECOk5Zxm6YAPHs88SHRHA2/f0xN3FYnQsERGnUZnv70oXoM8//5xRo0aRl5eHr68vJtP/5i8xmUycOnXq8lI7EBUgMcrutBxuWxhPbmEJN3cN5eXbu5b7f0xERC6sRm+Df+yxx7j77rvJy8sjKyuL06dPlz3qQvkRMVK7YF8W3tkdF7OJz5KPMXfNXqMjiYjUSZUuQKmpqTz88MN4eXnVRB6Reu+qNoH865bOALzy3X4+2JRicCIRkbqn0gVo0KBBbN68uSayiMhvRvQIY8J1rQGYsmI7P+zVWnsiItWp0qswDh48mCeeeIKdO3fSuXNnXF1dyz1/0003VVs4kfos9k9tOXr6DCu2pPLge0l8NL437YI1Jk1EpDpUehC02Xzhk0Ymkwmr1VrlUEbTIGhxFIUlVsa8kcDGg6cI8fNgxYN9CfbzMDqWiIhDqtFB0Dab7YKPulB+RByJu4uF10ZH0aqxN8ezz3L3W5vIKywxOpaIiNPTRIgiDs7Py5W3xvUksIEbO4/n8NB7SZRYbZd+oYiIXFCFLoHNmzeP+++/Hw8PD+bNm3fRfR9++OFqC2cUXQITR5ScksUdr8VzttjGyJ7N+dctnTRHkIjI/1PtEyFGRESwefNmGjVqRERExIUPZjJx4MCByid2MCpA4qi+2ZHGX99NxG6HyTe044H+rYyOJCLiMGp0Juj6QAVIHNmbGw7y9Oc7AXhlZDeGRIYanEhExDHU6CBoETHWuL4RjOsbDsBjH25l0yHNwC4iUlmVngcI4OjRo6xcuZIjR45QVFR+1eq5c+dWSzARubB/Du5A6ukzfLMznfve3syKB/sSEehtdCwREadR6QIUFxfHTTfdRMuWLdm9ezedOnXi0KFD2O12rrzyyprIKCJ/YDGb+Pcd3bjjtXi2Hs3mrjcT+GR8Hxo1cDc6moiIU6j0JbApU6bw+OOPs337djw8PPj4449JSUmhf//+3HbbbTWRUUTOw9PNwutje9DM35PDJwu47+3NnC3WXFwiIhVR6QK0a9cuxowZA4CLiwtnzpyhQYMGPPPMM8yaNavaA4rIhTX2ceetcT3w9XAh6UgWsR8kY7PpvgYRkUupdAHy9vYuG/cTEhLCr7/+WvZcZmZm9SUTkQpp3cSH18ZE4Wox8dX2NGau2m10JBERh1fpAtSrVy/Wr18PwI033shjjz3G888/z913302vXr2qPaCIXFqvlo14YXgkAK/9cIB34g8ZG0hExMFVehD03LlzycvLA+Dpp58mLy+P5cuX06ZNG90BJmKgod2acvR0AXO+2cv0lTto6u/Jde2CjI4lIuKQKjURotVqZcOGDXTp0oWGDRvWYCxjaSJEcVZ2u51JH2/jg81H8XKzsPz+3nRu5md0LBGRWlFjEyFaLBYGDhzI6dOnqxRQRGqGyWTi+Vs6069NIAVFVu5euonUrDNGxxIRcTiVHgPUqVOnOrHel0hd5Woxs2DUlbQL9uFEbiHj3kwg+0yx0bFERBxKpQvQc889x+OPP84XX3zB8ePHycnJKfcQEeP5eriy5K4eNPFxZ296HuPfTaSoxGZ0LBERh1HhMUDPPPMMjz32GD4+Pv97sclU9s92ux2TyYTV6vwTsWkMkNQVv6Rmc/ur8eQXWRnevRkvDO9S7v9bEZG6pEZWg7dYLBw/fpxdu3ZddL/+/ftXPKmDUgGSumTtngzuXboZq83OozFteSSmjdGRRERqRI0Mgv69J/Xv3/+ij8uxYMECwsPD8fDwIDo6moSEhAvuu3jxYvr164e/vz/+/v7ExMScs7/dbmfatGmEhITg6elJTEwM+/btu6xsIs7u2iua8MzNHQF46du9fJx41OBEIiLGq9QYoJo4db58+XJiY2OZPn06SUlJREZGMmjQIDIyMs67/7p16xg5ciRr164lPj6esLAwBg4cSGpqatk+s2fPZt68eSxatIiNGzfi7e3NoEGDOHv2bLXnF3EGo6Jb8Nf+LQGY/Mk2fvpVs7aLSP1W4UtgZrMZPz+/S5agU6dOVSpAdHQ0PXr0YP78+QDYbDbCwsKYMGECkydPvuTrrVYr/v7+zJ8/nzFjxmC32wkNDeWxxx7j8ccfByA7O5ugoCDeeust7rjjjkseU5fApC6y2exMWLaFL7cdx8fDhU/G96FNkM+lXygi4iQq8/1dqZmgn376afz8qm9StaKiIhITE5kyZUrZNrPZTExMDPHx8RU6RkFBAcXFxQQEBABw8OBB0tLSiImJKdvHz8+P6Oho4uPjz1uACgsLKSwsLPuz7maTushsNvHibZGkZZ8l8fBp7npzEyse6kMTHw+jo4mI1LpKFaA77riDJk2aVNubZ2ZmYrVaCQoqP11/UFAQu3dXbEHHSZMmERoaWlZ40tLSyo7xx2P+/twfzZgxg6effrqy8UWcjoerhcVjohi28CcOZuZzz1ubWXZ/L7zdK70qjoiIU6vwGCBHvHV25syZLFu2jBUrVuDhcfl/i50yZQrZ2dllj5SUlGpMKeJYArzdePOuHvh7ubI9NZuez3/LPW9tYsn6g+xNz6USq+OIiDitCv+1ryZ+KQYGBmKxWEhPTy+3PT09neDg4Iu+ds6cOcycOZNvv/2WLl26lG3//XXp6emEhISUO2bXrl3Peyx3d3fc3d0v81OIOJ/wQG9eH9uDB99LJD2nkLjdGcTtLr3xoLGPO1e1DqRv60D6tm5EiJ+nwWlFRKpfhQuQzVb9s8i6ubnRvXt34uLiGDp0aNn7xMXF8be//e2Cr5s9ezbPP/88q1evJioqqtxzERERBAcHExcXV1Z4cnJy2LhxI+PHj6/2zyDirLq38Cd+8gB2Hs9hw/5M1u/PJOHgKU7kFrJiSyortpTeWdmqsfdvZSiQXi0b4efpanByqaz8whLW7TlB39aNaOjlZnQcEYdg+IX/2NhYxo4dS1RUFD179uTll18mPz+fcePGATBmzBiaNm3KjBkzAJg1axbTpk3j/fffJzw8vGxcT4MGDWjQoAEmk4mJEyfy3HPP0aZNGyIiIpg6dSqhoaFlJUtESpnNJjo19aNTUz/+2r8VZ4utJB05/VshOsn2o1n8eiKfX0/k83b8Ycwm6NKsYdkZoitbNMTdxWL0x5ALsNnsfLY1lZlf7yY9p5CrWgfy7r3RRscScQiGF6Dbb7+dEydOMG3aNNLS0ujatSurVq0qG8R85MgRzOb/DVVauHAhRUVFDB8+vNxxpk+fzlNPPQXA3//+d/Lz87n//vvJysriqquuYtWqVVUaJyRSH3i4WujTKpA+rQJ5YhBkFxQTf+AkG/ZnsmF/Jgcy80lOySI5JYv5a/fj4WqmZ0QjrmrdiL6tA2kf7IvZ7HjjBeuj5JQsnv58B1uOZJVtW78/k+SULLqGNTQsl4ijqPA8QPWJ5gESOb/UrDNlZWjD/pNk5hWWez7A240+rRqVnSEKC/AyKGn9lZ5zllmrdvNJUuklTC83C3+7rjW7jufy+dZjXN8xmEWjuxucUqRm1MhaYPWJCpDIpdntdvam57H+t0L084GTFBSVXwy5RSMv+rYO5KrWgfRu2Qh/b40/qSlni628sf4gC9buL/v3MOzKZvz9+isI8vVgT1oug17+AZMJvo3tT6vGDQxOLFL9VICqSAVIpPKKSmxsPZrF+n2lhWhLShZW2/9+vZhM0DHUt6wQ9QgPwMNV44eqym63s3pHOs9/tZOUU2cA6Na8IdOHdDznUtc9b20ibncGt0eFMWt4l/McTcS5qQBVkQqQSNXlni0m4eCpsjNEe9Pzyj3v5mImqoV/WSHq1NQPi8YPVcqu4zk88/lO4g+cBCDI150pN7TnpsjQ847F2nzoFMMXxeNqMfHj368j2E/jIqVuUQGqIhUgkeqXkXOWDb9msn5f6aDqtJzyixP7erjQp1UgfduUFqLwRl4OOQGrIziVX8TcNXt4f+MRbPbSMvnXq1vyQP9Wl5zV+7ZFP7Hp0Gnuv7olT97YvpYSi9QOFaAqUgESqVl2u50Dmfmlt9vvyyT+wElyz5aU26dpQ0/6/nZ32TVXNNH8Q0Cx1cY78Yd5+du95Pz28xrcOYTJN7Sr8IDzuF3p3LN0M95uFn6aPAA/L/1cpe5QAaoiFSCR2lVitbE9NbtsQsakw1kUWf83+aq7i5lBHYO5LaoZfVoF1stLZd/vPcEzn+/g1xP5ALQP8WX6kA70atmoUsex2ezc8O8f2ZOeyxODruCha1vXRFwRQ6gAVZEKkIixCopK2HSodELG73ZnsD/jf+OHQv08uPXKZgzv3ozwQG8DU9aOAyfyeP7LXWVLlQR4u/H4wCu4vUfYZRfBT5KOEvvBVhp5u7Fh8nUajC51hgpQFakAiTgOu93O9tRsPtx8lM+SU8su/QD0DA9geFQzBncOqXMr2uecLeaVuH289dMhiq12XMwmxvYJ5+EBbap8ObDYauOaF9aRmnWGZ4d2YnSvFtWUWsRYKkBVpAIk4pjOFltZszOdDxOP8uO+E/z+28vLzcKNnUMY3r0Z0REBTj142mqz8+HmFF5YvYeT+UUAXHtFY/755w7VOnfPWxsO8tTnOwkL8GTtY9fgYjFf+kUiDk4FqIpUgEQc3/HsM3ySlMpHiUc5mJlftr15gBfDuzdjWPdmNG3oXCvZJxw8xdOf72DHsRwAWjb2ZurgDlzbrkm1v1dBUQl9Z37H6YJi5o3sxk2RodX+HiK1TQWoilSARJyH3W4n8fBpPtx8lC+2HSP/t1mQTSbo2yqQ26KaMahjsEOPc0nNOsOMr3bxxbbjAPh4uDAxpi1jerfAtQbPzPz723289O1e2of48tXDVzn1mTMRUAGqMhUgEedUUFTC19vT+CjxaNnkgAA+7i78OTKU26Ka0S2socN80RcUlbDo+wO8+v2vFJbYMJvgjp7NeexPbWnUwL3G3/90fhF9Zn7HmWIrb43rwTVXVP+ZJpHapAJURSpAIs4v5VQBHyUe5aPEo6RmnSnb3rpJA4Z3b8at3ZrSxNeYmZDtdjsrtx5j5te7OZ5dOiFkdEQA04d0pENo7f7OeebznSzZcJBeLQNYdn/vWn1vkeqmAlRFKkAidYfNZufnAyf5MPEoX/9ynLPFpfMLWcwm+rdtzG3dm3Fd+ya4u9TOJbJtR7N4+vOdJB4+DZRO+PjPwe25vlOwIWemjmWd4erZaymx2VnxYB+6Nfev9Qwi1UUFqIpUgETqppyzxXy17TgfJh4tKyAA/l6u3Ny1KcO7N6NTU78aee+M3LO8sGoPHyUdxW4HT1cLD13binv7tTR8fNJjH2zl46SjDOoYxKujowzNIlIVKkBVpAIkUvf9eiKPjxKP8knSUdJzCsu2tw/x5bbuzbi5a2i1jMMpLLHy5oZDzP9uP3mFpXMY3dqtKX+/vp3DLEa6Lz2XP730AwDfxl5N6yY+BicSuTwqQFWkAiRSf5RYbfy4P5OPNh9lzc70siU4XC0mrmvXhNu6h3HNFY0rPU+O3W5nzc50nv9qF4dPFgAQGdaQ6UM6cKUDXma6d+lmvt2Vzm3dm/HCbZFGxxG5LCpAVaQCJFI/ZRUUsXLrMT7cfJTtqdll2wMbuHPrlU25rXsz2gRd+uzInrRcnv1iJ+v3ZwLQxMedSde345ZuTTE76DpmiYdPM2zhT7haTPzw92sJ8XOuOZREQAWoylSARGR3Wg4fbj7Kp1tSy2ZkhtKzOMO7N+OmyNBzlqTIKijipTV7eXfjEaw2O24uZu7rF8GD17R2iqU6RrwaT8LBU9x7VQT//HMHo+OIVJoKUBWpAInI74qtNr7bncFHiUdZuzuDElvpr0y331eo796MXi0bsWzTEeau2UtWQTEA13cM5skb29O8kZeR8Stl7e4Mxr21CW83CxsmX0dDLzejI4lUigpQFakAicj5nMgt5LPkVD7cfJQ96bll291dzBSWlI4dahfsw7Q/d6BP60CjYl42u93ODf/+kd1puTz2p7ZMGNDG6EgilaICVEUqQCJyMedbod7fy5XYgVcwskeYUy8s+umWVCYuT6aRtxvrJ12Hp5vjLiEi8kcqQFWkAiQiFXW22MrO4zm0atzgnDFBzqjEauOaOes4evoMz9zckTG9w42OJFJhlfn+dt6/poiIOAAPVwtXNvevE+UHwMVi5v6rWwLw2g8HKPltWgCRukYFSEREyrmtexgB3m4cPX2GL7cfNzqOSI1QARIRkXI83SyM6xMOwMJ1v6KRElIXqQCJiMg5RvdugZebhd1puazbe8LoOCLVTgVIRETO0dDLjb/0bA6UngUSqWtUgERE5Lzu6ReBq8VEwsFTJB4+bXQckWqlAiQiIucV4ufJ0K5NAVj0vc4CSd2iAiQiIhf01/4tMZlgzc509v2/2a9FnJ0KkIiIXFDrJj78qX0QAK/+cMDgNCLVRwVIREQu6oFrWgGly2QcyzpjcBqR6qECJCIiF3Vlc3+iIwIosdl5/ceDRscRqRYqQCIicknjfzsLtGzTEU7nFxmcRqTqVIBEROSS+rdtTPsQXwqKrLwdf9joOCJVpgIkIiKXZDKZeKB/6SKpb/10kIKiEoMTiVSNCpCIiFTI4M4hhAV4crqgmA82pRgdR6RKVIBERKRCXCxm7u9XehZo8Y8HKbbaDE4kcvlUgEREpMJuiwqjkbcbqVln+GLbMaPjiFw2wwvQggULCA8Px8PDg+joaBISEi64744dOxg2bBjh4eGYTCZefvnlc/bJzc1l4sSJtGjRAk9PT/r06cOmTZtq8BOIiNQfHq4W7r4qAoBF6w5gt9sNTiRyeQwtQMuXLyc2Npbp06eTlJREZGQkgwYNIiMj47z7FxQU0LJlS2bOnElwcPB597n33ntZs2YN77zzDtu3b2fgwIHExMSQmppakx9FRKTeuDO6Bd5uFvak57J2z/l/X4s4OpPdwPoeHR1Njx49mD9/PgA2m42wsDAmTJjA5MmTL/ra8PBwJk6cyMSJE8u2nTlzBh8fHz777DMGDx5ctr179+7ccMMNPPfccxXKlZOTg5+fH9nZ2fj6+lb+g4mI1HH/+moXr/1wgB7h/nz4QB+j44gAlfv+NuwMUFFREYmJicTExPwvjNlMTEwM8fHxl3XMkpISrFYrHh4e5bZ7enqyfv36C76usLCQnJyccg8REbmwu/tG4GoxsenQaTYfOmV0HJFKM6wAZWZmYrVaCQoKKrc9KCiItLS0yzqmj48PvXv35tlnn+XYsWNYrVbeffdd4uPjOX78+AVfN2PGDPz8/MoeYWFhl/X+IiL1RbCfB7d2awbAou9/NTiNSOUZPgi6ur3zzjvY7XaaNm2Ku7s78+bNY+TIkZjNF/6oU6ZMITs7u+yRkqL5LURELuX+/i0xmeDbXRnsScs1Oo5IpRhWgAIDA7FYLKSnp5fbnp6efsEBzhXRqlUrvv/+e/Ly8khJSSEhIYHi4mJatmx5wde4u7vj6+tb7iEiIhfXqnEDBnUo/X396g86CyTOxbAC5ObmRvfu3YmLiyvbZrPZiIuLo3fv3lU+vre3NyEhIZw+fZrVq1dz8803V/mYIiJS3gO/LZK6MvkYqVlnDE4jUnGGXgKLjY1l8eLFLF26lF27djF+/Hjy8/MZN24cAGPGjGHKlCll+xcVFZGcnExycjJFRUWkpqaSnJzM/v37y/ZZvXo1q1at4uDBg6xZs4Zrr72Wdu3alR1TRESqT9ewhvRu2YgSm53XfzxgdByRCnMx8s1vv/12Tpw4wbRp00hLS6Nr166sWrWqbGD0kSNHyo3dOXbsGN26dSv785w5c5gzZw79+/dn3bp1AGRnZzNlyhSOHj1KQEAAw4YN4/nnn8fV1bVWP5uISH0x/ppWxB84ybKEFCZc14YAbzejI4lckqHzADkqzQMkIlJxdrudP7+ynh3HcnhkQBse/VNboyNJPeUU8wCJiEjdYDKZeKB/6VigpfGHKCgqMTiRyKWpAImISJXd0CmYFo28yCooZlmCphIRx6cCJCIiVeZiMXNfv9LpRl7/8QDFVpvBiUQuTgVIRESqxfDuzQhs4M6x7LOsTD5mdByRi1IBEhGRauHhauHuq8KB0okRbTbdYyOOSwVIRESqzajoFjRwd2Fveh7f7c4wOk61y8wrRDdP1w0qQCIiUm38PF0Z1as5AAvr0CKpp/OLePi/W4h67lumfbbD6DhSDVSARESkWt3TNwI3i5nEw6fZdOiU0XGq7JsdafzppR9YubV0XNM7Px9my5HTBqeSqlIBEhGRatXE14Nh3ZsCsHCd854Fyi4oJnZ5Mve/k0hmXiGtmzSgf9vGADy1cofGODk5FSAREal29/VrickE3+3OYHdajtFxKi1uVzp/eul7PtmSitkEf+3fki8mXMULw7vQwN2FrUez+SjpqNExpQpUgEREpNq1bNyAGzoFA/Dq986zSGr2mWIe/3Ar9yzdTEZuIS0be/PR+D5MuaE9Hq4Wmvh68PCA1gDMXrWHnLPFBieWy6UCJCIiNeL35TFWbj1GyqkCg9Nc2ro9GQx66Qc+SjyKyQT39Yvgq4f7cWVz/3L73dUngpaB3mTmFfJK3D6D0kpVqQCJiEiN6NKsIX1bN8Jqs/PG+oNGx7mgnLPFTPpoG3e9uYm0nLOEN/Liw7/25h+DO+DhajlnfzcXM1OHdADgzQ2H2J+RV9uRpRqoAImISI0Z37/0ctGyTUc4mVdocJpz/bjvBNe/9APLN6dgMsHdfSP4+pGriQoPuOjrrr2iCQPaNaHEZueZL3ZqbiAnpAIkIiI1pm/rRnRq6svZYhtL4w8bHadMXmEJUz7Zzug3EjiWfZbmAV4su68X04Z0wNPt3LM+5zP1zx1ws5j5Ye8J4nbVvUkf6zoVIBERqTEmk6nsLNDSnw6RX1hicCLYsD+TQS/9wH8TjgAwtncLVk3sR3TLRpU6TnigN/f0iwDgmS92crbYWu1ZpeaoAImISI26vlMw4Y28yD5TzLJNKYblyC8sYeqnvzDq9Y2kZp2hmb8n798XzdM3d8LLzeWyjvnQta1p4uPOkVMFDj3OSc6lAiQiIjXKYjZx/9Wld4S9/uMBikpstZ4h/teTXP/vH3jn59LLcHf2as7qiVfTp1VglY7bwN2FKTe2A2DB2v2kZZ+tclapHSpAIiJS4269simNfdw5nn2Wz5JTa+19C4pKeGrlDkYu/pmUU2do2tCTd++J5rmhnfF2v7yzPn80tGtTrmzekIIiKzO/3lUtx5SapwIkIiI1zsPVwt19S8fLvPrDgVpZRiLh4Clu+PePvPXTIQBG9mzOqon9uKpN1c76/JHJZOLpmzphMsGnycfYXAfWP6sPVIBERKRWjOrVHB93F/Zn5PHtrvQae58zRVae+Xwnt78Wz+GTBYT4efD23T2ZcWtnfDxca+Q9Ozfz444eYQBMX7kDq9YJc3gqQCIiUit8PVy5s3cLABZ+/2uNzJ2TePgUN877kSUbDmK3w4ioZqx+9Gqu/m0R05r0+MAr8PFwYcexHJYbONhbKkYFSEREas24vuG4uZjZciSLhIPVd6nobLGVf321i+GL4jmYmU+QrztvjuvB7OGR+NbQWZ8/atTAnUdj2gIw55s9ZBdonTBHpgIkIiK1pomPB8O7NwNKzwJVh6Qjp7lx3o+89sMB7HYYdmUzvpnYn2uvaFItx6+M0b1b0KZJA07lF/HSt3tr/f2l4lSARESkVt3fryVmE6zbc4Jdx3Mu+zhni63M/Ho3wxf+xIET+TT2ceeNsVG8OCISP6/aOevzR64WM9OHdATgnZ8Psyct15AccmkqQCIiUqvCA725oXMIAIsu8yzQ1pQshryynkXf/4rNDrd0a8qaR69mQPug6ox6Wa5qE8igjkFYbXae/nyH1glzUCpAIiJS68b3L50Y8Yttx0k5VVDh1xWWWHlh9W5uXfgT+zLyCGzgzmuju/PS7V1p6OVWU3Er7Z+DO+DmYuanX0+y6pc0o+PIeagAiYhIrevU1I9+bQKx2uws/vFAhV7zS2o2N72ygQVrf8Vqs3NTZChrHr2agR2Dazht5YUFePHA1S0BeO7LXVonzAGpAImIiCF+Pwu0fFMKmXmFF9yvqMTG3G/2cPOCDexJz6WRtxsLR13JvJHd8Pd2nLM+fzT+mtaE+nmQmnWGV7+vWMmT2qMCJCIihujdqhFdmvlRWGJj6W+zNf/RjmPZ3LxgA/O+24/VZmdw5xC+efTqsjFEjszTzcKUG9sD8J91+zl6uuKX+qTmqQCJiIghTCZT2VmgpT8dIq+wpOy5YquNf3+7j5vnb2DX8Rz8vVyZ/5duLBh1JY0auBsVudL+3CWEnhEBFJbYmPHVbqPjyP+jAiQiIoYZ2DGYloHe5JwtYVnCEQB2p+UwdMEGXvp2LyU2O4M6BvHNo/35c5dQg9NWnslk4qkhHTGb4Mvtx4n/9aTRkeQ3KkAiImIYi9nE/b8NFl784wHmxe1jyCvr2XEsh4Zervz7jq4surM7jX2c56zPH3UI9WVUdOkSIE9/voMSq83gRMY6U2TliQ+3kpp1xtAcKkAiImKoW65sShMfd9JzCpm7Zi/FVjsx7YP45tGrublrU0wmk9ERqyz2T23x83Rld1ou7/92pqs+Kiyx8sC7iXyYeJR73tqEzcBFY1WARETEUO4ulrKzQL4eLrx0eySLx3SniY+Hwcmqj7+3G48PLF0n7MVv9nI6v8jgRLWvxGpj4rJkvt97Ak9XC88N7YTZbFy5dTHsnUVERH5zz1URtAnyoUOIr1Nf7rqYkT2b897GI+xOy+XFNXt4bmhnoyPVGpvNzqSPt/P1L2m4Wcy8NqY7UeEBhmbSGSARETGcyWSif9vGdbb8ALhYzDx1U+k6Ye9vPMKOY9kGJ6oddnvpkiAfJx3FYjbxyl+60a9NY6NjqQCJiIjUll4tGzG4Swg2Ozy9cme9WCdszjd7WBp/GJMJ5tzWhUEOMnO3CpCIiEgtevLG9ni4mkk4dIrPtx03Ok6N+s+6/SxYW7rg7XNDO3FLt2YGJ/ofFSAREZFa1LShJw9e0xqAGV/toqCo5BKvcE5vxx9i9qo9ADx5Y7uyqQAcheEFaMGCBYSHh+Ph4UF0dDQJCQkX3HfHjh0MGzaM8PBwTCYTL7/88jn7WK1Wpk6dSkREBJ6enrRq1Ypnn322XpxmFBER53D/1S1p5u/J8eyzLFz3q9Fxqt1HiUeZ9tkOAB6+rjX3X93K4ETnMrQALV++nNjYWKZPn05SUhKRkZEMGjSIjIyM8+5fUFBAy5YtmTlzJsHB57+GOGvWLBYuXMj8+fPZtWsXs2bNYvbs2bzyyis1+VFEREQqzMPVwj8HdwDg1R8OcORk3Vkn7Ovtx/n7R1sBGNc3nEf/1NbgROdnaAGaO3cu9913H+PGjaNDhw4sWrQILy8vlixZct79e/TowQsvvMAdd9yBu/v57xT46aefuPnmmxk8eDDh4eEMHz6cgQMHXvTMkoiISG0b1DGIvq0bUVRi47kvdxodp1qs25PBw8u2YLPDiKhmTB3cwWEnsjSsABUVFZGYmEhMTMz/wpjNxMTEEB8ff9nH7dOnD3FxcezduxeArVu3sn79em644YYLvqawsJCcnJxyDxERkZpkMpmYPqQjFrOJb3am8+O+E0ZHqpKNB07ywLuJFFvtDO4Swoxbuxg60eGlGFaAMjMzsVqtBAUFldseFBREWlraZR938uTJ3HHHHbRr1w5XV1e6devGxIkTGTVq1AVfM2PGDPz8/MoeYWFhl/3+IiIiFdU2yIcxvX9fJ2wnxU66Tti2o1ncs3QzZ4ttXNeuCS+N6IrFgcsPOMAg6Or2wQcf8N577/H++++TlJTE0qVLmTNnDkuXLr3ga6ZMmUJ2dnbZIyUlpRYTi4hIfTYxpi0B3m7sz8jj7fjDRseptD1puYxZkkBeYQm9Wgbwn1FX4ubi+PXCsISBgYFYLBbS09PLbU9PT7/gAOeKeOKJJ8rOAnXu3JnRo0fz6KOPMmPGjAu+xt3dHV9f33IPERGR2uDn6coTg64A4OU1e8nMKzQ4UcUdysznzjc2klVQTGRYQ14f2wMPV4vRsSrEsALk5uZG9+7diYuLK9tms9mIi4ujd+/el33cgoICzObyH8tisWCzOedpRRERqftGRIXRqakvuYUlzFm9x+g4FXIs6wyjXt/IidxC2gX7sHRcDxq4O88So4aeo4qNjWXx4sUsXbqUXbt2MX78ePLz8xk3bhwAY8aMYcqUKWX7FxUVkZycTHJyMkVFRaSmppKcnMz+/fvL9hkyZAjPP/88X375JYcOHWLFihXMnTuXW265pdY/n4iISEVYzCaeGlK6TtjyzSlsO5plbKBLyMwr5M7XN5KadYaIQG/euSeahl5uRseqFJPd4BkC58+fzwsvvEBaWhpdu3Zl3rx5REdHA3DNNdcQHh7OW2+9BcChQ4eIiIg45xj9+/dn3bp1AOTm5jJ16lRWrFhBRkYGoaGhjBw5kmnTpuHmVrF/OTk5Ofj5+ZGdna3LYSIiUmseXZ7Mii2pXNm8IR890Mch76LKLijmjsU/s+t4Dk0bevLBA71p2tDT6FhA5b6/DS9AjkgFSEREjJCec5Zr56yjoMjK3BGR3Hql46ydBZBfWMKdb2xky5EsAhu48+EDvYkI9DY6VpnKfH87/jBtERGReiLI14O/XVe6TtjMr3eTV+g464SdLbZy39ub2XIki4Zerrx7b0+HKj+VpQIkIiLiQO65KoIWjbzIyC1k/nf7L/2CWlBstfG395P46deTeLtZWDquJ+2CnfsKiQqQiIiIA3F3sTDtz6XrhL2x/gAHM/MNzWO12Yn9YCvf7srA3cXMG3f1IDKsoaGZqoMKkIiIiIO5rl0T+rdtTLHVzrNfGLdOmN1u5x8rtvP51mO4WkwsGt2dXi0bGZanOqkAiYiIOBiTycS0IR1wMZv4bncGa3dn1HoGu93Oc1/uYtmmFMwmePn2blx7RZNaz1FTVIBEREQcUKvGDbj7qtKpX575YidFJbU7oe+/4/bxxvqDAMwa1oXBXUJq9f1rmgqQiIiIg5pwXWsCG7hzMDOfNzccrLX3ff3HA7z87T4AnhrSgdui6t4i4SpAIiIiDsrHw5VJ15euEzYvbh8ZOWdr/D3f33iE577cBcATg67grr7nTkBcF6gAiYiIOLBhVzYjMqwh+UVWZq7aXaPv9VlyKv/4dDsAD/RvxYPXtKrR9zOSCpCIiIgDM5tNPH1T6TphnySlknTkdI28z5qd6cR+sBW7HUb3asGk66/AZHK8pTiqiwqQiIiIg+sa1pDbupcui/HUyh3YbNW7itWG/Zk89H4SVpudW7s15embOtbp8gMqQCIiIk7h79e3w8fdhW1Hs/ko8Wi1HTfx8CnuXbqZohIbgzoGMXt4F4dchLW6qQCJiIg4gcY+7jw8oA0As1fvJudscZWPueNYNne9uYkzxVb6tQlk3shuuFjqRzWoH59SRESkDhjbJ5yWjb3JzCti3m+3qV+u/Rl5jHkjgdyzJfQI9+e10VG4u1iqKanjUwESERFxEm4u5rJ1wt766RD7M3Iv6zgppwq48/WNnMwvonNTP964qweebvWn/IAKkIiIiFO55oomxLQPosRm5+nPd2K3V25AdHrOWUa9vpG0nLO0adKApXf3xNfDtYbSOi4VIBEREScz9c/tcbOY+XFfJmt2plf4dafyi7jz9Y0cOVVA8wAv3r03mgBvtxpM6rhUgERERJxMi0be3NuvdIbm577cxdli6yVfk3O2mLFLEtiXkUewrwfv3RtNkK9HTUd1WCpAIiIiTuiha1sT5OvOkVMFZYuWXsiZIiv3vLWJ7anZNPJ24917owkL8KqlpI5JBUhERMQJebu78OSN7QGY/91+jmefOe9+hSVW7n9nM5sOncbHw4W37+lJ6yYNajOqQ1IBEhERcVI3RYYS1cKfM8VWZnx17jphJVYbD/93Cz/uy8TLzcJb43rSMdTPgKSORwVIRETESZlMJp66qSMmE6zceoxNh06VPWez2fn7R9tYvSMdN4uZxWOi6N7C38C0jkUFSERExIl1aurHHT2aAzD9sx1YbXbsdjvTV+7gky2pWMwmFoy6kr6tAw1O6lhcjA4gIiIiVfP4wLZ8ue0YO4/nsGzTEY6ePsM7Px/GZIK5IyL5U4cgoyM6HJ0BEhERcXKNGrjz6J/aAvD0yp0sXPcrAP+6pTM3d21qZDSHpQIkIiJSB9zZqwVtgxpQZLUB8M/B7RnZs7nBqRyXCpCIiEgd4GoxM2tYF9o0acCTN7bj3n4tjY7k0DQGSEREpI7o1tyfNbH9jY7hFHQGSEREROodFSARERGpd1SAREREpN5RARIREZF6RwVIRERE6h0VIBEREal3VIBERESk3lEBEhERkXpHBUhERETqHRUgERERqXdUgERERKTeUQESERGRekcFSEREROodFSARERGpd1yMDuCI7HY7ADk5OQYnERERkYr6/Xv79+/xi1EBOo/c3FwAwsLCDE4iIiIilZWbm4ufn99F9zHZK1KT6hmbzcaxY8fw8fHBZDJV67FzcnIICwsjJSUFX1/faj12faKfY/XQz7F66OdYPfRzrB71+edot9vJzc0lNDQUs/nio3x0Bug8zGYzzZo1q9H38PX1rXf/YdYE/Ryrh36O1UM/x+qhn2P1qK8/x0ud+fmdBkGLiIhIvaMCJCIiIvWOClAtc3d3Z/r06bi7uxsdxanp51g99HOsHvo5Vg/9HKuHfo4Vo0HQIiIiUu/oDJCIiIjUOypAIiIiUu+oAImIiEi9owIkIiIi9Y4KUC1asGAB4eHheHh4EB0dTUJCgtGRnMqMGTPo0aMHPj4+NGnShKFDh7Jnzx6jYzm9mTNnYjKZmDhxotFRnE5qaip33nknjRo1wtPTk86dO7N582ajYzkVq9XK1KlTiYiIwNPTk1atWvHss89WaC2n+uyHH35gyJAhhIaGYjKZ+PTTT8s9b7fbmTZtGiEhIXh6ehITE8O+ffuMCeugVIBqyfLly4mNjWX69OkkJSURGRnJoEGDyMjIMDqa0/j+++956KGH+Pnnn1mzZg3FxcUMHDiQ/Px8o6M5rU2bNvHqq6/SpUsXo6M4ndOnT9O3b19cXV35+uuv2blzJy+++CL+/v5GR3Mqs2bNYuHChcyfP59du3Yxa9YsZs+ezSuvvGJ0NIeWn59PZGQkCxYsOO/zs2fPZt68eSxatIiNGzfi7e3NoEGDOHv2bC0ndWB2qRU9e/a0P/TQQ2V/tlqt9tDQUPuMGTMMTOXcMjIy7ID9+++/NzqKU8rNzbW3adPGvmbNGnv//v3tjzzyiNGRnMqkSZPsV111ldExnN7gwYPtd999d7ltt956q33UqFEGJXI+gH3FihVlf7bZbPbg4GD7Cy+8ULYtKyvL7u7ubv/vf/9rQELHpDNAtaCoqIjExERiYmLKtpnNZmJiYoiPjzcwmXPLzs4GICAgwOAkzumhhx5i8ODB5f67lIpbuXIlUVFR3HbbbTRp0oRu3bqxePFio2M5nT59+hAXF8fevXsB2Lp1K+vXr+eGG24wOJnzOnjwIGlpaeX+3/bz8yM6OlrfOf+PFkOtBZmZmVitVoKCgsptDwoKYvfu3Qalcm42m42JEyfSt29fOnXqZHQcp7Ns2TKSkpLYtGmT0VGc1oEDB1i4cCGxsbE8+eSTbNq0iYcffhg3NzfGjh1rdDynMXnyZHJycmjXrh0WiwWr1crzzz/PqFGjjI7mtNLS0gDO+53z+3OiAiRO6qGHHuKXX35h/fr1RkdxOikpKTzyyCOsWbMGDw8Po+M4LZvNRlRUFP/6178A6NatG7/88guLFi1SAaqEDz74gPfee4/333+fjh07kpyczMSJEwkNDdXPUWqULoHVgsDAQCwWC+np6eW2p6enExwcbFAq5/W3v/2NL774grVr19KsWTOj4zidxMREMjIyuPLKK3FxccHFxYXvv/+eefPm4eLigtVqNTqiUwgJCaFDhw7ltrVv354jR44YlMg5PfHEE0yePJk77riDzp07M3r0aB599FFmzJhhdDSn9fv3ir5zLk4FqBa4ubnRvXt34uLiyrbZbDbi4uLo3bu3gcmci91u529/+xsrVqzgu+++IyIiwuhITmnAgAFs376d5OTkskdUVBSjRo0iOTkZi8VidESn0Ldv33OmYdi7dy8tWrQwKJFzKigowGwu/1VksViw2WwGJXJ+ERERBAcHl/vOycnJYePGjfrO+X90CayWxMbGMnbsWKKioujZsycvv/wy+fn5jBs3zuhoTuOhhx7i/fff57PPPsPHx6fsWrafnx+enp4Gp3MePj4+54yb8vb2plGjRhpPVQmPPvooffr04V//+hcjRowgISGB1157jddee83oaE5lyJAhPP/88zRv3pyOHTuyZcsW5s6dy9133210NIeWl5fH/v37y/588OBBkpOTCQgIoHnz5kycOJHnnnuONm3aEBERwdSpUwkNDWXo0KHGhXY0Rt+GVp+88sor9ubNm9vd3NzsPXv2tP/8889GR3IqwHkfb775ptHRnJ5ug788n3/+ub1Tp052d3d3e7t27eyvvfaa0ZGcTk5Ojv2RRx6xN2/e3O7h4WFv2bKl/R//+Ie9sLDQ6GgObe3atef9fTh27Fi73V56K/zUqVPtQUFBdnd3d/uAAQPse/bsMTa0gzHZ7ZpuU0REROoXjQESERGRekcFSEREROodFSARERGpd1SAREREpN5RARIREZF6RwVIRERE6h0VIBEREal3VIBERESk3lEBEhGpAJPJxKeffmp0DBGpJipAIuLw7rrrLkwm0zmP66+/3uhoIuKktBiqiDiF66+/njfffLPcNnd3d4PSiIiz0xkgEXEK7u7uBAcHl3v4+/sDpZenFi5cyA033ICnpyctW7bko48+Kvf67du3c9111+Hp6UmjRo24//77ycvLK7fPkiVL6NixI+7u7oSEhPC3v/2t3POZmZnccssteHl50aZNG1auXFmzH1pEaowKkIjUCVOnTmXYsGFs3bqVUaNGcccdd7Br1y4A8vPzGTRoEP7+/mzatIkPP/yQb7/9tlzBWbhwIQ899BD3338/27dvZ+XKlbRu3brcezz99NOMGDGCbdu2ceONNzJq1ChOnTpVq59TRKqJ0cvRi4hcytixY+0Wi8Xu7e1d7vH888/b7Xa7HbA/8MAD5V4THR1tHz9+vN1ut9tfe+01u7+/vz0vL6/s+S+//NJuNpvtaWlpdrvdbg8NDbX/4x//uGAGwP7Pf/6z7M95eXl2wP71119X2+cUkdqjMUAi4hSuvfZaFi5cWG5bQEBA2T/37t273HO9e/cmOTkZgF27dhEZGYm3t3fZ83379sVms7Fnzx5MJhPHjh1jwIABF83QpUuXsn/29vbG19eXjIyMy/1IImIgFSARcQre3t7nXJKqLp6enhXaz9XVtdyfTSYTNputJiKJSA3TGCARqRN+/vnnc/7cvn17ANq3b8/WrVvJz88ve37Dhg2YzWauuOIKfHx8CA8PJy4urlYzi4hxdAZIRJxCYWEhaWlp5ba5uLgQGBgIwIcffkhUVBRXXXUV7733HgkJCbzxxhsAjBo1iunTpzN27FieeuopTpw4wYQJExg9ejRBQUEAPPXUUzzwwAM0adKEG264gdzcXDZs2MCECRNq94OKSK1QARIRp7Bq1SpCQkLKbbviiivYvXs3UHqH1rJly3jwwQcJCQnhv//9Lx06dADAy8uL1atX88gjj9CjRw+8vLwYNmwYc+fOLTvW2LFjOXv2LC+99BKPP/44gYGBDB8+vPY+oIjUKpPdbrcbHUJEpCpMJhMrVqxg6NChRkcRESehMUAiIiJS76gAiYiISL2jMUAi4vR0JV9EKktngERERKTeUQESERGRekcFSEREROodFSARERGpd1SAREREpN5RARIREZF6RwVIRERE6h0VIBEREal3/g/t6DA7mszWXgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "torch.save(model.state_dict(), \"model_after_train.pt\")\n",
    "#plt.plot(recall_, precision_, marker='.', label='BERT-model')\n",
    "plt.xlabel('Epoch')\n",
    "plt.ylabel('Train Loss')\n",
    "plt.legend()\n",
    "plt.plot(all_losses, label='BERT-model')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "AZW9qp_KDFyv"
   },
   "source": [
    "## Test the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "IYObJpUmDFyv",
    "outputId": "70d57e10-d7e2-446a-9174-d546dc8ce54a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1000/4160. Current accuracy: 0.781\n",
      "2000/4160. Current accuracy: 0.799\n",
      "3000/4160. Current accuracy: 0.7996666666666666\n",
      "4000/4160. Current accuracy: 0.80075\n",
      "Accuracy on test data: 0.8\n"
     ]
    }
   ],
   "source": [
    "total = len(test_data)\n",
    "number_right = 0\n",
    "model.eval()\n",
    "with torch.no_grad():\n",
    "    for idx, row in test_data.iterrows():\n",
    "        text_parts = preprocess_text(str(row['text']))\n",
    "        label = torch.tensor([row['label']]).float().to(device)\n",
    "        \n",
    "        overall_output = torch.zeros((1,2)).to(device)\n",
    "        try:\n",
    "            for part in text_parts:\n",
    "                if len(part) > 0:\n",
    "                    overall_output += model(part.reshape(1, -1))[0]\n",
    "        except RuntimeError:\n",
    "            print(\"GPU out of memory, skipping this entry.\")\n",
    "            continue\n",
    "            \n",
    "        overall_output = F.softmax(overall_output[0], dim=-1)\n",
    "        acc=[]\n",
    "        result = overall_output.max(0)[1].float().item()\n",
    " \n",
    "        if result == label.item():\n",
    "            number_right += 1\n",
    "            \n",
    "        if idx % iter_size == 0 and idx > 0:\n",
    "            print(\"{}/{}. Current accuracy: {}\".format(idx, total, number_right / idx))\n",
    "            acc.append((number_right / idx))\n",
    "            \n",
    "print(\"Accuracy on test data: {}\".format(number_right / total))\n",
    "            \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "tSSW54o1SljG",
    "outputId": "517d3b8a-d7bb-438e-d09f-211a08ccad85"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1000/4160. Current accuracy: 0.781\n",
      "2000/4160. Current accuracy: 0.799\n",
      "3000/4160. Current accuracy: 0.7996666666666666\n",
      "4000/4160. Current accuracy: 0.80075\n",
      "Accuracy on test data: 0.8\n"
     ]
    }
   ],
   "source": [
    "\n",
    "total = len(test_data)\n",
    "acc=[]\n",
    "number_right = 0\n",
    "model.eval()\n",
    "with torch.no_grad():\n",
    "    for idx, row in test_data.iterrows():\n",
    "        text_parts = preprocess_text(str(row['text']))\n",
    "        label = torch.tensor([row['label']]).float().to(device)\n",
    "        \n",
    "        overall_output = torch.zeros((1,2)).to(device)\n",
    "        try:\n",
    "            for part in text_parts:\n",
    "                if len(part) > 0:\n",
    "                    overall_output += model(part.reshape(1, -1))[0]\n",
    "        except RuntimeError:\n",
    "            print(\"GPU out of memory, skipping this entry.\")\n",
    "            continue\n",
    "            \n",
    "        overall_output = F.softmax(overall_output[0], dim=-1)\n",
    "        \n",
    "        result = overall_output.max(0)[1].float().item()\n",
    " \n",
    "        if result == label.item():\n",
    "            number_right += 1\n",
    "            \n",
    "        if idx % iter_size == 0 and idx > 0:\n",
    "            print(\"{}/{}. Current accuracy: {}\".format(idx, total, number_right / idx))\n",
    "            acc.append((number_right / idx))\n",
    "            #print(acc)\n",
    "            \n",
    "print(\"Accuracy on test data: {}\".format(number_right / total))\n",
    "            \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "uwoJs7SJR0P6",
    "outputId": "60682fa4-ac3e-49d5-e684-fd63ab581f9a"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.781, 0.799, 0.7996666666666666, 0.80075]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "acc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plot the metrics obtained"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 472
    },
    "id": "AYoBr6WTRGrM",
    "outputId": "deeb1265-a7a2-4126-a5f3-226636a31dcd"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlEAAAHHCAYAAACfqw0dAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABjfUlEQVR4nO3deVhU9f4H8PcwzMKAgOwgCCqmkriEgqgpKYFGC2Uu3RQ0l6vikpQ3qbSb3uK2qaWm/gptszBMy5ZrGqKmouRCigsKLriwqmwDDDBzfn8gkxMgOAKH5f16nnkunPmeM585d2Lefs73nCMRBEEAEREREd0TE7ELICIiImqNGKKIiIiIjMAQRURERGQEhigiIiIiIzBEERERERmBIYqIiIjICAxRREREREZgiCIiIiIyAkMUERERkREYooioVbp06RIkEgk+++yze153z549kEgk2LNnT6PXRUTtB0MUERERkREYooiIiIiMwBBFRNRGqNVqsUsgalcYoojIKP/+978hkUhw7tw5TJw4EVZWVrC3t8fixYshCAKuXLmCp556CpaWlnBycsIHH3xQYxs5OTmYOnUqHB0doVQq0bdvX3z++ec1xuXn52Py5MmwsrKCtbU1wsPDkZ+fX2tdZ8+exbPPPgsbGxsolUoMGDAA27dvN+o9Xr58GbNnz0aPHj1gZmYGW1tbjB07FpcuXaq1xgULFsDDwwMKhQKurq4ICwtDXl6efkxZWRn+/e9/44EHHoBSqYSzszOeeeYZpKenA6h7rlZt878mT54MCwsLpKen47HHHkOHDh3w/PPPAwB+//13jB07Fp07d4ZCoYCbmxsWLFiA0tLSWvfXuHHjYG9vDzMzM/To0QOvvfYaACAhIQESiQTbtm2rsd7XX38NiUSCxMTEe92tRG2GqdgFEFHrNn78ePTq1Qv//e9/8fPPP+M///kPbGxssH79eowYMQLvvPMONm3ahJdffhkDBw7EsGHDAAClpaUICAhAWloa5syZgy5duiAuLg6TJ09Gfn4+5s+fDwAQBAFPPfUU9u/fj5kzZ6JXr17Ytm0bwsPDa9Ry6tQpDBkyBJ06dcKiRYtgbm6Ob7/9FqGhofjuu+/w9NNP39N7++OPP3Dw4EFMmDABrq6uuHTpEtauXYuAgACcPn0aKpUKAFBcXIyHH34YZ86cwQsvvICHHnoIeXl52L59O65evQo7OztotVo8/vjjiI+Px4QJEzB//nwUFRVh165dSElJQbdu3e5531dWViI4OBhDhw7F+++/r68nLi4OJSUlmDVrFmxtbZGUlIRVq1bh6tWriIuL069/4sQJPPzww5DJZJgxYwY8PDyQnp6OH3/8EW+99RYCAgLg5uaGTZs21dh3mzZtQrdu3eDv73/PdRO1GQIRkRHeeOMNAYAwY8YM/bLKykrB1dVVkEgkwn//+1/98lu3bglmZmZCeHi4ftnKlSsFAMJXX32lX1ZeXi74+/sLFhYWQmFhoSAIgvD9998LAIR3333X4HUefvhhAYCwceNG/fKRI0cK3t7eQllZmX6ZTqcTBg8eLHTv3l2/LCEhQQAgJCQk3PU9lpSU1FiWmJgoABC++OIL/bIlS5YIAIStW7fWGK/T6QRBEIQNGzYIAITly5fXOaauui5evFjjvYaHhwsAhEWLFjWo7ujoaEEikQiXL1/WLxs2bJjQoUMHg2V31iMIghAVFSUoFAohPz9fvywnJ0cwNTUV3njjjRqvQ9Se8HAeEd2XadOm6X+WSqUYMGAABEHA1KlT9cutra3Ro0cPXLhwQb/sl19+gZOTE5577jn9MplMhnnz5qG4uBh79+7VjzM1NcWsWbMMXmfu3LkGddy8eRO7d+/GuHHjUFRUhLy8POTl5eHGjRsIDg7G+fPnce3atXt6b2ZmZvqfKyoqcOPGDXh6esLa2hrHjh3TP/fdd9+hb9++tXa6JBKJfoydnV2Nuu8cY4w790ttdavVauTl5WHw4MEQBAHHjx8HAOTm5mLfvn144YUX0Llz5zrrCQsLg0ajwZYtW/TLNm/ejMrKSkycONHouonaAoYoIrovf/8CtrKyglKphJ2dXY3lt27d0v9++fJldO/eHSYmhn+GevXqpX+++n+dnZ1hYWFhMK5Hjx4Gv6elpUEQBCxevBj29vYGjzfeeANA1Ryse1FaWoolS5bAzc0NCoUCdnZ2sLe3R35+PgoKCvTj0tPT0bt377tuKz09HT169ICpaePNojA1NYWrq2uN5RkZGZg8eTJsbGxgYWEBe3t7DB8+HAD0dVcH2vrq7tmzJwYOHIhNmzbpl23atAmDBg2Cp6dnY70VolaJc6KI6L5IpdIGLQOq5jc1FZ1OBwB4+eWXERwcXOuYe/3Snzt3LjZu3IgXX3wR/v7+sLKygkQiwYQJE/Sv15jq6khptdpalysUihohVKvV4tFHH8XNmzfxyiuvoGfPnjA3N8e1a9cwefJko+oOCwvD/PnzcfXqVWg0Ghw6dAirV6++5+0QtTUMUUQkCnd3d5w4cQI6nc4gCJw9e1b/fPX/xsfHo7i42KAblZqaarC9rl27Aqg6JBgYGNgoNW7ZsgXh4eEGZxaWlZXVODOwW7duSElJueu2unXrhsOHD6OiogIymazWMR07dgSAGtuv7so1xMmTJ3Hu3Dl8/vnnCAsL0y/ftWuXwbjq/VVf3QAwYcIEREZG4ptvvkFpaSlkMhnGjx/f4JqI2ioeziMiUTz22GPIysrC5s2b9csqKyuxatUqWFhY6A8/PfbYY6isrMTatWv147RaLVatWmWwPQcHBwQEBGD9+vXIzMys8Xq5ubn3XKNUKq3RPVu1alWNztCYMWPw559/1nopgOr1x4wZg7y8vFo7ONVj3N3dIZVKsW/fPoPnP/7443uq+c5tVv/84YcfGoyzt7fHsGHDsGHDBmRkZNRaTzU7OzuMHj0aX331FTZt2oRRo0bVOFxL1B6xE0VEopgxYwbWr1+PyZMn4+jRo/Dw8MCWLVtw4MABrFy5Eh06dAAAPPHEExgyZAgWLVqES5cuwcvLC1u3bjWYk1RtzZo1GDp0KLy9vTF9+nR07doV2dnZSExMxNWrV/Hnn3/eU42PP/44vvzyS1hZWcHLywuJiYn47bffYGtrazBu4cKF2LJlC8aOHYsXXngBPj4+uHnzJrZv345169ahb9++CAsLwxdffIHIyEgkJSXh4Ycfhlqtxm+//YbZs2fjqaeegpWVFcaOHYtVq1ZBIpGgW7du+Omnn+5pLlfPnj3RrVs3vPzyy7h27RosLS3x3XffGcxHq/bRRx9h6NCheOihhzBjxgx06dIFly5dws8//4zk5GSDsWFhYXj22WcBAMuWLbun/UjUZol1WiARtW7VlzjIzc01WB4eHi6Ym5vXGD98+HDhwQcfNFiWnZ0tTJkyRbCzsxPkcrng7e1tcBp/tRs3bgiTJk0SLC0tBSsrK2HSpEnC8ePHa5z2LwiCkJ6eLoSFhQlOTk6CTCYTOnXqJDz++OPCli1b9GMaeomDW7du6euzsLAQgoODhbNnzwru7u4Gl2uornHOnDlCp06dBLlcLri6ugrh4eFCXl6efkxJSYnw2muvCV26dBFkMpng5OQkPPvss0J6erp+TG5urjBmzBhBpVIJHTt2FP75z38KKSkptV7ioLb9LAiCcPr0aSEwMFCwsLAQ7OzshOnTpwt//vlnrfsrJSVFePrppwVra2tBqVQKPXr0EBYvXlxjmxqNRujYsaNgZWUllJaW3nW/EbUXEkFowpmeRETUJlRWVsLFxQVPPPEEYmJixC6HqEXgnCgiIqrX999/j9zcXIPJ6kTtHTtRRERUp8OHD+PEiRNYtmwZ7OzsDC4yStTesRNFRER1Wrt2LWbNmgUHBwd88cUXYpdD1KKwE0VERERkBHaiiIiIiIzAEEVERERkBF5sswnpdDpcv34dHTp0uK+7tBMREVHzEQQBRUVFcHFxqXF/yjsxRDWh69evw83NTewyiIiIyAhXrlyBq6trnc8zRDWh6ttWXLlyBZaWliJXQ0RERA1RWFgINzc3/fd4XRiimlD1ITxLS0uGKCIiolamvqk4nFhOREREZASGKCIiIiIjMEQRERERGYFzokSm1WpRUVEhdhktmkwmg1QqFbsMIiIiAwxRIhEEAVlZWcjPzxe7lFbB2toaTk5OvN4WERG1GAxRIqkOUA4ODlCpVAwHdRAEASUlJcjJyQEAODs7i1wRERFRFYYoEWi1Wn2AsrW1FbucFs/MzAwAkJOTAwcHBx7aIyKiFoETy0VQPQdKpVKJXEnrUb2vOH+MiIhaCoYoEfEQXsNxXxERUUvDEEVERERkBIYoIiIiIiMwRNE9CQgIwIsvvtho25s8eTJCQ0MbbXtERETNhSGKiIiIWhVBEJBfUo7UrCJR6+AlDqjBJk+ejL1792Lv3r348MMPAQAXL15EcXExFi5ciN9//x3m5uYICgrCihUrYGdnBwDYsmUL3nzzTaSlpUGlUqF///744Ycf8N577+Hzzz8H8NfE8YSEBAQEBIjy/oiIqGUQBAF5xeW4eqsE1/JLce1WKa7eKr3j5xKoy7WQS01wdtkomJiIc/IRQ1QLIQgCSiu0ory2mUzaoLPfPvzwQ5w7dw69e/fG0qVLAVTdksXX1xfTpk3DihUrUFpaildeeQXjxo3D7t27kZmZieeeew7vvvsunn76aRQVFeH333+HIAh4+eWXcebMGRQWFmLjxo0AABsbmyZ9r0REJD6tTkBOUZlBOLp6q8QgKGkqdfVux9JMhvzSCtiYy5uh6poYolqI0gotvJb8Ksprn14aDJW8/o+ClZUV5HI5VCoVnJycAAD/+c9/0L9/f7z99tv6cRs2bICbmxvOnTuH4uJiVFZW4plnnoG7uzsAwNvbWz/WzMwMGo1Gvz0iImr9KrQ6ZBWU4ertrtHfu0mZBaWo0Ap33YZEAjhZKtHJ2gyuHc3QqaMZOlmr7vjZDEqZuBdfZoii+/Lnn38iISEBFhYWNZ5LT09HUFAQRo4cCW9vbwQHByMoKAjPPvssOnbsKEK1RETUGMoqtLieX1ojHF29VYJrt0qRVVgG3d0zEkxNJHC2rgpJd4Yj145mcLVWwclKCblpy566zRDVQpjJpDi9NFi01zZWcXExnnjiCbzzzjs1nnN2doZUKsWuXbtw8OBB7Ny5E6tWrcJrr72Gw4cPo0uXLvdTNhERNRG1pvKvgHRHOKoOS7lFmnq3ITc1gav1X12jv0KSCp2szeBoqYRUpLlMjYUhqoWQSCQNOqQmNrlcDq32r7lbDz30EL777jt4eHjA1LT2+iUSCYYMGYIhQ4ZgyZIlcHd3x7Zt2xAZGVlje0RE1PQKSiv0E7SrOkhVgam6m3SrpP5bbKnk0lrDUXU3yc5cIdqE7+bS8r+1qUXx8PDA4cOHcenSJVhYWCAiIgKffPIJnnvuOfzrX/+CjY0N0tLSEBsbi08//RRHjhxBfHw8goKC4ODggMOHDyM3Nxe9evXSb+/XX39FamoqbG1tYWVlBZlMJvK7JCJqvQRBwE11ea3hqPr3Ik1lvduxVJqiU8fbh9luhyXXO+YlWatk7f6WXAxRdE9efvllhIeHw8vLC6Wlpbh48SIOHDiAV155BUFBQdBoNHB3d8eoUaNgYmICS0tL7Nu3DytXrkRhYSHc3d3xwQcfYPTo0QCA6dOnY8+ePRgwYACKi4t5iQMionrodAJyizV1Ttq+dqu0QWd725rLDQ+1Wd/uJt3uLFkq+Q/a+kgEQahn6hcZq7CwEFZWVigoKIClpaV+eVlZGS5evIguXbpAqVSKWGHrwX1GRO1FpVaHrMKyOzpI1XOTquYlXc8vQ7m2/tP/HS0Vtw+v/dVN6tTRDG4dzeBibdYqppCIpa7v778TfQ+uWbMG7733HrKystC3b1+sWrUKvr6+dY5fuXIl1q5di4yMDNjZ2eHZZ59FdHS0wRdrfdssKyvDSy+9hNjYWGg0GgQHB+Pjjz+Go6OjfkxGRgZmzZqlP/MsPDwc0dHRdc77ISIiaghNpRaZ+WUGF468ekc3KauwDNp6Tm0zkQDOVrfnH9VyCQBnayUUpuKe/t8eiJoINm/ejMjISKxbtw5+fn5YuXIlgoODkZqaCgcHhxrjv/76ayxatAgbNmzA4MGDce7cOUyePBkSiQTLly9v8DYXLFiAn3/+GXFxcbCyssKcOXPwzDPP4MCBAwAArVaLkJAQODk54eDBg8jMzERYWBhkMpnB9ZCIiIj+rrRc+9fp/rXMS8op0qC+Y0AyqQQudxxm+/slAJwslTCVtuzT/9sDUQ/n+fn5YeDAgVi9ejUAQKfTwc3NDXPnzsWiRYtqjJ8zZw7OnDmD+Ph4/bKXXnoJhw8fxv79+xu0zYKCAtjb2+Prr7/Gs88+CwA4e/YsevXqhcTERAwaNAj/+9//8Pjjj+P69ev67tS6devwyiuvIDc3F3J5w66MysN5jYf7jIhaiqKyiqpAdPP2HKS/XQLghrq83m0oZSY1DrXdOXHboUPbP7OtJWvxh/PKy8tx9OhRREVF6ZeZmJggMDAQiYmJta4zePBgfPXVV0hKSoKvry8uXLiAX375BZMmTWrwNo8ePYqKigoEBgbqx/Ts2ROdO3fWh6jExER4e3sbHN4LDg7GrFmzcOrUKfTv379R9wUREbUMVTe2rdB3kGrrJhWU1n/6v4XC1CAc/f1q27bm8nZ/ZltbIFqIysvLg1arNQgqAODo6IizZ8/Wus4//vEP5OXlYejQoRAEAZWVlZg5cyZeffXVBm8zKysLcrkc1tbWNcZkZWXpx9S2jern6qLRaKDR/HUBssLCwjrHAlX/sVLDcF8RUWP4+41t9ZO27zjLTV1e/5lt1iqZ4Rlt1oZX27Y0M2VIagda1SzpPXv24O2338bHH38MPz8/pKWlYf78+Vi2bBkWL14sdnmIjo7Gm2++We+46usglZSUwMzMrKnLahNKSkoAgNeQIqK7qr6x7d/D0Z2dpIbc2NbOQvHXHKS/XVDSxdoMFopW9fVJTUS0T4GdnR2kUimys7MNlmdnZ9d5M9rFixdj0qRJmDZtGoCqG9mq1WrMmDEDr732WoO26eTkhPLycuTn5xt0o/4+JikpqcY2qp+rS1RUFCIjI/W/FxYWws3NrcY4qVQKa2tr5OTkAABUKhX/xVIHQRBQUlKCnJwcWFtbQyrl2SZE7Vn1jW2v3J6D9PdLAGTml6GynjPbarux7Z3dpJZwY1tqHUQLUXK5HD4+PoiPj0doaCiAqkng8fHxmDNnTq3rlJSUwMTE8GyE6i9VQRAatE0fHx/IZDLEx8djzJgxAIDU1FRkZGTA398fAODv74+33noLOTk5+jP6du3aBUtLS3h5edX5nhQKBRQKRYPef3UYqw5SdHfW1tZ3DbBE1DbceWPb2rpJ2fd4Y9vaDrW1hhvbUusgaj8yMjIS4eHhGDBgAHx9fbFy5Uqo1WpMmTIFABAWFoZOnTohOjoaAPDEE09g+fLl6N+/v/5w3uLFi/HEE0/ow1R927SyssLUqVMRGRkJGxsbWFpaYu7cufD398egQYMAAEFBQfDy8sKkSZPw7rvvIisrC6+//joiIiIaHJLqI5FI4OzsDAcHB1RU1D9JsT2TyWTsQBG1EQY3tr19faQ7D7Xd641t77yIZFu6sS21DqKGqPHjxyM3NxdLlixBVlYW+vXrhx07dugncWdkZBh0nl5//XVIJBK8/vrruHbtGuzt7fHEE0/grbfeavA2AWDFihUwMTHBmDFjDC62WU0qleKnn37CrFmz4O/vD3Nzc4SHh2Pp0qWNvg+kUikDAhG1WlqdgILSCtwqKUd+STluqg1/rvrfclwvqApKDb2xbW3hqPrQW3u4sS21DrztSxNq6HUmiIhagvJKHfJLynGrpCoI3VLX/Dm/pBw3S8qRf3t5QWlFvReO/Dve2JZauhZ/nSgiImo6peVa3LwdfvJLKm4Hn3Lcut0pulVyRyi6PaZYU2n063VQmqKjSo6OKhmsVXLYmMthrZLpl1XfooQ3tqW2hCGKiKgFEwQBRZpK5KurgtDfD5XdMghGFbc7RuUNOo2/NiYSwMpMho7mcn0A6qiSo+PtUGSjksP69vKqoFS1XMZbkFA7xBBFRNRM7pw/VOehMvVfh8pu3T5sVt8p+3WRSSVVXSHVHV0hc8NgdGfnqKNKBkuljPONiBqIIYqIyAh3zh+qCj5NM3+omplMWhV+bneIrO/oBHX828/VAclcLuXcIqImxBBFRO1eQ+cPVR8qa7T5Q3d2harnE91epj9sZl71PC/+SNTyMEQRUZtRPX+oIYfK7vz5fuYPWd95qKy6E3THfKI7D5Vx/hBR28IQRUQtUvX8obseKmvk+UPVQeiuh8ruCEicP0TUvjFEEVGTa8j8oTuD0E11OQrL7m/+kMEp9ndOoL4dhDh/iIjuF0MUETWYIAgordAazA+qvSvUuPOHakyavmP+kE0t1ybi/CEiag4MUUQEACgsq0DC2RzkFZc36fyh6iBU36GyjuZyWJlx/hARtVwMUUQEAHj52z+x83R2g8bKpSZ3HCr7KxTZ/O1n69tdIxuVHB2Uppw/RERtCkMUEeHU9QLsPJ0NEwnwmLfz7bPJ/t4duj3HiPOHiIgAMEQREYDVu9MAAI/3ccFHz/UXuRoiotaBkw2I2rnUrCL8LyULEgkwZ4Sn2OUQEbUaDFFE7dyq3ecBAI/1dsYDjh1EroaIqPVgiCJqx9JyivHzyUwA7EIREd0rhiiidmxNQhoEAQjyckQvZ0uxyyEialUYoojaqYt5avyQfA0AMG9kd5GrISJqfRiiiNqpjxPSoBOAET0d0LuTldjlEBG1OgxRRO3QlZsl2Hq8qgs1l3OhiIiMwhBF1A59vCcNWp2AYQ/Yo3/njmKXQ0TUKjFEEbUz1/JLseXoVQDAPHahiIiMxhBF1M6s25OOCq2Awd1sMcDDRuxyiIhaLYYoonYkq6AMm/+4AoBn5BER3S+GKKJ2ZP2+dJRrdfD1sMGgrrZil0NE1KoxRBG1EzlFZfj6cAYAdqGIiBoDQxRRO/Hp7xehqdShf2drDPFkF4qI6H4xRBG1AzeKNfgy8TKAqi6URCIRuSIiotaPIYqoHfh0/0WUVmjRx9UKAQ/Yi10OEVGbwBBF1Mbll5Tji4OXAABzR7ALRUTUWBiiiNq4DfsvQl2uRS9nSwT2chC7HCKiNoMhiqgNKyitwMYDlwAA80d6sgtFRNSIRA9Ra9asgYeHB5RKJfz8/JCUlFTn2ICAAEgkkhqPkJAQ/Zjs7GxMnjwZLi4uUKlUGDVqFM6fP69//tKlS7VuQyKRIC4uTj+utudjY2ObZicQNZHPD15CkaYSPRw7IMjLSexyiIjaFFFD1ObNmxEZGYk33ngDx44dQ9++fREcHIycnJxax2/duhWZmZn6R0pKCqRSKcaOHQsAEAQBoaGhuHDhAn744QccP34c7u7uCAwMhFqtBgC4ubkZbCMzMxNvvvkmLCwsMHr0aIPX27hxo8G40NDQJt0fRI2pqKwCMfsvAgDmjPCEiQm7UEREjclUzBdfvnw5pk+fjilTpgAA1q1bh59//hkbNmzAokWLaoy3sTG8z1dsbCxUKpU+RJ0/fx6HDh1CSkoKHnzwQQDA2rVr4eTkhG+++QbTpk2DVCqFk5Phv8i3bduGcePGwcLCwmC5tbV1jbFErcUXiZdRUFqBbvbmeMzbWexyiIjaHNE6UeXl5Th69CgCAwP/KsbEBIGBgUhMTGzQNmJiYjBhwgSYm5sDADQaDQBAqVQabFOhUGD//v21buPo0aNITk7G1KlTazwXEREBOzs7+Pr6YsOGDRAE4a71aDQaFBYWGjyIxKDWVBp0oaTsQhERNTrRQlReXh60Wi0cHR0Nljs6OiIrK6ve9ZOSkpCSkoJp06bpl/Xs2ROdO3dGVFQUbt26hfLycrzzzju4evUqMjMza91OTEwMevXqhcGDBxssX7p0Kb799lvs2rULY8aMwezZs7Fq1aq71hQdHQ0rKyv9w83Nrd73QdQUNh2+jJvqcnjYqvBEHxexyyEiapNEPZx3P2JiYuDt7Q1fX1/9MplMhq1bt2Lq1KmwsbGBVCpFYGAgRo8eXWsXqbS0FF9//TUWL15c47k7l/Xv3x9qtRrvvfce5s2bV2dNUVFRiIyM1P9eWFjIIEXNrrRci//bdwEAEPGIJ0ylop8/QkTUJon219XOzg5SqRTZ2dkGy7Ozs+udh6RWqxEbG1vrITgfHx8kJycjPz8fmZmZ2LFjB27cuIGuXbvWGLtlyxaUlJQgLCys3nr9/Pxw9epV/SHD2igUClhaWho8iJrbN0kZyCsuh2tHM4T27yR2OUREbZZoIUoul8PHxwfx8fH6ZTqdDvHx8fD397/runFxcdBoNJg4cWKdY6ysrGBvb4/z58/jyJEjeOqpp2qMiYmJwZNPPgl7+/pvg5GcnIyOHTtCoVDUO5ZILGUVWqzbmw6gqgslYxeKiKjJiHo4LzIyEuHh4RgwYAB8fX2xcuVKqNVq/dl6YWFh6NSpE6Kjow3Wi4mJQWhoKGxta96JPi4uDvb29ujcuTNOnjyJ+fPnIzQ0FEFBQQbj0tLSsG/fPvzyyy81tvHjjz8iOzsbgwYNglKpxK5du/D222/j5ZdfbsR3T9T4vj1yBTlFGrhYKTHmIVexyyEiatNEDVHjx49Hbm4ulixZgqysLPTr1w87duzQTzbPyMiAiYnhv6RTU1Oxf/9+7Ny5s9ZtZmZmIjIyEtnZ2XB2dkZYWFitc542bNgAV1fXGuEKqJpbtWbNGixYsACCIMDT01N/OQailkpTqcXaPVVdqFkB3SA3ZReKiKgpSYT6ztsnoxUWFsLKygoFBQWcH0VN7uvDGXh120k4Wiqwd+EjUMqkYpdERNQqNfT7m/9UJWoDKrQ6fLwnDQDwz2HdGKCIiJoBQxRRG7Dt+DVcvVUKOwsFnvPtLHY5RETtAkMUUStXqdVhTUJ1F6orzOTsQhERNQeGKKJW7scT13H5RglszOV4fhC7UEREzYUhiqgV0+oErNpd1YWa9nAXqOSt9iYEREStDkMUUSv288lMXMhVw1olQ5i/h9jlEBG1KwxRRK2UTidg9e7zAIAXhnSBhYJdKCKi5sQQRdRK/XoqC+eyi9FBaYrwwR5il0NE1O4wRBG1QoIg4KPbc6GmDOkCKzOZyBUREbU/DFFErdBvZ3JwJrMQ5nIpXhjiIXY5RETtEkMUUSsjCAI+iq+aCxU+2APWKrnIFRERtU8MUUStzJ7UXJy8VgCVXIppD3cVuxwionaLIYqoFREEAR/e7kJNHOQOG3N2oYiIxMIQRdSK7E/LQ/KVfChMTTCdXSgiIlExRBG1EnfOhXrezx32HRQiV0RE1L4xRBG1Eocu3MQfl25BbmqCfw5nF4qISGwMUUStRHUXasJANzhaKkWuhoiIGKKIWoE/Lt1E4oUbkEklmDm8m9jlEBERGKKIWoXqLtSzPm5wsTYTuRoiIgIYoohavOMZt/D7+TyYmkgwO4BdKCKiloIhiqiFW3X7HnlP9+8ENxuVyNUQEVE1hiiiFuzk1QLsPpsDEwkQ8Yin2OUQEdEdGKKIWrCPdlfNhQrt1wkeduYiV0NERHdiiCJqoU5fL8Su09mQSIDZ7EIREbU4DFFELdTqhKou1ON9XODpYCFyNURE9HcMUUQt0LnsIvxyMgsAMHcEu1BERC0RQxRRC7T69hl5o3s74QHHDiJXQ0REtWGIImph0nKK8eOJ6wCAOexCERG1WAxRRC3MxwlpEATgUS9HPOhiJXY5RERUB4YoohbkUp4aP/xZ1YWaN6K7yNUQEdHdMEQRtSAf70mDVifgkR728HZlF4qIqCUTPUStWbMGHh4eUCqV8PPzQ1JSUp1jAwICIJFIajxCQkL0Y7KzszF58mS4uLhApVJh1KhROH/+fL3bmTlzpsGYjIwMhISEQKVSwcHBAQsXLkRlZWXjvnmiO1y5WYKtx64BAOaOZBeKiKilMxXzxTdv3ozIyEisW7cOfn5+WLlyJYKDg5GamgoHB4ca47du3Yry8nL97zdu3EDfvn0xduxYAIAgCAgNDYVMJsMPP/wAS0tLLF++HIGBgTh9+jTMzf+64vP06dOxdOlS/e8q1V/3JNNqtQgJCYGTkxMOHjyIzMxMhIWFQSaT4e23326KXUGEtXvTUakT8HB3OzzUuaPY5RARUX0EEfn6+goRERH637VareDi4iJER0c3aP0VK1YIHTp0EIqLiwVBEITU1FQBgJCSkmKwTXt7e+GTTz7RLxs+fLgwf/78Orf7yy+/CCYmJkJWVpZ+2dq1awVLS0tBo9E09O0JBQUFAgChoKCgwetQ+3TtVong+erPgvsrPwlJF2+IXQ4RUbvW0O9v0Q7nlZeX4+jRowgMDNQvMzExQWBgIBITExu0jZiYGEyYMEHfYdJoNAAApVJpsE2FQoH9+/cbrLtp0ybY2dmhd+/eiIqKQklJif65xMREeHt7w9HRUb8sODgYhYWFOHXq1L2/WaJ6rNubjgqtgEFdbTDQw0bscoiIqAFEO5yXl5cHrVZrEFQAwNHREWfPnq13/aSkJKSkpCAmJka/rGfPnujcuTOioqKwfv16mJubY8WKFbh69SoyMzP14/7xj3/A3d0dLi4uOHHiBF555RWkpqZi69atAICsrKxa66p+ri4ajUYf5ACgsLCw3vdBlF1Yhtg/rgAA5nEuFBFRqyHqnKj7ERMTA29vb/j6+uqXyWQybN26FVOnToWNjQ2kUikCAwMxevRoCIKgHzdjxgz9z97e3nB2dsbIkSORnp6Obt26GV1TdHQ03nzzTaPXp/Zp/d4LKK/UYaBHR/h3tRW7HCIiaiDRDufZ2dlBKpUiOzvbYHl2djacnJzuuq5arUZsbCymTp1a4zkfHx8kJycjPz8fmZmZ2LFjB27cuIGuXbvWuT0/Pz8AQFpa1a02nJycaq2r+rm6REVFoaCgQP+4cuXKXd8HUW6RBpsOXwYAzB3RHRKJROSKiIiooUQLUXK5HD4+PoiPj9cv0+l0iI+Ph7+//13XjYuLg0ajwcSJE+scY2VlBXt7e5w/fx5HjhzBU089VefY5ORkAICzszMAwN/fHydPnkROTo5+zK5du2BpaQkvL686t6NQKGBpaWnwILqbT3+/AE2lDv3crPFwdzuxyyEionsg6uG8yMhIhIeHY8CAAfD19cXKlSuhVqsxZcoUAEBYWBg6deqE6Ohog/ViYmIQGhoKW9uahz7i4uJgb2+Pzp074+TJk5g/fz5CQ0MRFBQEAEhPT8fXX3+Nxx57DLa2tjhx4gQWLFiAYcOGoU+fPgCAoKAgeHl5YdKkSXj33XeRlZWF119/HREREVAoFE28V6i9uKkux5eHqrpQ80eyC0VE1NqIGqLGjx+P3NxcLFmyBFlZWejXrx927Nihn8SdkZEBExPDZllqair279+PnTt31rrNzMxMREZGIjs7G87OzggLC8PixYv1z8vlcvz222/6wObm5oYxY8bg9ddf14+RSqX46aefMGvWLPj7+8Pc3Bzh4eEG15Uiul8x+y+gpFwL705WCOhhL3Y5RER0jyTCnTOuqVEVFhbCysoKBQUFPLRHBvJLyjH0nQQUayrxf5N8EPTg3ecBEhFR82no97fot30hao82HLiEYk0lejlb4lEvx/pXICKiFochiqiZFZZVYOOBiwCAuSM8OReKiKiVYogiamafH7iEorJKdHewwCgexiMiarUYooiaUbGmEjHVXaiR3WFiwi4UEVFrxRBF1Iy+TLyM/JIKdLU3R4i3s9jlEBHRfWCIImomJeWV+OT3CwCAOY94QsouFBFRq8YQRdRMNh3KwE11OdxtVXiyr4vY5RAR0X1iiCJqBmUVWqzfV9WFigjwhKmU/+kREbV2/EtO1Ay+ScpAXrEGnazN8PRDncQuh4iIGgFDFFETK6vQYt3edADA7Ee6QcYuFBFRm8C/5kRNLO7oVWQXauBspcSzPq5il0NERI2EIYqoCZVX6rA2IQ0AMCugGxSmUpErIiKixsIQRdSEvjt2FdcLyuDQQYFxA9zELoeIiBoRQxRRE6nQ6vDxnqou1D+Hd4NSxi4UEVFbwhBF1ES+P34NV26Wws5Cjn/4dha7HCIiamQMUURNoFKrw5rbc6GmP9wVZnJ2oYiI2hqGKKIm8NOJTFy6UYKOKhkmDnIXuxwiImoCDFFEjUyrE7Bq93kAwLSHu8JcYSpyRURE1BQYooga2f9SMpGeq4aVmQxh/uxCERG1VQxRRI1IpxOwKr5qLtQLQ7qgg1ImckVERNRUGKKIGtHO01lIzS5CB4UpJg/xELscIiJqQgxRRI1EEAR8dLsLNXmIB6zM2IUiImrLGKKIGkn8mRycziyEuVyKF4Z0EbscIiJqYgxRRI1AEAR8dPuMvLDBHuhoLhe5IiIiamoMUUSNYO+5XJy4WgAzmRTThrILRUTUHjBEEd0nQRDwYXxVF2rioM6wtVCIXBERETUHhiii+3Qg7QaOZ+RDYWqC6cO6il0OERE1E4YoovtUPRfqOd/OcOigFLkaIiJqLgxRRPfh0IUbSLp4E3KpCWYO7yZ2OURE1IwYoojuw0e350KNG+gKJyt2oYiI2hOGKCIjHbl0EwfTb0AmlWBWgKfY5RARUTNjiCIy0ke7q65O/qyPKzpZm4lcDRERNTfRQ9SaNWvg4eEBpVIJPz8/JCUl1Tk2ICAAEomkxiMkJEQ/Jjs7G5MnT4aLiwtUKhVGjRqF8+fP65+/efMm5s6dix49esDMzAydO3fGvHnzUFBQYPBatb1ObGxs4+8AapWSr+Rj37lcSE0kmDWcXSgiovZI1BC1efNmREZG4o033sCxY8fQt29fBAcHIycnp9bxW7duRWZmpv6RkpICqVSKsWPHAqi6Xk9oaCguXLiAH374AcePH4e7uzsCAwOhVqsBANevX8f169fx/vvvIyUlBZ999hl27NiBqVOn1ni9jRs3GrxeaGhok+0Lal1W3Z4L9XT/TuhsqxK5GiIiEoNEEARBrBf38/PDwIEDsXr1agCATqeDm5sb5s6di0WLFtW7/sqVK7FkyRJkZmbC3Nwc586dQ48ePZCSkoIHH3xQv00nJye8/fbbmDZtWq3biYuLw8SJE6FWq2FqagqgqhO1bdu2+wpOhYWFsLKyQkFBASwtLY3eDrUsKdcK8Piq/TCRAPEvBaCLnbnYJRERUSNq6Pe3UZ2ohIQEowurVl5ejqNHjyIwMPCvYkxMEBgYiMTExAZtIyYmBhMmTIC5edWXmEajAQAolX+dJWViYgKFQoH9+/fXuZ3qnVQdoKpFRETAzs4Ovr6+2LBhA+rLmxqNBoWFhQYPantW3b4u1JN9XRigiIjaMaNC1KhRo9CtWzf85z//wZUrV4x64by8PGi1Wjg6Ohosd3R0RFZWVr3rJyUlISUlxaC71LNnT3Tu3BlRUVG4desWysvL8c477+Dq1avIzMyss45ly5ZhxowZBsuXLl2Kb7/9Frt27cKYMWMwe/ZsrFq16q41RUdHw8rKSv9wc3Or931Q63ImsxC/nsqGRALMGcG5UERE7ZlRIeratWuYM2cOtmzZgq5duyI4OBjffvstysvLG7u+OsXExMDb2xu+vr76ZTKZDFu3bsW5c+dgY2MDlUqFhIQEjB49GiYmNd9qYWEhQkJC4OXlhX//+98Gzy1evBhDhgxB//798corr+Bf//oX3nvvvbvWFBUVhYKCAv3D2IBJLdfq22fkhXg7w9Ohg8jVEBGRmIwKUXZ2dliwYAGSk5Nx+PBhPPDAA5g9ezZcXFwwb948/Pnnnw3ahlQqRXZ2tsHy7OxsODk53XVdtVqN2NjYWieD+/j4IDk5Gfn5+cjMzMSOHTtw48YNdO1qeE+zoqIijBo1Ch06dMC2bdsgk8nu+pp+fn64evWq/pBhbRQKBSwtLQ0e1Haczy7CLylVHU12oYiI6L7PznvooYcQFRWFOXPmoLi4GBs2bICPjw8efvhhnDp1qs715HI5fHx8EB8fr1+m0+kQHx8Pf3//u75mXFwcNBoNJk6cWOcYKysr2Nvb4/z58zhy5Aieeuop/XOFhYUICgqCXC7H9u3bDeZQ1SU5ORkdO3aEQqGodyy1TasT0iAIwKgHndDTiQGZiKi9MzpEVVRUYMuWLXjsscfg7u6OX3/9FatXr0Z2djbS0tLg7u6uv/RAXSIjI/HJJ5/g888/x5kzZzBr1iyo1WpMmTIFABAWFoaoqKga68XExCA0NBS2trY1nouLi8OePXv0lzl49NFHERoaiqCgIAB/BSi1Wo2YmBgUFhYiKysLWVlZ0Gq1AIAff/wRn376KVJSUpCWloa1a9fi7bffxty5c43dXdTKXcgtxo9/XgcAzB3JLhQREQGm9Q+pae7cufjmm28gCAImTZqEd999F71799Y/b25ujvfffx8uLi533c748eORm5uLJUuWICsrC/369cOOHTv0k80zMjJqzGVKTU3F/v37sXPnzlq3mZmZicjISGRnZ8PZ2RlhYWFYvHix/vljx47h8OHDAABPT8Mvw4sXL8LDwwMymQxr1qzBggULIAgCPD09sXz5ckyfPr3hO4nalDUJ6dAJQGAvBzzoYiV2OURE1AIYdZ2okSNHYtq0aXjmmWfqPLxVWVmJAwcOYPjw4fddZGvF60S1DZdvqDHig73Q6gT8EDEEfd2sxS6JiIiaUEO/v43qRN05j6nODZuatusARW3Hxwnp0OoEDH/AngGKiIj0jJoTFR0djQ0bNtRYvmHDBrzzzjv3XRRRS3H1Vgm+O3YVADBvZHeRqyEiopbEqBC1fv169OzZs8byBx98EOvWrbvvoohairV70lGpEzDU0w4+7h3FLoeIiFoQo0JUVlYWnJ2dayy3t7ev88rgRK1NZkEp4o5UdaHm8rpQRET0N0aFKDc3Nxw4cKDG8gMHDtR7Rh5Ra7F+7wWUa3Xw62IDv641L6dBRETtm1ETy6dPn44XX3wRFRUVGDFiBICqyeb/+te/8NJLLzVqgURiyCksw9dJGQCA+ZwLRUREtTAqRC1cuBA3btzA7Nmz9ffLUyqVeOWVV2q9OCZRa7N+3wWUV+rg494R/t3YhSIiopqMuk5UteLiYpw5cwZmZmbo3r07b4nyN7xOVOuUV6zB0Hd2o6xCh89f8MXwB+zFLomIiJpRk14nqpqFhQUGDhx4P5sganE++f0Cyip06OtmjWHd7cQuh4iIWiijQ9SRI0fw7bffIiMjQ39Ir9rWrVvvuzAiMdxUl+PLxMsAgHkjPCGRSESuiIiIWiqjzs6LjY3F4MGDcebMGWzbtg0VFRU4deoUdu/eDSsr3leMWq8N+y+ipFyLB10sMaKng9jlEBFRC2ZUiHr77bexYsUK/Pjjj5DL5fjwww9x9uxZjBs3Dp07d27sGomaRUFJBT47eAlA1dXJ2YUiIqK7MSpEpaenIyQkBAAgl8uhVqshkUiwYMEC/N///V+jFkjUXDYevIhiTSV6OnXAo70cxS6HiIhaOKNCVMeOHVFUVAQA6NSpE1JSUgAA+fn5KCkpabzqiJpJYVkFNuy/CACYO6I7TEzYhSIiorszamL5sGHDsGvXLnh7e2Ps2LGYP38+du/ejV27dmHkyJGNXSNRk/vi4CUUllWiu4MFRvd2ErscIiJqBYwKUatXr0ZZWRkA4LXXXoNMJsPBgwcxZswYvP76641aIFFTK9ZU4tPbXag5IzzZhSIioga55xBVWVmJn376CcHBwQAAExMTLFq0qNELI2ouXx26jPySCnS1M8fjfXjvRyIiaph7nhNlamqKmTNn6jtRRK1ZSXklPtl3AQAw+xFPSNmFIiKiBjJqYrmvry+Sk5MbuRSi5vf14QzcUJejs40KT/VjF4qIiBrOqDlRs2fPRmRkJK5cuQIfHx+Ym5sbPN+nT59GKY6oKZVVaLH+dhcq4pFukEmN+jcFERG1U0aFqAkTJgAA5s2bp18mkUggCAIkEgm0Wm3jVEfUhGKTMpBbpEEnazM83d9V7HKIiKiVMSpEXbx4sbHrIGpWmkot1u2t6kLNCugGuSm7UEREdG+MClHu7u6NXQdRs4o7chVZhWVwslRi7AB2oYiI6N4ZFaK++OKLuz4fFhZmVDFEzaG8Uoe1e9IBADOHd4XCVCpyRURE1BoZFaLmz59v8HtFRQVKSkogl8uhUqkYoqhF23b8Kq7ll8K+gwITfHnDbCIiMo5RE0Fu3bpl8CguLkZqaiqGDh2Kb775prFrJGo0FVodViekAQD+OawrlDJ2oYiIyDiNNpu2e/fu+O9//1ujS0XUkvyQfB1XbpbC1lyO5/04t4+IiIzXqKckmZqa4vr16425SaJGo9UJWHO7CzV9WFeYydmFIiIi4xk1J2r79u0GvwuCgMzMTKxevRpDhgxplMKIGttPJ67jYp4aHVUyTBrELhQREd0fo0JUaGiowe8SiQT29vYYMWIEPvjgg8aoi6hR6XQCVu2u6kJNHdoF5gqjPvpERER6Rn2T6HS6xq6DqEn9LyULaTnFsFSaImywh9jlEBFRG8DLNFObV9WFOg8AeGFoF1gqZSJXREREbYFRIWrMmDF45513aix/9913MXbs2Hva1po1a+Dh4QGlUgk/Pz8kJSXVOTYgIAASiaTGIyQkRD8mOzsbkydPhouLC1QqFUaNGoXz588bbKesrAwRERGwtbWFhYUFxowZg+zsbIMxGRkZCAkJgUqlgoODAxYuXIjKysp7em/UMuw6k42zWUWwUJhiyuAuYpdDRERthFEhat++fXjsscdqLB89ejT27dvX4O1s3rwZkZGReOONN3Ds2DH07dsXwcHByMnJqXX81q1bkZmZqX+kpKRAKpXqg5sgCAgNDcWFCxfwww8/4Pjx43B3d0dgYCDUarV+OwsWLMCPP/6IuLg47N27F9evX8czzzyjf16r1SIkJATl5eU4ePAgPv/8c3z22WdYsmRJg98btQyCIOCj+KoQPXmwB6xU7EIREVEjEYygVCqFs2fP1lh+5swZQalUNng7vr6+QkREhP53rVYruLi4CNHR0Q1af8WKFUKHDh2E4uJiQRAEITU1VQAgpKSkGGzT3t5e+OSTTwRBEIT8/HxBJpMJcXFxBnUDEBITEwVBEIRffvlFMDExEbKysvRj1q5dK1haWgoajabB76+goEAAIBQUFDR4HWpcv53OEtxf+Unotfh/wo3ihv9/R0RE7VdDv7+N6kR5e3tj8+bNNZbHxsbCy8urQdsoLy/H0aNHERgYqF9mYmKCwMBAJCYmNmgbMTExmDBhAszNzQEAGo0GAKBUKg22qVAosH//fgDA0aNHUVFRYfC6PXv2ROfOnfWvm5iYCG9vbzg6OurHBAcHo7CwEKdOnaqzHo1Gg8LCQoMHiUcQBHx0+4y8Sf7usDGXi1wRERG1JUadnbd48WI888wzSE9Px4gRIwAA8fHx+OabbxAXF9egbeTl5UGr1RoEFQBwdHTE2bNn610/KSkJKSkpiImJ0S+rDkNRUVFYv349zM3NsWLFCly9ehWZmZkAgKysLMjlclhbW9d43aysLP2Y2uqqfq4u0dHRePPNN+utnZrHvvN5+PNKPpQyE0x/uKvY5RARURtjVCfqiSeewPfff4+0tDTMnj0bL730Eq5evYrffvutxjWkmkpMTAy8vb3h6+urXyaTybB161acO3cONjY2UKlUSEhIwOjRo2Fi0vQnIkZFRaGgoED/uHLlSpO/JtVOuGMu1PN+7rCzUIhcERERtTVGX3EwJCTE4Ky4e2VnZwepVFrjrLjs7Gw4OTnddV21Wo3Y2FgsXbq0xnM+Pj5ITk5GQUEBysvLYW9vDz8/PwwYMAAA4OTkhPLycuTn5xt0o+58XScnpxpnCVbXebfaFAoFFAp+WbcEiek3cPTyLchNTfDPYexCERFR4zOqPfPHH3/g8OHDNZYfPnwYR44cadA25HI5fHx8EB8fr1+m0+kQHx8Pf3//u64bFxcHjUaDiRMn1jnGysoK9vb2OH/+PI4cOYKnnnoKQFXIkslkBq+bmpqKjIwM/ev6+/vj5MmTBmcJ7tq1C5aWlg2e80Xi+vB2F+ofvp3hYKmsZzQREdG9MypERURE1Hqo6tq1a4iIiGjwdiIjI/HJJ5/g888/x5kzZzBr1iyo1WpMmTIFABAWFoaoqKga68XExCA0NBS2trY1nouLi8OePXv0lzl49NFHERoaiqCgIABV4Wrq1KmIjIxEQkICjh49iilTpsDf3x+DBg0CAAQFBcHLywuTJk3Cn3/+iV9//RWvv/46IiIi2GlqBQ5fuIHDF29CLjXBP4ezC0VERE3DqMN5p0+fxkMPPVRjef/+/XH69OkGb2f8+PHIzc3FkiVLkJWVhX79+mHHjh36SdwZGRk15jKlpqZi//792LlzZ63bzMzMRGRkJLKzs+Hs7IywsDAsXrzYYMyKFStgYmKCMWPGQKPRIDg4GB9//LH+ealUip9++gmzZs2Cv78/zM3NER4eXuvhQ2p5qu+RN3aAK5ytzESuhoiI2iqJIAjCva5ka2uLn376qcZht4MHDyIkJAS3bt1qtAJbs8LCQlhZWaGgoACWlpZil9MuHL18E2PWJsLURII9CwPg2lEldklERNTKNPT726jDeUFBQfoz0arl5+fj1VdfxaOPPmrMJokaxUfxVV2oMQ+5MkAREVGTMupw3vvvv49hw4bB3d0d/fv3BwAkJyfD0dERX375ZaMWSNRQf17Jx95zuZCaSDD7kW5il0NERG2cUSGqU6dOOHHiBDZt2oQ///wTZmZmmDJlCp577jnIZLw3GYlj1e6qM/JC+3WCu625yNUQEVFbZ/R1oszNzTF06FB07twZ5eXlAID//e9/AIAnn3yycaojaqCUawX47UwOTCRABLtQRETUDIwKURcuXMDTTz+NkydPQiKRQBAESCQS/fNarbbRCiRqiNW3z8h7oq8LutpbiFwNERG1B0ZNLJ8/fz66dOmCnJwcqFQqpKSkYO/evRgwYAD27NnTyCUS3d3ZrELsOJUFiQSY84in2OUQEVE7YVQnKjExEbt374adnR1MTEwglUoxdOhQREdHY968eTh+/Hhj10lUp+rrQj3W2xndHTuIXA0REbUXRnWitFotOnSo+rKys7PD9evXAQDu7u5ITU1tvOqI6pGWU4RfTmYCAOaMYBeKiIiaj1GdqN69e+PPP/9Ely5d4Ofnh3fffRdyuRz/93//h65deZsNaj6rd6dBEIAgL0f0cuYFTYmIqPkYFaJef/11qNVqAMDSpUvx+OOP4+GHH4atrS02b97cqAUS1eVinhrb/6zqgs4b2V3kaoiIqL0xKkQFBwfrf/b09MTZs2dx8+ZNdOzY0eAsPaKmtCYhDToBGNnTAb07WYldDhERtTNGXyfq72xsbBprU0T1yrhRgm3HrwEA5rILRUREIjBqYjmR2NbuTYNWJ2DYA/bo52YtdjlERNQOMURRq3P1Vgm2HL0KAJg/kmfkERGROBiiqNVZtzcdFVoBg7vZwsedh5GJiEgcDFHUqmQVlOHbP6q6UDwjj4iIxMQQRa3Kur3pKNfq4NvFBoO62opdDhERtWMMUdRq5BSV4ZukDADAvBHsQhERkbgYoqjV+GTfBWgqdXioszWGeLILRURE4mKIolYhr1iDrw7d7kKN7M6LuhIRkegYoqhV+PT3iyit0KKPqxWGP2AvdjlEREQMUdTy3VKX48vESwCq5kKxC0VERC0BQxS1eBsOXIS6XAsvZ0uM7OUgdjlEREQAGKKohSsorcBnBy4BAOaN9GQXioiIWgyGKGrRPjtwCUWaSvRw7IAgLyexyyEiItJjiKIWq6isAjH7LwAA5ozwhIkJu1BERNRyMERRi/VF4mUUllWim705HvN2FrscIiIiAwxR1CKpNZX49PeqLtTcEd0hZReKiIhaGIYoapG+OnQZt0oq4GGrwuN92IUiIqKWhyGKWpzSci0+ud2FinjEE6ZSfkyJiKjl4bcTtThfJ2Ugr7gcbjZmCO3fSexyiIiIaiV6iFqzZg08PDygVCrh5+eHpKSkOscGBARAIpHUeISEhOjHFBcXY86cOXB1dYWZmRm8vLywbt06/fOXLl2qdRsSiQRxcXH6cbU9Hxsb2zQ7gfTKKrRYvzcdADA7wBMydqGIiKiFMhXzxTdv3ozIyEisW7cOfn5+WLlyJYKDg5GamgoHh5pXpt66dSvKy8v1v9+4cQN9+/bF2LFj9csiIyOxe/dufPXVV/Dw8MDOnTsxe/ZsuLi44Mknn4SbmxsyMzMNtvt///d/eO+99zB69GiD5Rs3bsSoUaP0v1tbWzfSO6e6fHvkCnKKNHCxUmLMQ65il0NERFQnUf+Zv3z5ckyfPh1TpkzRd4xUKhU2bNhQ63gbGxs4OTnpH7t27YJKpTIIUQcPHkR4eDgCAgLg4eGBGTNmoG/fvvoOl1QqNdiGk5MTtm3bhnHjxsHCwsLg9aytrQ3GKZXKptsZBE2lFmv3VHWhZj3iCbkpu1BERNRyifYtVV5ejqNHjyIwMPCvYkxMEBgYiMTExAZtIyYmBhMmTIC5ubl+2eDBg7F9+3Zcu3YNgiAgISEB586dQ1BQUK3bOHr0KJKTkzF16tQaz0VERMDOzg6+vr7YsGEDBEG4az0ajQaFhYUGD2q4LUevIrOgDI6WCoz1YReKiIhaNtEO5+Xl5UGr1cLR0dFguaOjI86ePVvv+klJSUhJSUFMTIzB8lWrVmHGjBlwdXWFqakpTExM8Mknn2DYsGG1bicmJga9evXC4MGDDZYvXboUI0aMgEql0h8SLC4uxrx58+qsKTo6Gm+++Wa9tVNNFVodPk6o6kLNHN4NSplU5IqIiIjuTtQ5UfcjJiYG3t7e8PX1NVi+atUqHDp0CNu3b4e7uzv27duHiIgIuLi4GHS9AKC0tBRff/01Fi9eXGP7dy7r378/1Go13nvvvbuGqKioKERGRup/LywshJubm7FvsV3ZduwaruWXws5Cged8O4tdDhERUb1EC1F2dnaQSqXIzs42WJ6dnQ0np7vfaFatViM2NhZLly41WF5aWopXX30V27Zt05+x16dPHyQnJ+P999+vEaK2bNmCkpIShIWF1Vuvn58fli1bBo1GA4VCUesYhUJR53NUt0qtDqsT0gAA/xzWlV0oIiJqFUSbEyWXy+Hj44P4+Hj9Mp1Oh/j4ePj7+9913bi4OGg0GkycONFgeUVFBSoqKmBiYvi2pFIpdDpdje3ExMTgySefhL29fb31Jicno2PHjgxJTWD7n9eRcbMENuZyPD+IXSgiImodRD2cFxkZifDwcAwYMAC+vr5YuXIl1Go1pkyZAgAICwtDp06dEB0dbbBeTEwMQkNDYWtra7Dc0tISw4cPx8KFC2FmZgZ3d3fs3bsXX3zxBZYvX24wNi0tDfv27cMvv/xSo64ff/wR2dnZGDRoEJRKJXbt2oW3334bL7/8ciPvAdLqBKzeXdWFmv5wV6jkrfYIMxERtTOifmONHz8eubm5WLJkCbKystCvXz/s2LFDP9k8IyOjRlcpNTUV+/fvx86dO2vdZmxsLKKiovD888/j5s2bcHd3x1tvvYWZM2cajNuwYQNcXV1rPWtPJpNhzZo1WLBgAQRBgKenp/5yDNS4fjpxHRfy1LBWyTDJ313scoiIiBpMItR33j4ZrbCwEFZWVigoKIClpaXY5bQ4Op2A4JX7cD6nGC89+gDmjuwudklEREQN/v7m1QxJNDtOZeF8TjE6KE0RPsRD7HKIiIjuCUMUiUKnE/BR/HkAwJQhXWCplIlcERER0b1hiCJR/HYmG2ezimChMMUL7EIREVErxBBFzU4QBHy0u6oLFebvDmuVXOSKiIiI7h1DFDW7Pam5SLlWCJVcimkPdxW7HCIiIqMwRFGzEgQBH96eCzVpkDtszNmFIiKi1okhiprV7+fzkHwlH0qZCbtQRETUqjFEUbMRhL/OyPuHrzvsO/AWOkRE1HoxRFGzSbxwA0cu34Lc1AT/HM4uFBERtW4MUdRsqrtQEwa6wdFSKXI1RERE94chippF0sWbOHThJmRSCWYO7yZ2OURERPeNIYqaxarb14UaO8ANLtZmIldDRER0/xiiqMkdy7iF38/nwdREglnsQhERURvBEEVNbtXtuVDPPNQJbjYqkashIiJqHAxR1KROXM1HQmoupCYSRDziKXY5REREjYYhiprUR/FpAICn+rrA3dZc5GqIiIgaD0MUNZlT1wvw25lsSCRAxAh2oYiIqG1hiKIms3p3VRfq8T4u6GZvIXI1REREjYshippEalYR/peSBQCYyy4UERG1QQxR1CRWJ1R1oR7zdsIDjh1EroaIiKjxMURRo0vLKcZPJ64DAOY80l3kaoiIiJoGQxQ1ujUJaRAE4FEvR3i5WIpdDhERUZNgiKJGdSlPjR+SrwEA5o1gF4qIiNouhihqVGsS0qATgEd62MPb1UrscoiIiJoMQxQ1mis3S7DteFUXau5IdqGIiKhtY4iiRvPxnnRU6gQ83N0OD3XuKHY5RERETYohihrFtfxSbDl6BQAwj10oIiJqBxiiqFGs35uOCq0A/662GOhhI3Y5RERETY4hiu5bVkEZYpPYhSIiovaFIYru2/p96SjX6jDQoyMGdWUXioiI2geGKLovOUVl+PpwBoCqLpREIhG5IiIioubBEEX35dPfL0JTqUP/ztYY6mkndjlERETNRvQQtWbNGnh4eECpVMLPzw9JSUl1jg0ICIBEIqnxCAkJ0Y8pLi7GnDlz4OrqCjMzM3h5eWHdunX1bmfmzJkGYzIyMhASEgKVSgUHBwcsXLgQlZWVjfvmW7kbxRp8mXgZQNXVydmFIiKi9sRUzBffvHkzIiMjsW7dOvj5+WHlypUIDg5GamoqHBwcaozfunUrysvL9b/fuHEDffv2xdixY/XLIiMjsXv3bnz11Vfw8PDAzp07MXv2bLi4uODJJ5/Uj5s+fTqWLl2q/12lUul/1mq1CAkJgZOTEw4ePIjMzEyEhYVBJpPh7bffbuzd0GrF7L+I0gotvDtZIaCHvdjlEBERNStRO1HLly/H9OnTMWXKFH3HSKVSYcOGDbWOt7GxgZOTk/6xa9cuqFQqgxB18OBBhIeHIyAgAB4eHpgxYwb69u1bo8OlUqkMtmVp+deNcnfu3InTp0/jq6++Qr9+/TB69GgsW7YMa9asMQhx7Vl+STk+P3gJADB3hCe7UERE1O6IFqLKy8tx9OhRBAYG/lWMiQkCAwORmJjYoG3ExMRgwoQJMDc31y8bPHgwtm/fjmvXrkEQBCQkJODcuXMICgoyWHfTpk2ws7ND7969ERUVhZKSEv1ziYmJ8Pb2hqOjo35ZcHAwCgsLcerUqTrr0Wg0KCwsNHi0VRv2X4S6XItezpZ41Mux/hWIiIjaGNEO5+Xl5UGr1RoEFQBwdHTE2bNn610/KSkJKSkpiImJMVi+atUqzJgxA66urjA1NYWJiQk++eQTDBs2TD/mH//4B9zd3eHi4oITJ07glVdeQWpqKrZu3QoAyMrKqrWu6ufqEh0djTfffLPe2lu7gtIKbLzdhZrHLhQREbVTos6Juh8xMTHw9vaGr6+vwfJVq1bh0KFD2L59O9zd3bFv3z5ERETAxcVF3/WaMWOGfry3tzecnZ0xcuRIpKeno1u3bkbXFBUVhcjISP3vhYWFcHNzM3p7LdXnBy+hqKwSDzhaIPhBJ7HLISIiEoVoIcrOzg5SqRTZ2dkGy7Ozs+HkdPcvZrVajdjYWIOJ4QBQWlqKV199Fdu2bdOfsdenTx8kJyfj/fffNzh0eCc/Pz8AQFpaGrp16wYnJ6cac6iq67xbbQqFAgqF4q61t3ZFZRWI2X8RADBnRHeYmLALRURE7ZNoc6Lkcjl8fHwQHx+vX6bT6RAfHw9/f/+7rhsXFweNRoOJEycaLK+oqEBFRQVMTAzfllQqhU6nq3N7ycnJAABnZ2cAgL+/P06ePImcnBz9mF27dsHS0hJeXl4Nen9t1ZeHLqOgtAJd7c0R4u0sdjlERESiEfVwXmRkJMLDwzFgwAD4+vpi5cqVUKvVmDJlCgAgLCwMnTp1QnR0tMF6MTExCA0Nha2trcFyS0tLDB8+HAsXLoSZmRnc3d2xd+9efPHFF1i+fDkAID09HV9//TUee+wx2Nra4sSJE1iwYAGGDRuGPn36AACCgoLg5eWFSZMm4d1330VWVhZef/11REREtPlO092oNZX49PfbXahHPCFlF4qIiNoxUUPU+PHjkZubiyVLliArKwv9+vXDjh079JO4MzIyanSVUlNTsX//fuzcubPWbcbGxiIqKgrPP/88bt68CXd3d7z11lv6i2nK5XL89ttv+sDm5uaGMWPG4PXXX9dvQyqV4qeffsKsWbPg7+8Pc3NzhIeH1zh82N5sOnwZN9XlcLdV4cm+LmKXQ0REJCqJIAiC2EW0VYWFhbCyskJBQYHBdahao9JyLR5+NwF5xRq8+2wfjBvQ9ibMExERAQ3//hb9ti/UOnyTlIG8Yg1cO5rh6f6dxC6HiIhIdAxRVK+yCi3W70sHAMwO8IRMyo8NERERvw2pXnFHriC7UAMXKyXG+LALRUREBDBEUT3KK3VYu6eqCzUzoBsUplKRKyIiImoZGKLorr47dhXXC8rg0EHByeRERER3YIiiOlVodViTkAYAmDm8G5QydqGIiIiqMURRnbYdv4art0phZyHHc76dxS6HiIioRWGIolpVanX4+HYXasawrjCTswtFRER0J4YoqtWPJ67j0o0SdFTJ8Lyfu9jlEBERtTgMUVSDVidg1e6qLtS0h7vCXCHq3YGIiIhaJIYoquGXk5m4kKuGlZkMYf7sQhEREdWGIYoM6HQCVu0+DwB4YUgXdFDKRK6IiIioZWKIIgO/nsrCuexidFCYYvIQD7HLISIiarEYokhPEAR8dHsu1JQhHrAyYxeKiIioLgxRpPfbmRycySyEuVyKF4Z2EbscIiKiFo0higDc7kLFV82FChvsAWuVXOSKiIiIWjaGKAIA7DmXi5PXCmAmk2Iau1BERET1YogiCIKAD3+r6kJNHNQZthYKkSsiIiJq+RiiCPvT8pB8JR8KUxNMH9ZV7HKIiIhaBYaodu7OuVD/8OsMhw5KkSsiIiJqHRii2rlDF27ij0u3IJea4J/DuoldDhERUavBENXOVV+dfPxANzhZsQtFRETUUAxR7dgfl27iYPoNyKQSzAxgF4qIiOheMES1Y9VzoZ71cUUnazORqyEiImpdGKLaqeMZt/D7+TxITSSYHeApdjlEREStDkNUO7Xq9j3ynu7fCW42KpGrISIian0Yotqhk1cLsPtsDkwkQMQj7EIREREZgyGqHao+I++pfp3Qxc5c5GqIiIhaJ4aodub09ULsPJ0NCbtQRERE94Uhqp1ZnVDVhQrxdoang4XI1RAREbVeDFHtyLnsIvwvJQsAMHdEd5GrISIiat1ED1Fr1qyBh4cHlEol/Pz8kJSUVOfYgIAASCSSGo+QkBD9mOLiYsyZMweurq4wMzODl5cX1q1bp3/+5s2bmDt3Lnr06AEzMzN07twZ8+bNQ0FBgcFr1fY6sbGxjb8DmtHq3WkQBGDUg07o4dRB7HKIiIhaNVMxX3zz5s2IjIzEunXr4Ofnh5UrVyI4OBipqalwcHCoMX7r1q0oLy/X/37jxg307dsXY8eO1S+LjIzE7t278dVXX8HDwwM7d+7E7Nmz4eLigieffBLXr1/H9evX8f7778PLywuXL1/GzJkzcf36dWzZssXg9TZu3IhRo0bpf7e2tm78ndBM0nOL8eOJ6wCAuSM5F4qIiOh+SQRBEMR6cT8/PwwcOBCrV68GAOh0Ori5uWHu3LlYtGhRveuvXLkSS5YsQWZmJszNq84y6927N8aPH4/Fixfrx/n4+GD06NH4z3/+U+t24uLiMHHiRKjVapiaVuVKiUSCbdu2ITQ01Oj3V1hYCCsrKxQUFMDS0tLo7TSGyG+TsfXYNQT2csSn4QNErYWIiKgla+j3t2iH88rLy3H06FEEBgb+VYyJCQIDA5GYmNigbcTExGDChAn6AAUAgwcPxvbt23Ht2jUIgoCEhAScO3cOQUFBdW6neidVB6hqERERsLOzg6+vLzZs2ID68qZGo0FhYaHBoyW4lKfGD8lVXah57EIRERE1CtEO5+Xl5UGr1cLR0dFguaOjI86ePVvv+klJSUhJSUFMTIzB8lWrVmHGjBlwdXWFqakpTExM8Mknn2DYsGF11rFs2TLMmDHDYPnSpUsxYsQIqFQq/SHB4uJizJs3r86aoqOj8eabb9Zbe3P7eE8atDoBAT3s0cfVWuxyiIiI2gRR50Tdj5iYGHh7e8PX19dg+apVq3Do0CFs374d7u7u2LdvHyIiIuDi4mLQ9QKq2nUhISHw8vLCv//9b4Pn7jwc2L9/f6jVarz33nt3DVFRUVGIjIw02L6bm9t9vMv7d+VmCbYeuwaAZ+QRERE1JtFClJ2dHaRSKbKzsw2WZ2dnw8nJ6a7rqtVqxMbGYunSpQbLS0tL8eqrr2Lbtm36M/b69OmD5ORkvP/++wYhqqioCKNGjUKHDh2wbds2yGSyu76mn58fli1bBo1GA4VCUesYhUJR53NiWbs3HZU6AUM97eDj3lHscoiIiNoM0eZEyeVy+Pj4ID4+Xr9Mp9MhPj4e/v7+d103Li4OGo0GEydONFheUVGBiooKmJgYvi2pVAqdTqf/vbCwEEFBQZDL5di+fTuUSmW99SYnJ6Njx44tLiTdzfX8UsQduQIAmDeSXSgiIqLGJOrhvMjISISHh2PAgAHw9fXFypUroVarMWXKFABAWFgYOnXqhOjoaIP1YmJiEBoaCltbW4PllpaWGD58OBYuXAgzMzO4u7tj7969+OKLL7B8+XIAfwWokpISfPXVVwYTwO3t7SGVSvHjjz8iOzsbgwYNglKpxK5du/D222/j5Zdfboa90njW701HhVbAoK428O1iI3Y5REREbYqoIWr8+PHIzc3FkiVLkJWVhX79+mHHjh36yeYZGRk1ukqpqanYv38/du7cWes2Y2NjERUVheeffx43b96Eu7s73nrrLcycORMAcOzYMRw+fBgA4OlpeKbaxYsX4eHhAZlMhjVr1mDBggUQBAGenp5Yvnw5pk+f3ti7oMnkFJbhmz9ud6E4F4qIiKjRiXqdqLZOzOtELfvpNGL2X8QA946Im+kPiUTSrK9PRETUWrX460RR08kt0mDT4csAgLkjuzNAERERNQGGqDbo098voKxCh75u1hjW3U7scoiIiNokhqg25qa6HF8equpCzR/pyS4UERFRE2GIamNi9l9ASbkWvTtZ4pEeNW/iTERERI2DIaoNyS8px+cHb8+FGsG5UERERE2JIaoN2XjgEoo1lejp1AGP9nKsfwUiIiIyGkNUG1FYVoENBy4CqOpCmZiwC0VERNSUGKLaiM8PXEJRWSW6O1hgdO+733uQiIiI7h9DVBtQrKlEzO0u1JwRnuxCERERNQOGqDbgy8TLyC+pQFc7czzex0XscoiIiNoFhqhWrqS8Ep/8fgEAEPGIJ6TsQhERETULhqhW7uvDGbipLkdnGxWe6scuFBERUXNhiGrFyiq0WLe3ugvVDaZS/t9JRETUXPit24p9k5SBvGINOlmb4en+rmKXQ0RE1K4wRLVSVV2odADA7Ee6QW7K/yuJiIiaE795W6m4o1eRXaiBs5USz/qwC0VERNTcGKJaofJKHdbtqepCzRzeDQpTqcgVERERtT8MUa3Q1mNXcS2/FPYdFBg/0E3scoiIiNolhqhWpkKrw5o9aQCAfw7rCqWMXSgiIiIxmIpdAN2bnCINrMxkKLXQ4nk/d7HLISIiarcYolqZTtZm+HHOUFy9VQozObtQREREYuHhvFZIIpHAzUYldhlERETtGkMUERERkREYooiIiIiMwBBFREREZASGKCIiIiIjMEQRERERGYEhioiIiMgIDFFERERERmCIIiIiIjICQxQRERGRERiiiIiIiIzAEEVERERkBIYoIiIiIiMwRBEREREZwVTsAtoyQRAAAIWFhSJXQkRERA1V/b1d/T1eF4aoJlRUVAQAcHNzE7kSIiIiuldFRUWwsrKq83mJUF/MIqPpdDpcv34dHTp0gEQiabTtFhYWws3NDVeuXIGlpWWjbbct4r5qOO6re8P91XDcVw3HfdVwTbmvBEFAUVERXFxcYGJS98wndqKakImJCVxdXZts+5aWlvyPrIG4rxqO++recH81HPdVw3FfNVxT7au7daCqcWI5ERERkREYooiIiIiMwBDVCikUCrzxxhtQKBRil9LicV81HPfVveH+ajjuq4bjvmq4lrCvOLGciIiIyAjsRBEREREZgSGKiIiIyAgMUURERERGYIgiIiIiMgJDVAu1Zs0aeHh4QKlUws/PD0lJSXcdHxcXh549e0KpVMLb2xu//PJLM1UqvnvZV5999hkkEonBQ6lUNmO14tm3bx+eeOIJuLi4QCKR4Pvvv693nT179uChhx6CQqGAp6cnPvvssyavsyW41321Z8+eGp8riUSCrKys5ilYRNHR0Rg4cCA6dOgABwcHhIaGIjU1td712uPfLGP2VXv9m7V27Vr06dNHfyFNf39//O9//7vrOmJ8phiiWqDNmzcjMjISb7zxBo4dO4a+ffsiODgYOTk5tY4/ePAgnnvuOUydOhXHjx9HaGgoQkNDkZKS0syVN7973VdA1dVtMzMz9Y/Lly83Y8XiUavV6Nu3L9asWdOg8RcvXkRISAgeeeQRJCcn48UXX8S0adPw66+/NnGl4rvXfVUtNTXV4LPl4ODQRBW2HHv37kVERAQOHTqEXbt2oaKiAkFBQVCr1XWu017/Zhmzr4D2+TfL1dUV//3vf3H06FEcOXIEI0aMwFNPPYVTp07VOl60z5RALY6vr68QERGh/12r1QouLi5CdHR0rePHjRsnhISEGCzz8/MT/vnPfzZpnS3Bve6rjRs3ClZWVs1UXcsFQNi2bdtdx/zrX/8SHnzwQYNl48ePF4KDg5uwspanIfsqISFBACDcunWrWWpqyXJycgQAwt69e+sc057/Zt2pIfuKf7P+0rFjR+HTTz+t9TmxPlPsRLUw5eXlOHr0KAIDA/XLTExMEBgYiMTExFrXSUxMNBgPAMHBwXWObyuM2VcAUFxcDHd3d7i5ud31XzbtXXv9XN2Pfv36wdnZGY8++igOHDggdjmiKCgoAADY2NjUOYafrSoN2VcA/2ZptVrExsZCrVbD39+/1jFifaYYolqYvLw8aLVaODo6Gix3dHSsc35FVlbWPY1vK4zZVz169MCGDRvwww8/4KuvvoJOp8PgwYNx9erV5ii5Vanrc1VYWIjS0lKRqmqZnJ2dsW7dOnz33Xf47rvv4ObmhoCAABw7dkzs0pqVTqfDiy++iCFDhqB37951jmuvf7Pu1NB91Z7/Zp08eRIWFhZQKBSYOXMmtm3bBi8vr1rHivWZMm3SrRO1MP7+/gb/khk8eDB69eqF9evXY9myZSJWRq1Zjx490KNHD/3vgwcPRnp6OlasWIEvv/xSxMqaV0REBFJSUrB//36xS2nxGrqv2vPfrB49eiA5ORkFBQXYsmULwsPDsXfv3jqDlBjYiWph7OzsIJVKkZ2dbbA8OzsbTk5Ota7j5OR0T+PbCmP21d/JZDL0798faWlpTVFiq1bX58rS0hJmZmYiVdV6+Pr6tqvP1Zw5c/DTTz8hISEBrq6udx3bXv9mVbuXffV37elvllwuh6enJ3x8fBAdHY2+ffviww8/rHWsWJ8phqgWRi6Xw8fHB/Hx8fplOp0O8fHxdR4L9vf3NxgPALt27apzfFthzL76O61Wi5MnT8LZ2bmpymy12uvnqrEkJye3i8+VIAiYM2cOtm3bht27d6NLly71rtNeP1vG7Ku/a89/s3Q6HTQaTa3PifaZatJp62SU2NhYQaFQCJ999plw+vRpYcaMGYK1tbWQlZUlCIIgTJo0SVi0aJF+/IEDBwRTU1Ph/fffF86cOSO88cYbgkwmE06ePCnWW2g297qv3nzzTeHXX38V0tPThaNHjwoTJkwQlEqlcOrUKbHeQrMpKioSjh8/Lhw/flwAICxfvlw4fvy4cPnyZUEQBGHRokXCpEmT9OMvXLggqFQqYeHChcKZM2eENWvWCFKpVNixY4dYb6HZ3Ou+WrFihfD9998L58+fF06ePCnMnz9fMDExEX777Tex3kKzmTVrlmBlZSXs2bNHyMzM1D9KSkr0Y/g3q4ox+6q9/s1atGiRsHfvXuHixYvCiRMnhEWLFgkSiUTYuXOnIAgt5zPFENVCrVq1SujcubMgl8sFX19f4dChQ/rnhg8fLoSHhxuM//bbb4UHHnhAkMvlwoMPPij8/PPPzVyxeO5lX7344ov6sY6OjsJjjz0mHDt2TISqm1/1afh/f1Tvn/DwcGH48OE11unXr58gl8uFrl27Chs3bmz2usVwr/vqnXfeEbp16yYolUrBxsZGCAgIEHbv3i1O8c2stv0EwOCzwr9ZVYzZV+31b9YLL7wguLu7C3K5XLC3txdGjhypD1CC0HI+UxJBEISm7XURERERtT2cE0VERERkBIYoIiIiIiMwRBEREREZgSGKiIiIyAgMUURERERGYIgiIiIiMgJDFBEREZERGKKIiJrJnj17IJFIkJ+fL3YpRNQIGKKIiIiIjMAQRURERGQEhigiajd0Oh2io6PRpUsXmJmZoW/fvtiyZQuAvw61/fzzz+jTpw+USiUGDRqElJQUg2189913ePDBB6FQKODh4YEPPvjA4HmNRoNXXnkFbm5uUCgU8PT0RExMjMGYo0ePYsCAAVCpVBg8eDBSU1Ob9o0TUZNgiCKidiM6OhpffPEF1q1bh1OnTmHBggWYOHEi9u7dqx+zcOFCfPDBB/jjjz9gb2+PJ554AhUVFQCqws+4ceMwYcIEnDx5Ev/+97+xePFifPbZZ/r1w8LC8M033+Cjjz7CmTNnsH79elhYWBjU8dprr+GDDz7AkSNHYGpqihdeeKFZ3j8RNS7egJiI2gWNRgMbGxv89ttv8Pf31y+fNm0aSkpKMGPGDDzyyCOIjY3F+PHjAQA3b96Eq6srPvvsM4wbNw7PP/88cnNzsXPnTv36//rXv/Dzzz/j1KlTOHfuHHr06IFdu3YhMDCwRg179uzBI488gt9++w0jR44EAPzyyy8ICQlBaWkplEplE+8FImpM7EQRUbuQlpaGkpISPProo7CwsNA/vvjiC6Snp+vH3RmwbGxs0KNHD5w5cwYAcObMGQwZMsRgu0OGDMH58+eh1WqRnJwMqVSK4cOH37WWPn366H92dnYGAOTk5Nz3eySi5mUqdgFERM2huLgYAPDzzz+jU6dOBs8pFAqDIGUsMzOzBo2TyWT6nyUSCYCq+VpE1LqwE0VE7YKXlxcUCgUyMjLg6elp8HBzc9OPO3TokP7nW7du4dy5c+jVqxcAoFevXjhw4IDBdg8cOIAHHngAUqkU3t7e0Ol0BnOsiKjtYieKiNqFDh064OWXX8aCBQug0+kwdOhQFBQU4MCBA7C0tIS7uzsAYOnSpbC1tYWjoyNee+012NnZITQ0FADw0ksvYeDAgVi2bBnGjx+PxMRErF69Gh9//DEAwMPDA+Hh4XjhhRfw0UcfoW/fvrh8+TJycnIwbtw4sd46ETURhigiajeWLVsGe3t7REdH48KFC7C2tsZDDz2EV199VX847b///S/mz5+P8+fPo1+/fvjxxx8hl8sBAA899BC+/fZbLFmyBMuWLYOzszOWLl2KyZMn619j7dq1ePXVVzF79mzcuHEDnTt3xquvvirG2yWiJsaz84iI8NeZc7du3YK1tbXY5RBRK8A5UURERERGYIgiIiIiMgIP5xEREREZgZ0oIiIiIiMwRBEREREZgSGKiIiIyAgMUURERERGYIgiIiIiMgJDFBEREZERGKKIiIiIjMAQRURERGQEhigiIiIiI/w/4Qepp18rfFgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "plt.plot(acc, label='BERT-model')\n",
    "plt.title('model accuracy')\n",
    "plt.ylabel('accuracy')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['test'], loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test the model on any input data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "-G4xt3XGDFyy"
   },
   "outputs": [],
   "source": [
    "def test(text):\n",
    "    text_parts = preprocess_text(text)\n",
    "    overall_output = torch.zeros((1,2)).to(device)\n",
    "    try:\n",
    "        for part in text_parts:\n",
    "            if len(part) > 0:\n",
    "                overall_output += model(part.reshape(1, -1))[0]\n",
    "    except RuntimeError:\n",
    "        print(\"GPU out of memory, skipping this entry.\")\n",
    "\n",
    "    overall_output = F.softmax(overall_output[0], dim=-1)\n",
    "\n",
    "    value, result = overall_output.max(0)\n",
    "\n",
    "    term = \"fake\"\n",
    "    if result.item() == 0:\n",
    "        term = \"real\"\n",
    "\n",
    "    print(\"{} at {}%\".format(term, value.item() * 100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "uyXL2mq0DFyz",
    "outputId": "72ac248a-9caa-487a-e9ee-b2375d5a9d11"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fake at 58.98287296295166%\n",
      "real at 76.57802700996399%\n"
     ]
    }
   ],
   "source": [
    "fake1 = \"\"\"\n",
    "Americans to fund killing babies in abortion that she has been caught trying to add taxpayer financing of abortions to the bill to combat the Coronavirus and provide economic stimulus to the nation as it deals with the COVD-19 outbreak.\n",
    "Nancy Pelosi has a long history of promoting abortion and her first act after becoming Speaker in 2019 was pushing legislation to use tax money for abortions. So it’s no surprise she is trying to exploit the Coronavirus pandemic to push abortion funding again.\n",
    "As The Daily Caller reports: House Speaker Nancy Pelosi sought to include a potential way to guarantee federal funding for abortion into the coronavirus economic stimulus plan, according to multiple senior White House officials.\n",
    "Speaking to the Daily Caller, those officials alleged that while negotiating the stimulus with U.S. Treasury Secretary Steve Mnuchin, Pelosi tried to lobby for “several” provisions that stalled bipartisan commitment to the effort. One was a mandate for up to $1 billion to reimburse laboratory claims, which White House officials say would set a precedent of health spending without protections outlined in the Hyde Amendment.\n",
    "LifeNews depends on the support of readers like you to combat the pro-abortion media. Please donate now.\n",
    "“A New mandatory funding stream that does not have Hyde protections would be unprecedented,” one White House official explained. “Under the guise of protecting people, Speaker Pelosi is working to make sure taxpayer dollars are spent covering abortion—which is not only backwards, but goes against historical norms.”\n",
    "A second White House official referred to the provision as a “slush fund” and yet another questioned “what the Hyde Amendment and abortion have to do with protecting Americans from coronavirus?”\n",
    "Americans should insist to their members of Congress that we need a clean bill that provides aggressive action to help patients and spur the economy. Killing babies with our tax dollars is not the answer to the coronavirus and the situation should not be exploited for political gain.\n",
    "\"\"\"\n",
    "\n",
    "true1 = \"\"\"\n",
    "Price spikes, however, would cause demand to wither and some expensive avocados might be leftover, and stores might try to ration avocados, he added.\n",
    "\"Exactly what the retail strategy would be in this case, I’m not sure. But we would have vastly fewer avocados,\" Sumner said.\n",
    "Just how fast avocados would disappear, if at all, would depend on whether the Trump administration enacts a full or partial border closure. White House economic adviser Larry Kudlow told CNBC he’s looking for ways to keep some commerce flowing.\n",
    "\"We are looking at different options, particularly if you can keep those freight lanes, the truck lanes, open,\" he said this week.  \n",
    "Ben Holtz owns Rocky H Ranch, a 70-acre family-run avocado farm in northern San Diego County. He agreed avocados would run out within weeks.\n",
    "\"Mexico is the big player today. California is not. You shut down the border and California can’t produce to meet the demand,\" Holtz said. \"There will be people without their guacamole.\"\n",
    "While Mexico’s avocado harvest is year-round, California’s is limited to April through July. Growers in the state have picked only about 3 percent of what’s expected to be a much smaller crop of about 175 million pounds this year, Holtz said. A heat wave last summer reduced the crop size.\n",
    "California’s avocado harvest has averaged approximately 300 million pounds in recent years, according to data from the California Avocado Commission. By contrast, the U.S. has imported more than 1.5 billion pounds of avocados from Mexico annually. Representatives from the commission did not respond to requests for this article.\n",
    "Altogether, the U.S. received 43 percent of its fruit and vegetable imports from Mexico in 2016, according to the U.S. Department of Agriculture.\n",
    "Also affecting this year’s avocado supply, a California avocado company in March recalled shipments to six states last month after fears the fruit might be contaminated with a bacterium that can cause health risks.\n",
    "Until the early 2000s, California was the nation’s leading supplier of avocados, Holtz said. Mexico gradually overtook the state and now dominates sales in the U.S.\n",
    "\"It’s a very big possibility,\" Holtz said of avocado shortages. \"Three weeks would dry up the Mexican inventory. California alone consumes more avocados than are grown in our state. Cold storage supply chain is basically three weeks or less of inventory. Most of the time it’s seven days.\"\n",
    "A spokeswoman for the California Restaurant Association said \"we haven’t heard concerns from restaurants, it doesn’t mean they aren’t worried.\" A national grocers association said it will \"continue to closely monitor any developments\" at the border, but did not have information about the potential impact on avocados.\n",
    "\"\"\"\n",
    "\n",
    "test(fake1)\n",
    "test(true1)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "AcJQpaG7DFy1"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "machine_shape": "hm",
   "provenance": []
  },
  "gpuClass": "standard",
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "0ef749989aea417d85312c9cd35fae92": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "13a152edcd754d548380a76eef3b2e93": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "1afe0d53ff7a491289188d006854bb51": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "1c4802f3bc1d49668f64e12760fdc507": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "1f81e5915f874b5587dabed20015cb44": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_4c936325266b48f18169c318997b28cc",
       "IPY_MODEL_9b527fd28ec04babb148c619acc41812",
       "IPY_MODEL_b5df15a216b24e8b8ae7bf1c273d6d80"
      ],
      "layout": "IPY_MODEL_13a152edcd754d548380a76eef3b2e93"
     }
    },
    "1f8817ce2adc41f3b4551593bca5a7cb": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_6ee4b55cbd3543898d1784a644a7709c",
      "placeholder": "​",
      "style": "IPY_MODEL_a2e89bfe27b24215b52e6c07f228708d",
      "value": " 232k/232k [00:00&lt;00:00, 1.43MB/s]"
     }
    },
    "2030babffefa490e978b2821379db31e": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_3432d85ffbe042b1b704a80dd32a4344",
      "max": 231508,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_22a8f852ea1d4916800b7a2e871a1d5d",
      "value": 231508
     }
    },
    "22a8f852ea1d4916800b7a2e871a1d5d": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "23ff8d463e0141169dac6ff90f1c5b51": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "2afee066d10144079c8c5f62abd687aa": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "2ca0b99eab7948bd9d47e902ce8e24b1": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "2fea4c3af4e14fb890e8c134de63e155": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_0ef749989aea417d85312c9cd35fae92",
      "max": 570,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_f1fc1da606844656bb4f4bc942494efc",
      "value": 570
     }
    },
    "3432d85ffbe042b1b704a80dd32a4344": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "3dd0e03a3e3e48d99c1f8f5014cfa489": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_bab4328b642a4dd99c30320267b25bef",
       "IPY_MODEL_2030babffefa490e978b2821379db31e",
       "IPY_MODEL_1f8817ce2adc41f3b4551593bca5a7cb"
      ],
      "layout": "IPY_MODEL_1afe0d53ff7a491289188d006854bb51"
     }
    },
    "4c936325266b48f18169c318997b28cc": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_c11bc5cc20c24d03b6bbd7da89443c2f",
      "placeholder": "​",
      "style": "IPY_MODEL_e4ef2cda58c14f8bb5c2c32fe76a45ef",
      "value": "Downloading pytorch_model.bin: 100%"
     }
    },
    "510b7278236b47648152af204c6bf814": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_67e7897e39bc4bb199624d64946aa5aa",
       "IPY_MODEL_c0df9cf96bdb4fc59e819e5e4d091b3c",
       "IPY_MODEL_baf9fcf55c5e41b990703851f7885c08"
      ],
      "layout": "IPY_MODEL_c46a91dd682a4d068307130e386e5fda"
     }
    },
    "5b11bcff43ed410c85eaeda499954dcf": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "5eb0cb0e927b4275988f0eb6b27f55fc": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "67e7897e39bc4bb199624d64946aa5aa": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_b67cc694d4174d44a061982a8f6ec257",
      "placeholder": "​",
      "style": "IPY_MODEL_83263151fcea4d05813186cef90a9195",
      "value": "Downloading (…)okenizer_config.json: 100%"
     }
    },
    "6ee4b55cbd3543898d1784a644a7709c": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "7dadf6731b2c4fba922be12ec00117d8": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "7f7f10cf3b4f41218fedc1738eb19d4d": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "83263151fcea4d05813186cef90a9195": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "8d10a7496eae433998d009b206963194": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "8df05f505d584e4a9c226f7f7dc61ccd": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "9b527fd28ec04babb148c619acc41812": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_8d10a7496eae433998d009b206963194",
      "max": 440473133,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_5b11bcff43ed410c85eaeda499954dcf",
      "value": 440473133
     }
    },
    "a2e89bfe27b24215b52e6c07f228708d": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "a6f322562ddc481fa10495bc791d3877": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "a891f9ad215e4ad9b52ef55029d91642": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "b5df15a216b24e8b8ae7bf1c273d6d80": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_d80f8795d39043259e7b448f640b5532",
      "placeholder": "​",
      "style": "IPY_MODEL_5eb0cb0e927b4275988f0eb6b27f55fc",
      "value": " 440M/440M [00:01&lt;00:00, 375MB/s]"
     }
    },
    "b67cc694d4174d44a061982a8f6ec257": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "bab4328b642a4dd99c30320267b25bef": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_a891f9ad215e4ad9b52ef55029d91642",
      "placeholder": "​",
      "style": "IPY_MODEL_a6f322562ddc481fa10495bc791d3877",
      "value": "Downloading (…)solve/main/vocab.txt: 100%"
     }
    },
    "baf9fcf55c5e41b990703851f7885c08": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_7dadf6731b2c4fba922be12ec00117d8",
      "placeholder": "​",
      "style": "IPY_MODEL_23ff8d463e0141169dac6ff90f1c5b51",
      "value": " 28.0/28.0 [00:00&lt;00:00, 1.53kB/s]"
     }
    },
    "bbbc3b7bb5194588a6625bdb64d127c5": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_2afee066d10144079c8c5f62abd687aa",
      "placeholder": "​",
      "style": "IPY_MODEL_8df05f505d584e4a9c226f7f7dc61ccd",
      "value": " 570/570 [00:00&lt;00:00, 25.5kB/s]"
     }
    },
    "c0df9cf96bdb4fc59e819e5e4d091b3c": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_7f7f10cf3b4f41218fedc1738eb19d4d",
      "max": 28,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_ef75d6657b35415781c0e749192ab48b",
      "value": 28
     }
    },
    "c11bc5cc20c24d03b6bbd7da89443c2f": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "c3c353aa1d124509afebe084c07d7092": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_c733eac88cc34337b6b4b694eac5026d",
       "IPY_MODEL_2fea4c3af4e14fb890e8c134de63e155",
       "IPY_MODEL_bbbc3b7bb5194588a6625bdb64d127c5"
      ],
      "layout": "IPY_MODEL_2ca0b99eab7948bd9d47e902ce8e24b1"
     }
    },
    "c46a91dd682a4d068307130e386e5fda": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "c733eac88cc34337b6b4b694eac5026d": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_d0dc819b2b9247d185d07eb47d406201",
      "placeholder": "​",
      "style": "IPY_MODEL_1c4802f3bc1d49668f64e12760fdc507",
      "value": "Downloading (…)lve/main/config.json: 100%"
     }
    },
    "d0dc819b2b9247d185d07eb47d406201": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "d80f8795d39043259e7b448f640b5532": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "e4ef2cda58c14f8bb5c2c32fe76a45ef": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "ef75d6657b35415781c0e749192ab48b": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "f1fc1da606844656bb4f4bc942494efc": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}