Fine-tuning BERT.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/mareloraby/Sentiment-Arcs/blob/main/Fine-tuning%20BERT.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hskss95zD2MA",
        "outputId": "30642f4f-359b-4e52-e188-8463846b3730"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive') #Mount google drive to load training and testing data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "LTF2TsBED_my"
      },
      "outputs": [],
      "source": [
        "!pip install transformers\n",
        "!pip install tensorflow\n",
        "\n",
        "from IPython.display import clear_output\n",
        "clear_output() # clear output window"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "20NIuVOuE_N3"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "import re\n",
        "from tqdm import tqdm\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "%matplotlib inline"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Loading Data"
      ],
      "metadata": {
        "id": "stLi7UNBE8ul"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Loading dataset for fine-tuning"
      ],
      "metadata": {
        "id": "6P5KAi78CL8S"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "5HbPVi4bED5Q",
        "outputId": "dd99c815-7ec8-4e83-adb6-d30af87ec9d8"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                    text  label\n",
              "0      \"QT @user In the original draft of the 7th boo...      2\n",
              "1      \"Ben Smith / Smith (concussion) remains out of...      1\n",
              "2      Sorry bout the stream last night I crashed out...      1\n",
              "3      Chase Headley's RBI double in the 8th inning o...      1\n",
              "4      @user Alciato: Bee will invest 150 million in ...      2\n",
              "...                                                  ...    ...\n",
              "59894  \"LONDON (AP) \"\" Prince George celebrates his s...      1\n",
              "59895  Harper's Worst Offense against Refugees may be...      1\n",
              "59896  Hold on... Sam Smith may do the theme to Spect...      2\n",
              "59897  Gonna watch Final Destination 5 tonight. I alw...      1\n",
              "59898  \"Interview with Devon Alexander \\\"\"\"\"Speed Kil...      1\n",
              "\n",
              "[59899 rows x 2 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-8427d42d-574b-4c20-b056-3423b758f31e\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>\"QT @user In the original draft of the 7th boo...</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>\"Ben Smith / Smith (concussion) remains out of...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Sorry bout the stream last night I crashed out...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Chase Headley's RBI double in the 8th inning o...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>@user Alciato: Bee will invest 150 million in ...</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>59894</th>\n",
              "      <td>\"LONDON (AP) \"\" Prince George celebrates his s...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>59895</th>\n",
              "      <td>Harper's Worst Offense against Refugees may be...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>59896</th>\n",
              "      <td>Hold on... Sam Smith may do the theme to Spect...</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>59897</th>\n",
              "      <td>Gonna watch Final Destination 5 tonight. I alw...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>59898</th>\n",
              "      <td>\"Interview with Devon Alexander \\\"\"\"\"Speed Kil...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>59899 rows × 2 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-8427d42d-574b-4c20-b056-3423b758f31e')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-8427d42d-574b-4c20-b056-3423b758f31e button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-8427d42d-574b-4c20-b056-3423b758f31e');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ],
      "source": [
        "import pandas as pd\n",
        "\n",
        "df = pd.read_csv('/content/drive/MyDrive/NLP/TweetEval/trainTest.csv')\n",
        "dv = pd.read_csv('/content/drive/MyDrive/NLP/TweetEval/validation.csv')\n",
        "\n",
        "appended = df.append(dv, ignore_index = True)\n",
        "df = appended\n",
        "df"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### **Pre - processing and Cleaning**"
      ],
      "metadata": {
        "id": "TYWsStjRCW24"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "iSJxXk6BCdC7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "outputId": "7d8e9f51-8610-42c8-947e-988f493489f2"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                    text  label\n",
              "0      In the original draft of the 7th book, Remus L...      2\n",
              "1      \"Ben Smith / Smith (concussion) remains out of...      1\n",
              "2      Sorry bout the stream last night I crashed out...      1\n",
              "3      Chase Headley's RBI double in the 8th inning o...      1\n",
              "4      Alciato: Bee will invest 150 million in Januar...      2\n",
              "...                                                  ...    ...\n",
              "59894  \"LONDON (AP) \"\" Prince George celebrates his s...      1\n",
              "59895  Harper's Worst Offense against Refugees may be...      1\n",
              "59896  Hold on... Sam Smith may do the theme to Spect...      2\n",
              "59897  Gonna watch Final Destination 5 tonight. I alw...      1\n",
              "59898  \"Interview with Devon Alexander \\\"\"\"\"Speed Kil...      1\n",
              "\n",
              "[59813 rows x 2 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-e57fe4e7-8d7e-4f40-a725-cf7a043c043e\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>In the original draft of the 7th book, Remus L...</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>\"Ben Smith / Smith (concussion) remains out of...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Sorry bout the stream last night I crashed out...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Chase Headley's RBI double in the 8th inning o...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Alciato: Bee will invest 150 million in Januar...</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>59894</th>\n",
              "      <td>\"LONDON (AP) \"\" Prince George celebrates his s...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>59895</th>\n",
              "      <td>Harper's Worst Offense against Refugees may be...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>59896</th>\n",
              "      <td>Hold on... Sam Smith may do the theme to Spect...</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>59897</th>\n",
              "      <td>Gonna watch Final Destination 5 tonight. I alw...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>59898</th>\n",
              "      <td>\"Interview with Devon Alexander \\\"\"\"\"Speed Kil...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>59813 rows × 2 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e57fe4e7-8d7e-4f40-a725-cf7a043c043e')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-e57fe4e7-8d7e-4f40-a725-cf7a043c043e button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-e57fe4e7-8d7e-4f40-a725-cf7a043c043e');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ],
      "source": [
        "import re\n",
        "\n",
        "#Removing duplicates including retweets and quotations\n",
        "\n",
        "df['text'] = df['text'].apply(lambda x: re.sub(r'RT (@.*?)[\\s]', '', x) )\n",
        "df['text'] = df['text'].apply(lambda x: re.sub(r'\\\"RT ', '', x))\n",
        "df['text'] = df['text'].apply(lambda x: re.sub(r'RT ', '', x) if (x.find(\"RT \") == 0) else x)\n",
        "df['text'] = df['text'].apply(lambda x: re.sub(r'QT |\\\"QT ', '', x) if (x.find(\"QT \") == 0 or x.find(\"QT \") == 1 ) else x)\n",
        "df['text'] = df['text'].apply(lambda x: re.sub(r'(@.*?)[\\s]', '', x) if (x.find(\"@\") == 0) else x )\n",
        "\n",
        "df.drop_duplicates(keep=False,inplace=True)\n",
        "df"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import seaborn as sns\n",
        "from matplotlib import rcParams\n",
        "rcParams['figure.figsize'] = 12, 8\n",
        "sns.set(style='whitegrid', palette='rocket', font_scale=1.2)\n",
        "\n",
        "sns.countplot(df.label)\n",
        "plt.xlabel('labels')\n"
      ],
      "metadata": {
        "id": "PXcvHBYZmIOb",
        "outputId": "e6f685c6-ae81-496f-80b6-99442c166ff6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 581
        }
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/seaborn/_decorators.py:43: FutureWarning: Pass the following variable as a keyword arg: x. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.\n",
            "  FutureWarning\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0.5, 0, 'labels')"
            ]
          },
          "metadata": {},
          "execution_count": 6
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 864x576 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAvEAAAHpCAYAAAALVWOKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de5CW9X3//xdnCLvYxKakgOIhyqorCHhk0ATTaC1qTSIWcDBmII2atpJEBiFftaRBk65Rg07RBDQl1DpBjQZwrG1DTFTUoDZmOUhqiAcMMYmn3QALsvv7wx93XfGwhIXlA4/HDMPs9bnu635fjPf4vC+u+6ZTS0tLSwAAgGJ07ugBAACA7SPiAQCgMCIeAAAKI+IBAKAwXTt6gNI0NzfnD3/4Q7p165ZOnTp19DgAAOyBWlpasnnz5vTu3TudO2973V3Eb6c//OEPWb16dUePAQDAXuDQQw9NdXX1NttF/Hbq1q1bkjf+QLt3797B0wAAsCfatGlTVq9eXWnPtxLx22nrLTTdu3dPjx49OngaAAD2ZO90+7YPtgIAQGFEPAAAFEbEAwBAYUQ8AAAURsQDAEBhRDwAABRGxAMAQGFEPAAAFEbEAwBAYUQ8AAAURsQDAEBhRDwAABRGxAMAQGFEPAAAFEbEAwBAYUQ8AAAURsQDAEBhRDwAABRGxAN7vc1Nmzp6BGgX/luGvUfXjh4AoKN169E9nzv87I4eA3bYTStu7+gRgF3ElXgAACiMiAcAgMKIeAAAKIyIBwCAwoh4AAAojIgHAIDCiHgAACiMiAcAgMKIeAAAKIyIBwCAwoh4AAAojIgHAIDCiHgAACiMiAcAgMKIeAAAKIyIBwCAwoh4AAAojIgHAIDCiHgAACiMiAcAgMKIeAAAKIyIBwCAwoh4AAAojIgHAIDCiHgAACiMiAcAgMKIeAAAKIyIBwCAwoh4AAAojIgHAIDCiHgAACiMiAcAgMLssoivq6vL6NGjM2zYsIwcOTLTp0/Pyy+/3GqfQYMGZfDgwRk6dGjl11NPPVVZb25uzjXXXJMRI0Zk6NChmThxYtauXdvqGA899FDOPPPMDBkyJKeeemruueeeVusvv/xyJk+enGHDhuXYY4/N5Zdfnk2bNu28EwcAgHa2yyK+S5cuqauryyOPPJK7774769aty7Rp07bZ79vf/naeeOKJyq9BgwZV1ubMmZNFixZl/vz5eeCBB9KvX79ccMEFaW5uTpI8//zzufDCCzNhwoT89Kc/zaWXXppp06blZz/7WeUYl1xySdavX58lS5Zk4cKFqa+vz9e+9rWd/wcAAADtZJdF/Be/+MUcfvjh6datW/bdd99MmDAhjz766HYd47bbbsukSZNy0EEHpXfv3pkyZUrWrFmTxx57LEny/e9/P4ceemjGjBmT7t27Z9SoURk1alRuu+22JG9E/gMPPJCpU6dmn332Sd++fXPxxRfnzjvvTFNTU7ufMwAA7AxdO+qJly5dmpqamm22f+lLX8rmzZvTr1+/jBs3Luecc06SpKGhIWvXrk1tbW1l3z59+mTgwIFZuXJljjnmmKxatarVepLU1tZm8eLFSZJVq1alV69eOfjggyvrRx55ZDZs2JA1a9a87TzvpL6+frvOF9h9DR8+vKNHgHaz9cIWsGfrkIi/5557smDBgsyfP7/V9u985zsZOnRoOnfunIcffjiXXHJJXn/99YwfPz6NjY1J3gj3N6uurq6sNTY25sMf/nCr9T59+rRar66u3ubxW9e2R21tbXr06LFdjwGAnc2bUtgzNDU1vetF413+7TSLFy/OFVdckdmzZ+eII45otXbCCSekZ8+e6d69e0466aScf/75+cEPfpAkqaqqSvLGFfk3a2hoqKxVVVVts/7aa6+1Wn9rrG/df+s+AACwu9ulEb9gwYLMmDEjN954Y44//vj33L9z585paWlJ8sYV8/79+7d6R9LQ0JBnn302hx12WJKkpqZmm3csy5cvr9wmU1NTk/Xr1+fpp5+urNfX16dnz5458MADd/j8AABgV9hlET9v3rxcffXVmTt37tv+Vd/y5cvz85//PJs2bcrrr7+eBx98MLfccktGjx5d2Wfs2LGZO3du1qxZk/Xr16euri4HHHBA5XhnnXVWnnrqqdxxxx3ZvHlz7r///ixZsiRjx45NkgwYMCAjR45MXV1dXn311bz44ouZNWtWPvnJT7o1BgCAYuyye+JnzpyZrl275rzzzmu1ffHixenXr19+85vfpK6uLuvWrUuXLl3Sr1+/TJ48OePGjavsO2nSpDQ0NGT8+PHZsGFDhg8fntmzZ6dz5zfei+y3336ZPXt2rrrqqsyYMSMf+tCHcuWVV2bIkCGVY9TV1WXGjBkZNWpUunTpktNOOy2XXnrprvlDAACAdtCpZev9KrTJ1g8Z+GAr7Fk+d/jZHT0C7LCbVtze0SMA7eS9mnOXf7AVAADYMSIeAAAKI+IBAKAwIh4AAAoj4gEAoDAiHgAACiPiAQCgMCIeAAAKI+IBAKAwIh4AAAoj4gEAoDAiHgAACiPiAQCgMCIeAAAKI+IBAKAwIh4AAAoj4gEAoDAiHgAACiPiAQCgMCIeAAAKI+IBAKAwIh4AAAoj4gEAoDAiHgAACiPiAQCgMCIeAAAKI+IBAKAwIh4AAAoj4gEAoDAiHgAACiPiAQCgMCIeAAAKI+IBAKAwIh4AAAoj4gEAoDAiHgAACiPiAQCgMCIeAAAKI+IBAKAwIh4AAAoj4gEAoDAiHgAACiPiAQCgMCIeAAAKI+IBAKAwIh4AAAoj4gEAoDAiHgAACiPiAQCgMCIeAAAKI+IBAKAwIh4AAAoj4gEAoDAiHgDoMK83beroEWCHdcR/x113+TMCAPz/uvbonm8fMaGjx4Ad8tnl393lz+lKPAAAFEbEAwBAYUQ8AAAURsQDAEBhRDwAABRGxAMAQGFEPAAAFEbEAwBAYUQ8AAAURsQDAEBhdlnE19XVZfTo0Rk2bFhGjhyZ6dOn5+WXX261z4oVKzJ27NgMGTIkH/3oRzNv3rxW6xs3bszll1+eY489NsOGDcvkyZPzyiuvtNpn0aJFOeWUUzJ48OCceeaZWbp0aav15557LhMnTszQoUMzYsSIXHvttWlpadk5Jw0AADvBLov4Ll26pK6uLo888kjuvvvurFu3LtOmTausNzY2ZtKkSRk5cmQeffTRXHfddbnhhhty7733Vva58sorU19fn4ULF2bJkiVZv359pk6dWll//PHHM3369EybNi3Lli3LhAkTcuGFF+aFF15IkmzZsiUXXHBB+vXrlwceeCDz58/PwoULc/PNN++qPwYAANhhuyziv/jFL+bwww9Pt27dsu+++2bChAl59NFHK+v33XdfOnfunIsuuig9evTIUUcdlTFjxuTWW29N8sZV+LvuuisXX3xx+vbtm3322SdTp07Nj370o0qkf+9738vJJ5+cUaNGpXv37hkzZkwOOeSQ3HnnnUmSZcuW5ZlnnsmUKVPSu3fvHHTQQZk0aVLlOQAAoARdO+qJly5dmpqamsrPq1atyuGHH57Onf/vfUVtbW0WLFiQJPnVr36VpqamHHnkkZX1gw8+OL169crKlSvTr1+/rFq1Kqeffnqr56mtrc2qVasqzzFw4MD06dOn1frzzz+fxsbGVFVVtXn++vr67TthYLc1fPjwjh4B2s1jjz3W0SNsF68/9hS7+rXXIRF/zz33ZMGCBZk/f35lW2NjY6qrq1vt16dPnzQ2NlbWk2yzT3V1dat93hzoW4+xZs2ad32OrWvbE/G1tbXp0aNHm/cHgF1BFEPHaO/XXlNT07teNN7l306zePHiXHHFFZk9e3aOOOKIyvaqqqpKjG/12muvVcJ66+8NDQ2t9mloaGi1z1vX33qMt3uONx8fAAB2d7s04hcsWJAZM2bkxhtvzPHHH99qraamJitWrEhzc3Nl2/Llyyu33BxwwAHp0aNHq3ckTz/9dDZs2FDZp6amZpt3LG8+Rk1NTZ555plWob98+fIMGDBAxAMAUIxdFvHz5s3L1Vdfnblz577tXzeccsop2bJlS2bPnp1NmzblySefzIIFCzJu3LgkSc+ePXPWWWdl1qxZefHFF/Pqq6+mrq4uH/nIR9K/f/8kyTnnnJMf/vCHuf/++7N58+bccccdWb16dT7xiU8kSY4++ujsv//+qaury/r167NmzZrMmTOn8hwAAFCCTi276EvSBw0alK5du6Z79+6tti9evDj9+vVL8sb3xM+YMSMrV67M+9///kycODHnnXdeZd+NGzdm5syZuffee7Nly5aceOKJmTFjRv7kT/6kss+iRYsya9asrFu3LgMHDsz06dNzwgknVNafe+65/OM//mMee+yx9OrVK+ecc04mT56cTp06tek8tt6f5J542LN87vCzO3oE2GE3rbi9o0f4o3z7iAkdPQLskM8u/267H/O9mnOXRfyeQsTDnknEsycQ8dAxOiLid/kHWwEAgB0j4gEAoDAiHgAACiPiAQCgMCIeAAAKI+IBAKAwIh4AAAoj4gEAoDAiHgAACiPiAQCgMCIeAAAKI+IBAKAwIh4AAAoj4gEAoDAiHgAACiPiAQCgMCIeAAAKI+IBAKAwIh4AAAoj4gEAoDAiHgAACiPiAQCgMCIeAAAKI+IBAKAwIh4AAAoj4gEAoDAiHgAACiPiAQCgMCIeAAAKI+IBAKAwIh4AAAoj4gEAoDAiHgAACiPiAQCgMCIeAAAKI+IBAKAwIh4AAAoj4gEAoDAiHgAACiPiAQCgMCIeAAAKI+IBAKAwIh4AAAoj4gEAoDAiHgAACiPiAQCgMCIeAAAKI+IBAKAwIh4AAAoj4gEAoDAiHgAACiPiAQCgMCIeAAAKI+IBAKAwIh4AAAoj4gEAoDAiHgAACiPiAQCgMCIeAAAKI+IBAKAwbY74F154IS0tLdtsb2lpyQsvvNCuQwEAAO+szRH/sY99LC+99NI221955ZV87GMfa9ehAACAd9bmiH+7q/BJsnHjxnTv3r3dBgIAAN5d1/fa4YYbbkiSdOrUKXPnzs373ve+ylpzc3Mef/zxHHzwwTtvQgAAoJX3jPgf/OAHSd64Ev8f//Ef6dKlS2WtW7duGTBgQL7yla/svAkBAIBW3jPi77vvviTJhAkTcsMNN2SfffbZ6UMBAADvrM33xH/3u9/d4YBfvHhxxo8fn2HDhmXQoEHbrA8aNCiDBw/O0KFDK7+eeuqpynpzc3OuueaajBgxIkOHDs3EiROzdu3aVsd46KGHcuaZZ2bIkCE59dRTc88997Raf/nllzN58uQMGzYsxx57bC6//PJs2rRph84LAAB2pfe8Ev9mDz/8cJYuXZrf/e53aW5ubrV21VVXvefj+/Tpk/Hjx2fjxo358pe//Lb7fPvb385xxx33tmtz5szJokWLMn/+/PTt2zdf+9rXcsEFF+Tuu+9O586d8/zzz+fCCy/M//t//y9//dd/nQcffDCTJ09O//79M2TIkCTJJZdcki5dumTJkiXZuHFjLrzwwnzta1/L5Zdfvj1/FAAA0GHafCX+W9/6Vs4///z853/+Z9auXZt169a1+tUWJ554Yk4//fTst99+f9Swt912WyZNmpSDDjoovXv3zpQpU7JmzZo89thjSZLvf//7OfTQQzNmzJh07949o0aNyqhRo3LbbbclSZ5//vk88MADmTp1avbZZ5/07ds3F198ce688840NTX9UTMBAMCu1uYr8f/2b/+Wyy67LOeee+7OnCdf+tKXsnnz5vTr1y/jxo3LOeeckyRpaGjI2rVrU1tbW9m3T58+GThwYFauXJljjjkmq1atarWeJLW1tVm8eHGSZNWqVenVq1erb9M58sgjs2HDhqxZsyY1NTVtnrO+vn5HThPYjQwfPryjR4B2s/XCVim8/thT7OrXXpsjvqGhISeddNLOnCXf+c53MnTo0HTu3DkPP/xwLrnkkrz++usZP358Ghsbk7wR7m9WXV1dWWtsbMyHP/zhVut9+vRptV5dXb3N47eubY/a2tr06NFjux4DADubKIaO0d6vvaampne9aNzm22n+4i/+Ig8//HC7DPVOTjjhhPTs2TPdu3fPSSedlPPPP7/yFZdVVVVJ3ngz8WYNDQ2Vtaqqqm3WX3vttVbrb431rftv3QcAAHZ3bb4Sf9RRR+Wb3/xmfvGLX6SmpibdunVrtX7GGWe0+3CdO3eu/Eux1dXV6d+/f+rr63PkkUcmeSPAn3322Rx22GFJkpqamvzkJz9pdYzly5dXbpOpqanJ+vXr8/TTT1duqamvr0/Pnj1z4IEHtvv8AACwM7Q54rf+g07z5s3bZq1Tp05tivgtW7bk9ddfz+bNm5Ok8mHSbt26ZeXKlWlubs6gQYPSuXPnPPLII7nlllvy+c9/vvL4sWPHZu7cuTn++OPTt2/f1NXV5YADDqj89cVZZ52VOXPm5I477siZZ56Zhx56KEuWLMm//uu/JkkGDBiQkSNHpq6uLl//+tfT1NSUWbNm5ZOf/KRbYwAAKEabI37VqlU7/GR33313pk2bVvl58ODBSd54Y/CHP/whdXV1WbduXbp06ZJ+/fpl8uTJGTduXGX/SZMmpaGhIePHj8+GDRsyfPjwzJ49O507v3FX0H777ZfZs2fnqquuyowZM/KhD30oV155ZeXrJZOkrq4uM2bMyKhRo9KlS5ecdtppufTSS3f43AAAYFfp1LL1fhXaZOuHDHywFfYsnzv87I4eAXbYTStu7+gR/ijfPmJCR48AO+Szy7/b7sd8r+Zs85X4G2644V3X/+7v/m77pwMAALZbmyN+67fEbPX666/nN7/5Tbp3754/+7M/E/EAALCLtDni77vvvm22/f73v8/UqVMzduzYdh0KAAB4Z23+nvi3s++++2by5Mmpq6trr3kAAID3sEMRnyRdu3bNiy++2B6zAAAAbdDm22kef/zxVj+3tLTkxRdfzJw5c1JbW9vugwEAAG+vzRE/fvz4dOrUKW/9Rsphw4blq1/9arsPBgAAvL02R/x///d/t/q5c+fO+cAHPuC70gEAYBdrc8T3799/Z84BAAC0UZsjPknWrFmTOXPm5H//93+TJIccckgmTpyYAw88cKcMBwAAbKvN307z4IMP5owzzsjKlSszZMiQDBkyJCtWrMiZZ56ZpUuX7swZAQCAN2nzlfhrrrkm48aNy5e//OVW27/61a/mG9/4Rm6//fZ2Hw4AANhWm6/Er169OuPGjdtm+/jx47N69ep2HQoAAHhnbY74qqqqrFu3bpvtL7zwQqqqqtp1KAAA4J21OeI//vGP57LLLstPfvKTbNiwIRs2bMiPf/zjXHHFFfn4xz++M2cEAADepM33xE+dOjXTpk3LZz/72XTq1Kmy/dRTT82UKVN2ynAAAMC22hzxvXv3zqxZs/Lss8+2+orJ/fbbb6cNBwAAbKvNEf8P//APOeKII/K5z30u+++/f2X7t771raxYsSLXXXfdThkQAABorc33xC9btiwnnXTSNttPOumkLFu2rF2HAgAA3lmbI76hoSHve9/7ttnes2fPvPrqq+06FAAA8M7aHPH7779/HnzwwW22P/jggxkwYEC7DgUAALyzNt8TP378+Fx99dXZtGlTRowYkeSNgL/++uvzhS98YacNCAAAtNbmiD/33HPz+9//Ptdee22+/vWvJ0m6d++ez3zmM5kwYcJOGxAAAGitzRGfvPENNZ/97Gfzi1/8Ikny4Q9/+G3vkwcAAHae7Yr4JOnVq1cGDx68M2YBAADaoM0fbAUAAHYPIh4AAAoj4gEAoDAiHgAACiPiAQCgMCIeAAAKI+IBAKAwIh4AAAoj4gEAoDAiHgAACiPiAQCgMCIeAAAKI+IBAKAwIh4AAAoj4gEAoDAiHgAACiPiAQCgMCIeAAAKI+IBAKAwIh4AAAoj4gEAoDAiHgAACiPiAQCgMCJ+N9K0samjR4B24b9lANi5unb0APyfHj175Oj9R3T0GLDDlj37UEePAAB7NFfiAQCgMCIeAAAKI+IBAKAwIh4AAAoj4gEAoDAiHgAACiPiAQCgMCIeAAAKI+IBAKAwIh4AAAoj4gEAoDAiHgAACiPiAQCgMCIeAAAKI+IBAKAwuzTiFy9enPHjx2fYsGEZNGjQNusrVqzI2LFjM2TIkHz0ox/NvHnzWq1v3Lgxl19+eY499tgMGzYskydPziuvvNJqn0WLFuWUU07J4MGDc+aZZ2bp0qWt1p977rlMnDgxQ4cOzYgRI3LttdempaWl/U8WAAB2kl0a8X369Mn48eMzffr0bdYaGxszadKkjBw5Mo8++miuu+663HDDDbn33nsr+1x55ZWpr6/PwoULs2TJkqxfvz5Tp06trD/++OOZPn16pk2blmXLlmXChAm58MIL88ILLyRJtmzZkgsuuCD9+vXLAw88kPnz52fhwoW5+eabd/7JAwBAO9mlEX/iiSfm9NNPz3777bfN2n333ZfOnTvnoosuSo8ePXLUUUdlzJgxufXWW5O8cRX+rrvuysUXX5y+fftmn332ydSpU/OjH/2oEunf+973cvLJJ2fUqFHp3r17xowZk0MOOSR33nlnkmTZsmV55plnMmXKlPTu3TsHHXRQJk2aVHkOAAAoQdeOHmCrVatW5fDDD0/nzv/3vqK2tjYLFixIkvzqV79KU1NTjjzyyMr6wQcfnF69emXlypXp169fVq1aldNPP73VcWtra7Nq1arKcwwcODB9+vRptf7888+nsbExVVVVbZ63vr7+jzrPdzN8+PB2PyZ0lMcee6yjR2gzrz32JCW99hKvP/Ycu/q1t9tEfGNjY6qrq1tt69OnTxobGyvrSbbZp7q6utU+bw70rcdYs2bNuz7H1rXtifja2tr06NGjzfvD3sb/mKFjeO1Bx2jv115TU9O7XjTebb6dpqqqqhLjW7322muVsN76e0NDQ6t9GhoaWu3z1vW3HuPtnuPNxwcAgN3dbhPxNTU1WbFiRZqbmyvbli9fnpqamiTJAQcckB49erR6R/L0009nw4YNlX1qamq2ecfy5mPU1NTkmWeeaRX6y5cvz4ABA0Q8AADF2KURv2XLljQ1NWXz5s1J3vhrgqampjQ3N+eUU07Jli1bMnv27GzatClPPvlkFixYkHHjxiVJevbsmbPOOiuzZs3Kiy++mFdffTV1dXX5yEc+kv79+ydJzjnnnPzwhz/M/fffn82bN+eOO+7I6tWr84lPfCJJcvTRR2f//fdPXV1d1q9fnzVr1mTOnDmV5wAAgBLs0oi/++67M3jw4EycODFJMnjw4AwePDg//elPU1VVlTlz5uTHP/5xjj766Pz93/99Pv/5z+e0006rPH769Ok57LDDMnr06IwaNSo9evTIP//zP1fWhw0blpkzZ2bmzJkZPnx4vvOd72T27NmVyO/SpUtuvPHGrF27NiNGjMj48eNz+umnV+YBAIASdGrxLx1tl60fMthZH2w9ev8R7X5M2NWWPftQR4+w3T53+NkdPQLssJtW3N7RI/xRvn3EhI4eAXbIZ5d/t92P+V7NudvcEw8AALSNiAcAgMKIeAAAKIyIBwCAwoh4AAAojIgHAIDCiHgAACiMiAcAgMKIeAAAKIyIBwCAwoh4AAAojIgHAIDCiHgAACiMiAcAgMKIeAAAKIyIBwCAwoh4AAAojIgHAIDCiHgAACiMiAcAgMKIeAAAKIyIBwCAwoh4AAAojIgHAIDCiHgAACiMiAcAgMKIeAAAKIyIBwCAwoh4AAAojIgHAIDCiHgAACiMiAcAgMKIeAAAKIyIBwCAwoh4AAAojIgHAIDCiHgAACiMiAcAgMKIeAAAKIyIBwCAwoh4AAAojIgHAIDCiHgAACiMiAcAgMKIeAAAKIyIBwCAwoh4AAAojIgHAIDCiHgAACiMiAcAgMKIeAAAKIyIBwCAwoh4AAAojIgHAIDCiHgAACiMiAcAgMKIeAAAKIyIBwCAwoh4AAAojIgHAIDCiHgAACiMiAcAgMKIeAAAKIyIBwCAwoh4AAAojIgHAIDC7FYRf/311+ewww7L0KFDK7+++MUvVtZXrFiRsWPHZsiQIfnoRz+aefPmtXr8xo0bc/nll+fYY4/NsGHDMnny5Lzyyiut9lm0aFFOOeWUDB48OGeeeWaWLl26S84NAADay24V8Uly9NFH54knnqj8uuaaa5IkjY2NmTRpUkaOHJlHH3001113XW644Ybce++9lcdeeeWVqa+vz8KFC7NkyZKsX78+U6dOraw//vjjmT59eqZNm5Zly5ZlwoQJufDCC/PCCy/s8vMEAIA/1m4X8e/kvvvuS+fOnXPRRRelR48eOeqoozJmzJjceuutSd64Cn/XXXfl4osvTt++fbPPPvtk6tSp+dGPflSJ9O9973s5+eSTM2rUqHTv3j1jxozJIYcckjvvvLMjTw0AALZL144e4K3q6+tz/PHHp1evXpVbYvbbb7+sWrUqhx9+eDp3/r/3HbW1tVmwYEGS5Fe/+lWamppy5JFHVtYPPvjg9OrVKytXrky/fv2yatWqnH766a2er7a2NqtWrfqj5mxvw4cPb/djQkd57LHHOnqENvPaY09S0msv8fpjz7GrX3u7VcSfeuqp+eQnP5l+/frlxRdfzDe+8Y185jOfyd13353GxsZUV1e32r9Pnz5pbGxMksrvb92nurq61T59+vTZ5hhr1qzZ7llra2vTo0eP7X4c7C38jxk6htcedIz2fu01NTW960Xj3ep2mkMPPTT9+/dPp06d0rdv38ycOTO//e1v88QTT6SqqqoS41u99tprqaqqSpLK7w0NDa32aWhoaLXPW9fffAwAACjBbhXxb9WpU6d06tQpLS0tqampyYoVK9Lc3FxZX758eWpqapIkBxxwQHr06NHqHcvTTz+dDRs2VPapqanZ5h3Nm48BAAAl2K0i/p577slLL72UJPn973+fyy67LB/4wAcydOjQnHLKKdmyZUtmz56dTZs25cknn8yCBQsybty4JEnPnj1z1llnZdasWXnxxRfz6quvpq6uLh/5yEfSv3//JMk555yTH/7wh7n//vuzeTCXBw0AAAo8SURBVPPm3HHHHVm9enU+8YlPdNg5AwDA9tqt7on/wQ9+kK985SvZsGFD+vTpk2OOOSa33HJL5XaXOXPmZMaMGbnpppvy/ve/P5///Odz2mmnVR4/ffr0zJw5M6NHj86WLVty4oknZsaMGZX1YcOGZebMmZk5c2bWrVuXgQMHZvbs2ZXIBwCAEnRqaWlp6eghSrL1QwY764OtR+8/ot2PCbvasmcf6ugRttvnDj+7o0eAHXbTits7eoQ/yrePmNDRI8AO+ezy77b7Md+rOXer22kAAID3JuIBAKAwIh4AAAoj4gEAoDAiHgAACiPiAQCgMCIeAAAKI+IBAKAwIh4AAAoj4gEAoDAiHgAACiPiAQCgMCIeAAAKI+IBAKAwIh4AAAoj4gEAoDAiHgAACiPiAQCgMCIeAAAKI+IBAKAwIh4AAAoj4gEAoDAiHgAACiPiAQCgMCIeAAAKI+IBAKAwIh4AAAoj4gEAoDAiHgAACiPiAQCgMCIeAAAKI+IBAKAwIh4AAAoj4gEAoDAiHgAACiPiAQCgMCIeAAAKI+IBAKAwIh4AAAoj4gEAoDAiHgAACiPiAQCgMCIeAAAKI+IBAKAwIh4AAAoj4gEAoDAiHgAACiPiAQCgMCIeAAAKI+IBAKAwIh4AAAoj4gEAoDAiHgAACiPiAQCgMCIeAAAKI+IBAKAwIh4AAAoj4gEAoDAiHgAACiPiAQCgMCIeAAAKI+IBAKAwIh4AAAoj4gEAoDAiHgAACrNXRnxzc3OuueaajBgxIkOHDs3EiROzdu3ajh4LAADaZK+M+Dlz5mTRokWZP39+HnjggfTr1y8XXHBBmpubO3o0AAB4T107eoCOcNttt2XSpEk56KCDkiRTpkzJiBEj8thjj+WYY45518e2tLQkSTZt2rRTZtv3gx/YKceFXampqamjR9huVX+6T0ePADusxNdeknT/0z4dPQLskJ3x2tvamlvb8606tbzTyh6qoaEhRx99dBYsWJDBgwdXto8ePTp/8zd/k/POO+89H7969eqdPSYAAOTQQw9NdXX1Ntv3uivxjY2NSZI+fVq/66+urq6svZvevXvn0EMPTbdu3dKpU6edMiMAAHu3lpaWbN68Ob17937b9b0u4quqqpK8cUX9zRoaGipr76Zz585v+24IAADaU8+ePd9xba/7YGt1dXX69++f+vr6yraGhoY8++yzOeywwzpwMgAAaJu9LuKTZOzYsZk7d27WrFmT9evXp66uLgcccECGDx/e0aMBAMB72utup0mSSZMmpaGhIePHj8+GDRsyfPjwzJ49O50775XvaQAAKMxe9+00AABQOpeeAQCgMCIeAAAKI+IBAKAwIh4AAAoj4gEAoDAinr1Cc3NzrrnmmowYMSJDhw7NxIkTs3bt2o4eC/Z4ixcvzvjx4zNs2LAMGjSoo8eBvUZdXV1Gjx6dYcOGZeTIkZk+fXpefvnljh6LdiTi2SvMmTMnixYtyvz58/PAAw+kX79+ueCCC9Lc3NzRo8EerU+fPhk/fnymT5/e0aPAXqVLly6pq6vLI488krvvvjvr1q3LtGnTOnos2pHviWevcPLJJ2fSpEkZP358kuS1117LiBEjcsstt+SYY47p4Olgz/fII4/kvPPOy1NPPdXRo8BeacmSJfnSl76Uxx9/vKNHoZ24Es8er6GhIWvXrk1tbW1lW58+fTJw4MCsXLmyAycDgF1j6dKlqamp6egxaEddO3oA2NkaGxuTvBHub1ZdXV1ZA4A91T333JMFCxZk/vz5HT0K7ciVePZ4VVVVSd64Iv9mDQ0NlTUA2BMtXrw4V1xxRWbPnp0jjjiio8ehHYl49njV1dXp379/6uvrK9saGhry7LPP5rDDDuvAyQBg51mwYEFmzJiRG2+8Mccff3xHj0M7E/HsFcaOHZu5c+dmzZo1Wb9+ferq6nLAAQdk+PDhHT0a7NG2bNmSpqambN68OUnS1NSUpqYm3wwFO9m8efNy9dVXZ+7cuf5ft4fy7TTsFZqbm3Pttdfm9ttvz4YNGzJ8+PDMmDEjAwYM6OjRYI925513vu3X2s2bNy/HHXdcB0wEe4dBgwala9eu6d69e6vtixcvTr9+/TpoKtqTiAcAgMK4nQYAAAoj4gEAoDAiHgAACiPiAQCgMCIeAAAKI+IBAKAwIh6At3XppZfm/PPPb/P+J598cv7lX/5lh5+3vY4DsCcT8QAAUBgRDwAAhRHxALyn5cuXZ9KkSTnhhBMydOjQfOpTn8qPf/zjbfZramrKl7/85QwbNizHHXdcrrnmmjQ3N1fWN2/enOuvvz4nn3xyjjzyyIwePTq33Xbbuz73f/3Xf+Wss87KkCFDcvTRR+fss8/OihUr2v0cAUrStaMHAGD319jYmL/6q7/K1KlT07Vr19x111256KKLsnDhwhx44IGV/b773e/m05/+dG6//fY8+eST+cd//Mfsu++++fSnP50kueyyy7J8+fJ85StfycCBA/Pzn/88l19+ebp06ZIxY8Zs87y//e1vM3ny5Fx88cX5y7/8y2zatCkrVqxIly5ddtm5A+yORDwA7+m4445r9fMXvvCFLFmyJPfee28uvPDCyvaamppcfPHFSZKDDjoov/zlL3PzzTfn05/+dJ577rncddddWbx4cQ4++OAkyX777Zdf/vKXmT9//jtG/ObNm3PaaadlwIABSVJ5LMDeTMQD8J5eeumlzJo1Kw8//HB+97vfZcuWLWlqasoLL7zQar+jjjqq1c/Dhg3LTTfdlMbGxtTX16elpSVnn312q31ef/31d7yyPmjQoIwcOTJnnHFGRowYkWOPPTannHJK/vzP/7x9TxCgMCIegPd06aWX5te//nWmTJmSAQMGpGfPnvnCF76QzZs3t/kYLS0tSZJ///d/T69evVqtderU6W0f06VLl8yZMyc///nP89BDD+W+++7LN77xjXzzm9/MqFGj/vgTAiiciAfgPf30pz/NlClT8rGPfSxJsn79+jz//PM59NBDW+33s5/9rNXPTzzxRPr27ZuqqqocccQRSZJf//rX2xXgnTp1yuDBgzN48OBccMEFmThxYu68804RD+zVRDwA7+nAAw/MwoULM3z48DQ3N+eb3/xmtmzZss1+K1euzPXXX5/TTz899fX1mTdvXuUe+YEDB+ZTn/pULrvsslxyySUZOnRoNmzYkPr6+rz00kv527/9222O9/jjj2fp0qUZOXJkPvjBD+aZZ57JU089tc0tOQB7GxEPwHu66qqrcsUVV2TMmDH50z/900ycODEbN27cZr8JEybkhRdeyKc+9al069Yt5557bs4777zK+j/90z/l5ptvzo033pjnn38+vXv3ziGHHJJzzz33bZ+3uro6//M//5Nbb701r776aj74wQ/mjDPOyEUXXbTTzhWgBJ1att6kCAAAFME/9gQAAIUR8QAAUBgRDwAAhRHxAABQGBEPAACFEfEAAFAYEQ8AAIUR8QAAUJj/Dz/BInyAGHRzAAAAAElFTkSuQmCC\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "E3eAorweCsdv"
      },
      "outputs": [],
      "source": [
        "# balance the classes\n",
        "\n",
        "class_1,class_2,class_0 = df.label.value_counts()\n",
        "c0 = df[df['label'] == 0]\n",
        "c2 = df[df['label'] == 2]\n",
        "c1 = df[df['label'] == 1]\n",
        "\n",
        "df_1 = c1.sample(class_0 + 1500)\n",
        "df_2 = c2.sample(class_0)\n",
        "undersampled_df = pd.concat([df_1,df_2,c0],axis=0)\n",
        "\n",
        "df = undersampled_df.sample(frac=1, ignore_index = True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 562
        },
        "id": "PTr2NwDf3UIq",
        "outputId": "ab31c850-4995-42eb-cfce-25ee1bdf8ff3"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/seaborn/_decorators.py:43: FutureWarning: Pass the following variable as a keyword arg: x. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.\n",
            "  FutureWarning\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 864x576 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAvEAAAHpCAYAAAALVWOKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dfZCXdb3/8Re3C7G7lN1YiHlD4ooryGJaDJVrphnqkIWtOJhnoBI7Hql0uGnSQ+VNbZmax80OdhqzDoW3IY7HscgiSVFP2gKLHaMMzbAi2G1xIXZ/fzh8f26rsuTC7gWPx0zT7PX5fK99X47X8Px+uXbt19HR0REAAKAw+vf2AAAAwK4R8QAAUDAiHgAACkbEAwBAwQzs7QGKpr29PX/7298yaNCg9OvXr7fHAQBgL9TR0ZFt27Zl2LBh6d+/6+fuIn4X/e1vf8sTTzzR22MAALAPGD16dCoqKrocF/G7aNCgQUle+Ac6ePDgXp4GAIC90datW/PEE0+U2vMfifhdtOMRmsGDB6esrKyXpwEAYG/2co9v+8FWAAAoGBEPAAAFI+IBAKBgRDwAABSMiAcAgIIR8QAAUDAiHgAACkbEAwBAwYh4AAAoGBEPAAAFI+IBAKBgRDwAABSMiAcAgIIR8QAAUDAiHgAACkbEAwBAwYh4AAAoGBEPAAAFI+KBfd62tq29PQL0CP8uw75jYG8PANDbBpUNzifGfLi3x4BX7YbVt/T2CMAe4pN4AAAoGBEPAAAFI+IBAKBgRDwAABSMiAcAgIIR8QAAUDAiHgAACkbEAwBAwYh4AAAoGBEPAAAFI+IBAKBgRDwAABSMiAcAgIIR8QAAUDAiHgAACkbEAwBAwYh4AAAoGBEPAAAFI+IBAKBgRDwAABSMiAcAgIIR8QAAUDAiHgAACmaPRvzSpUszbdq01NTU5PDDD++09stf/jIf//jHM3HixNTU1OSDH/xg7r333k572tvbc9VVV2XixIkZP358ZsyYkaeffrrTngceeCCnn356xo0bl5NPPjl33313p/WNGzdm9uzZqampybHHHptLLrkkW7du3T0XDAAAu8EejfjKyspMmzYt8+fP77K2adOmfOADH8hdd92Vhx9+OOedd14+85nP5PHHHy/tWbhwYe66667cfPPNWb58eUaMGJHzzjsv7e3tSZL169dn1qxZmT59elauXJm5c+dm3rx5eeyxx0rnuOiii9La2pply5ZlyZIlaWxszJVXXrn7Lx4AAHrIHo34d73rXTn11FNz4IEHdll7z3vekylTpmS//fZL//79c/LJJ+ewww7LI488UtqzaNGizJw5M4ceemiGDRuWiy++OOvWrSvtuf322zN69OhMnTo1gwcPTm1tbWpra7No0aIkL0T+8uXLM2fOnAwfPjz7779/Lrzwwtx2221pa2vbM/8QAADgVRrY2wO8nD/+8Y/5zW9+k6qqqiRJc3Nznn766VRXV5f2VFZW5qCDDsqaNWvy9re/PU1NTZ3Wk6S6ujpLly5NkjQ1NWXo0KEZNWpUaf2oo47Kli1bsm7dutL36o7GxsZXc3lAHzJhwoTeHgF6zIs//AL2Xn0y4v/2t7/lggsuSG1tbd75zncmSVpaWpK8EO4vVlFRUVpraWnJ2972tk7rlZWVndYrKiq6vP7F5++u6urqlJWV7dJrAGB386YU9g5tbW2v+KFxn/vtNM3NzZk5c2be+MY35ktf+lLpeHl5eWn9H/fvWCsvL++yvnnz5k7r/xjrO/bv2AMAAH1dn4r4jRs35qMf/Wje8pa35JprrsngwYNLaxUVFTnggAM6vSNpbm7OU089lSOOOCJJUlVV1eUdy6pVq0qPyVRVVaW1tTVPPvlkab2xsTFDhgzJIYccsjsvDQAAeswejfjt27enra0t27ZtS/LCXxO0tbWlvb09zz33XKZPn57DDz88X/nKVzJwYNcnferq6nLjjTdm3bp1aW1tTX19fQ4++ODSXx1OmTIla9euza233ppt27bl/vvvz7Jly1JXV5ckGTlyZCZNmpT6+vps2rQpGzZsyLXXXpszzjjDozEAABTGHn0m/s4778y8efNKX48dOzZJctNNN2XlypX59a9/nfXr1+eee+4p7TnttNPy+c9/Pkkyc+bMNDc3Z9q0admyZUsmTJiQhoaG9O//wnuRAw88MA0NDbniiiuyYMGCvPnNb87ll1+ecePGlc5XX1+fBQsWpLa2NgMGDMgpp5ySuXPn7onLBwCAHtGvo6Ojo7eHKJIdP2TgB1th7/KJMR/u7RHgVbth9S29PQLQQ3bWnH3qmXgAAGDnRDwAABSMiAcAgIIR8QAAUDAiHgAACkbEAwBAwYh4AAAoGBEPAAAFI+IBAKBgRDwAABSMiAcAgIIR8QAAUDAiHgAACkbE9yFtz7f19gjQI/y7DHTX39u29vYI8Kr1xr/HA/f4d+RllQ0pyzFvndjbY8Cr9vBTD/T2CEBBDCwbnP88cnpvjwGvysdWfWePf0+fxAMAQMGIeAAAKBgRDwAABSPiAQCgYEQ8AAAUjIgHAICCEfEAAFAwIh4AAApGxAMAQMGIeAAAKBgRDwAABSPiAQCgYEQ8AAAUjIgHAICCEfEAAFAwIh4AAApGxAMAQMGIeAAAKBgRDwAABSPiAQCgYEQ8AAAUjIgHAICCEfEAAFAwIh4AAApGxAMAQMGIeAAAKBgRDwAABSPiAQCgYEQ8AAAUjIgHAICCEfEAAFAwIh4AAApGxAMAQMGIeAAAKBgRDwAABSPiAQCgYEQ8AAAUjIgHAICCEfEAAFAwezTily5dmmnTpqWmpiaHH354l/XVq1enrq4u48aNy/HHH5+bbrqp0/rzzz+fSy65JMcee2xqamoye/bs/PWvf+2056677spJJ52UsWPH5vTTT8+KFSs6rf/+97/PjBkzMn78+EycODFf+9rX0tHR0fMXCwAAu8kejfjKyspMmzYt8+fP77LW0tKSmTNnZtKkSXnooYdy9dVX57rrrss999xT2nP55ZensbExS5YsybJly9La2po5c+aU1h999NHMnz8/8+bNy8MPP5zp06dn1qxZeeaZZ5Ik27dvz3nnnZcRI0Zk+fLlufnmm7NkyZJ861vf2v0XDwAAPWSPRvy73vWunHrqqTnwwAO7rN17773p379/zj///JSVleXoo4/O1KlT873vfS/JC5/C33HHHbnwwguz//77Z/jw4ZkzZ05+8pOflCL9Bz/4QU444YTU1tZm8ODBmTp1ag477LDcdtttSZKHH344v/vd73LxxRdn2LBhOfTQQzNz5szS9wAAgCIY2NsD7NDU1JQxY8akf////76iuro6ixcvTpL89re/TVtbW4466qjS+qhRozJ06NCsWbMmI0aMSFNTU0499dRO562urk5TU1Ppexx00EGprKzstL5+/fq0tLSkvLy82/M2Njb+U9f5SiZMmNDj54Te8sgjj/T2CN3m3mNvUqR7L3H/sffY0/den4n4lpaWVFRUdDpWWVmZlpaW0nqSLnsqKio67XlxoO84x7p1617xe+xY25WIr66uTllZWbf3w77GH8zQO9x70Dt6+t5ra2t7xQ+N+8xvpykvLy/F+A6bN28uhfWO/29ubu60p7m5udOef1z/x3O81Pd48fkBAKCv6zMRX1VVldWrV6e9vb10bNWqVamqqkqSHHzwwSkrK+v0juTJJ5/Mli1bSnuqqqq6vGN58Tmqqqryu9/9rlPor1q1KiNHjhTxAAAUxh6N+O3bt6etrS3btm1L8sJfE7S1taW9vT0nnXRStm/fnoaGhmzdujWPP/54Fi9enLPOOitJMmTIkEyZMiXXXnttNmzYkE2bNqW+vj7vec97csABByRJzjzzzPz4xz/O/fffn23btuXWW2/NE088kQ9+8INJkmOOOSZvfetbU19fn9bW1qxbty4LFy4sfQ8AACiCPRrxd955Z8aOHZsZM2YkScaOHZuxY8dm5cqVKS8vz8KFC/PTn/40xxxzTC644IJ88pOfzCmnnFJ6/fz583PEEUdk8uTJqa2tTVlZWb785S+X1mtqanLZZZflsssuy4QJE/Ltb387DQ0NpcgfMGBAvvGNb+Tpp5/OxIkTM23atJx66qmleQAAoAj6dfgvHe2SHT9ksLt+sPWYt07s8XPCnvbwUw/09gi77BNjPtzbI8CrdsPqW3p7hH/Kfx45vbdHgFflY6u+0+Pn3Flz9pln4gEAgO4R8QAAUDAiHgAACkbEAwBAwYh4AAAoGBEPAAAFI+IBAKBgRDwAABSMiAcAgIIR8QAAUDAiHgAACkbEAwBAwYh4AAAoGBEPAAAFI+IBAKBgRDwAABSMiAcAgIIR8QAAUDAiHgAACkbEAwBAwYh4AAAoGBEPAAAFI+IBAKBgRDwAABSMiAcAgIIR8QAAUDAiHgAACkbEAwBAwYh4AAAoGBEPAAAFI+IBAKBgRDwAABSMiAcAgIIR8QAAUDAiHgAACkbEAwBAwYh4AAAoGBEPAAAFI+IBAKBgRDwAABSMiAcAgIIR8QAAUDAiHgAACkbEAwBAwYh4AAAoGBEPAAAFI+IBAKBgRDwAABSMiAcAgIIR8QAAUDAiHgAACkbEAwBAwYh4AAAoGBEPAAAFI+IBAKBg+lzE/+lPf8pnPvOZvPOd78wxxxyTurq6rFy5srT+wAMP5PTTT8+4ceNy8skn5+677+70+o0bN2b27NmpqanJsccem0suuSRbt27ttOfb3/52jj/++IwbNy51dXVpamraI9cGAAA9oc9F/IIFC7Jhw4YsXbo0Dz74YE466aR8/OMfz+bNm7N+/frMmjUr06dPz8qVKzN37tzMmzcvjz32WOn1F110UVpbW7Ns2bIsWbIkjY2NufLKK0vrS5cuzfXXX5+rr746Dz30UCZNmpSZM2empaWlNy4XAAB2WZ+L+N/97nd5//vfn/322y8DBgzIRz7ykbS2tuapp57K7bffntGjR2fq1KkZPHhwamtrU1tbm0WLFiVJ1q9fn+XLl2fOnDkZPnx49t9//1x44YW57bbb0tbWliRZtGhRpk6dmqOPPjplZWU5//zzkyT33Xdfr10zAADsioG9PcA/+tjHPpZbbrklJ510Ul772tfmu9/9bg4++OCMHj06DQ0Nqa6u7rS/uro6S5cuTZI0NTVl6NChGTVqVGn9qKOOypYtW7Ju3bpUVVWlqakp06ZNK633798/Y8aMyZo1azJlypRuz9nY2Pgqr7SrCRMm9Pg5obc88sgjvT1Ct7n32JsU6d5L3H/sPfb0vdfnIn78+PG54447MmnSpAwYMCCvfe1r8x//8R8ZPHhwWlpa8ra3va3T/srKytKjMC0tLamoqOi0vuPrF++prKzssmdXH6eprq5OWVnZLr0G9iX+YIbe4d6D3tHT915bW9srfmjcpyK+vb095557bo477rg89NBDGTZsWH7yk5/kYx/7WL773e+mvLw8zc3NnV6zefPmlJeXJ0nKy8u7xPiO/S/e84/naG5uzhve8IbddVkAANCj+tQz8Zs2bcrvf//7nHPOORk+fHgGDhyYE088MQceeGB+/vOfp6qqqss7klWrVqWqqipJUlVVldbW1jz55JOl9cbGxgwZMiSHHHJIac+Lz9He3p7Vq1fniCOO2ANXCAAAr16fivjXve51GTVqVL773e+mpaUl7e3t+dGPfpRf//rXOfLIIzNlypSsXbs2t956a7Zt25b7778/y5YtS11dXZJk5MiRmTRpUurr67Np06Zs2LAh1157bc4444zSoy91dXVZvHhxHn/88WzdujUNDQ1JkhNPPLHXrhsAAHZFn3qcJkmuv/76fPnLX8773ve+tLW15YADDsill16a4447LknS0NCQK664IgsWLMib3/zmXH755Rk3blzp9fX19VmwYEFqa2szYMCAnHLKKZk7d25pffLkyXnuuedywQUXZOPGjRkzZkwWLlxYetwGAAD6uj4X8QcffHCuv/76l12fOHFilixZ8rLr++23X6655ppX/B7nnntuzj333H92RAAA6FV96nEaAABg50Q8AAAUjIgHAICCEfEAAFAwIh4AAApGxAMAQMGIeAAAKBgRDwAABSPiAQCgYEQ8AAAUjIgHAICCEfEAAFAwIh4AAApGxAMAQMGIeAAAKJhuR/wzzzyTjo6OLsc7OjryzDPP9OhQAADAy+t2xL/3ve/NX/7yly7H//rXv+a9731vjw4FAAC8vG5H/Et9Cp8kzz//fAYPHtxjAwEAAK9s4M42XHfddUmSfv365cYbb8xrXvOa0lp7e3seffTRjBo1avdNCAAAdLLTiP/hD3+Y5IVP4v/nf/4nAwYMKK0NGjQoI0eOzOc///ndNyEAANDJTiP+3nvvTZJMnz491113XYYPH77bhwIAAF7eTiN+h+985zu7cw4AAKCbuh3xSfKLX/wiK1asyJ/+9Ke0t7d3Wrviiit6dDAAAOCldTviv/nNb+aqq67KoYcemje96U3p16/f7pwLAAB4Gd2O+O9+97v53Oc+l7PPPnt3zgMAAOxEt39PfHNzc9797nfvzlkAAIBu6HbEn3jiifnFL36xO2cBAAC6oduP0xx99NG55ppr8utf/zpVVVUZNGhQp/XTTjutx4cDAAC66nbE7/gPOt10001d1vr16yfiAQBgD+l2xDc1Ne3OOQAAgG7q9jPxAABA39DtT+Kvu+66V1z/13/911c9DAAAsHPdjvgf/vCHnb7++9//nj/+8Y8ZPHhw3vSmN4l4AADYQ7od8ffee2+XY3/+858zZ86c1NXV9ehQAADAy3tVz8S//vWvz+zZs1NfX99T8wAAADvxqn+wdeDAgdmwYUNPzAIAAHRDtx+nefTRRzt93dHRkQ0bNmThwoWprq7u8cEAAICX1u2InzZtWvr165eOjo5Ox2tqavLFL36xxwcDAABeWrcj/kc/+lGnr/v375/99tsvZWVlPT4UAADw8rod8QcccMDunAMAAOimbkd8kqxbty4LFy7M//3f/yVJDjvssMyYMSOHHHLIbhkOAADoqtu/nebnP/95TjvttKxZsybjxo3LuHHjsnr16px++ulZsWLF7pwRAAB4kW5/En/VVVflrLPOymc/+9lOx7/4xS/mq1/9am655ZYeHw4AAOiq25/EP/HEEznrrLO6HJ82bVqeeOKJHh0KAAB4ed2O+PLy8jz77LNdjj/zzDMpLy/v0aEAAICX1+2If9/73pfPfe5z+dnPfpYtW7Zky5Yt+elPf5pLL70073vf+3bnjAAAwIt0+5n4OXPmZN68efnYxz6Wfv36lY6ffPLJufjii3fLcAAAQFfdjvhhw4bl2muvzVNPPdXpV0weeOCBu204AACgq25H/L/927/lyCOPzCc+8Ym89a1vLR3/5je/mdWrV+fqq6/eLQMCAACddfuZ+Icffjjvfve7uxx/97vfnYcffrhHhwIAAF5etyO+ubk5r3nNa7ocHzJkSDZt2tSjQwEAAC+v2xH/1re+NT//+c+7HP/5z3+ekSNH9uhQAADAy+v2M/HTpk3LV77ylWzdujUTJ05M8kLAf/3rX8+nPvWp3TYgAADQWbcj/uyzz86f//znfO1rX8uXvvSlJMngwYPzL//yL5k+ffpuGxAAAOis24/TJC/8hppf/OIX+f73v5/vf//7WbFiRWbPnt3jQz300EOZNm1axo8fn2OPPTazZs0qrT3wwAM5/fTTM27cuJx88sm5++67O71248aNmT17dmpqanLsscfmkksuydatWzvt+fa3v53jjz8+48aNS11dXZqamnr8GgAAYHfZpYhPkqFDh2bs2LEZO3bsS/6g66u1cuXKzJo1K3V1dVmxYkWWL19eivj169dn1qxZmT59elauXJm5c+dm3rx5eeyxx0qvv+iii9La2pply5ZlyZIlaWxszJVXXllaX7p0aa6//vpcffXVeeihhzJp0qTMnDkzLS0tPX4tAACwO+xyxO9uX/3qV3PmmWfm9NNPz5AhQzJ48OCMHTs2SXL77bdn9OjRmTp1agYPHpza2trU1tZm0aJFSV6I/OXLl2fOnDkZPnx49t9//1x44YW57bbb0tbWliRZtGhRpk6dmqOPPjplZWU5//zzkyT33Xdf71wwAADsom4/E78ntLa25rHHHsv48eNzxhln5Omnn87BBx+c2bNn553vfGeamppSXV3d6TXV1dVZunRpkqSpqSlDhw7NqFGjSutHHXVUtmzZknXr1qWqqipNTU2ZNm1aab1///4ZM2ZM1qxZkylTpnR71sbGxld5tV1NmDChx88JveWRRx7p7RG6zb3H3qRI917i/mPvsafvvT4V8Zs3b057e3uWLFmSb37zmznssMNy++2357zzzstdd92VlpaWvO1tb+v0msrKytKjMC0tLamoqOi0vuPrF++prKzssmdXH6eprq5OWVnZLr0G9iX+YIbe4d6D3tHT915bW9srfmjcpx6nGTZsWJLkQx/6UMaMGZNBgwblzDPPzMiRI/Ozn/0s5eXlaW5u7vSazZs3p7y8PElSXl7eJcZ37H/xnn88R3Nzc2kdAAD6uj4V8RUVFTnwwAO7HO/Xr1+SpKqqqss7klWrVqWqqqq03tramieffLK03tjYmCFDhuSQQw55yXO0t7dn9erVOeKII3r8egAAYHfoUxGfvPD76G+99dasXbs227dvz6233pqnn3467373uzNlypSsXbs2t956a7Zt25b7778/y5YtS11dXZJk5MiRmTRpUurr67Np06Zs2LAh1157bc4444zSoy91dXVZvHhxHn/88WzdujUNDQ1JkhNPPLHXrhkAAHZFn3omPknOPffc/O1vf8uMGTPS2tqaww47LDfccENGjhyZJGloaMgVV1yRBQsW5M1vfnMuv/zyjBs3rvT6+vr6LFiwILW1tRkwYEBOOeWUzJ07t7Q+efLkPPfcc7nggguycePGjBkzJgsXLvQ4DQAAhdGvo6Ojo7eHKJIdP2Swu36w9Zi3Tuzxc8Ke9vBTD/T2CLvsE2M+3NsjwKt2w+pbenuEf8p/Hum//E6xfWzVd3r8nDtrzj73OA0AAPDKRDwAABSMiAcAgIIR8QAAUDAiHgAACkbEAwBAwYh4AAAoGBEPAAAFI+IBAKBgRDwAABSMiAcAgIIR8QAAUDAiHgAACkbEAwBAwYh4AAAoGBEPAAAFI+IBAKBgRDwAABSMiAcAgIIR8QAAUDAiHgAACkbEAwBAwYh4AAAoGBEPAAAFI+IBAKBgRDwAABSMiAcAgIIR8QAAUDAiHgAACkbEAwBAwYh4AAAoGBEPAAAFI+IBAKBgRDwAABSMiAcAgIIR8QAAUDAiHgAACkbEAwBAwYh4AAAoGBEPAAAFI+IBAKBgRDwAABSMiAcAgIIR8QAAUDAiHgAACkbEAwBAwYh4AAAoGBEPAAAFI+IBAKBgRDwAABSMiAcAgIIR8QAAUDAiHgAACkbEAwBAwfTZiP/kJz+Zww8/PA8++GDp2AMPPJDTTz8948aNy8knn5y7776702s2btyY2bNnp6amJscee2wuueSSbN26tdOeb3/72zn++OMzbty41NXVpampaY9cDwAA9JQ+GfF33HFHnn/++U7H1q9fn1mzZmX69OlZuXJl5s6dm3nz5uWxxx4r7bnooovS2tqaZcuWZcmSJWlsbMyVV15ZWl+6dGmuv/76XH311XnooYcyadKkzJw5My0tLXvs2gAA4NXqcxH/7LPP5uqrr84XvvCFTsdvv/32jB49OlOnTs3gwYNTW1ub2traLFq0KMkLkb98+fLMmTMnw4cPz/77758LL7wwt912W9ra2pIkixYtytSpU3P00UenrKws559/fpLkvvvu27MXCQAAr0KfiviOjo7Mnz8/s2bNyogRIzqtNTU1pbq6utOx6urq0uMwTU1NGTp0aEaNGlVaP+qoo7Jly5asW7fuJc/Rv3//jBkzJmvWrNldlwQAAD1uYG8P8GLf+9730tHRkY985CNd1lpaWvK2t72t07HKysrSozAtLS2pqKjotL7j6xfvqays7LLnn3mcprGxcZdfszMTJkzo8XNCb3nkkUd6e4Ruc++xNynSvZe4/9h77Ol7r89E/FNPPZWGhoZ8//vff8n18vLyNDc3dzq2efPmlJeXl9b/McZ37H/xnn88R3Nzc97whjfs8rzV1dUpKyvb5dfBvsIfzNA73HvQO3r63mtra3vFD437TMQ//PDD+etf/5ozzjij0/Hzzz8/p556aqqqqvKzn/2s09qqVatSVVWVJKmqqkpra2uefPLJ0iM1jY2NGTJkSA455JDSnsbGxrz//e9PkrS3t2f16tX5wAc+sLsvDwAAekyfeSb+lFNOyX333Zc777yz9L8k+eIXv5hPf/rTmTJlStauXZtbb70127Zty/33359ly5alrq4uSTJy5MhMmjQp9fX12bRpUzZs2JBrr702Z5xxRukT87q6uixevDiPP/54tm7dmoaGhiTJiSee2DsXDQAA/4Q+80n80KFDM3To0C7H99tvvwwfPjzDhw9PQ0NDrrjiiixYsCBvfvObc/nll2fcuHGlvfX19VmwYEFqa2szYMCAnHLKKZk7d25pffLkyXnuuedywQUXZOPGjRkzZkwWLlxYetwGAACKoM9E/EtZu3Ztp68nTpyYJUuWvOz+/fbbL9dcc80rnvPcc8/Nueee2xPjAQBAr+gzj9MAAADdI+IBAKBgRDwAABSMiAcAgIIR8QAAUDAiHgAACkbEAwBAwYh4AAAoGBEPAAAFI+IBAKBgRDwAABSMiAcAgIIR8QAAUDAiHgAACkbEAwBAwYh4AAAoGBEPAAAFI+IBAKBgRDwAABSMiAcAgIIR8QAAUDAiHgAACkbEAwBAwYh4AAAoGBEPAAAFI+IBAKBgRDwAABSMiAcAgIIR8QAAUDAiHgAACkbEAwBAwYh4AAAoGBEPAAAFI+IBAKBgRDwAABSMiAcAgIIR8QAAUDAiHgAACkbEAwBAwYh4AAAoGBEPAAAFI+IBAKBgRDwAABSMiAcAgIIR8QAAUDAiHgAACkbEAwBAwYh4AAAoGBEPAAAFI+IBABzqHRcAAA0VSURBVKBgRDwAABSMiAcAgIIR8QAAUDAiHgAACkbEAwBAwfSpiK+vr8/kyZNTU1OTSZMmZf78+dm4cWOnPatXr05dXV3GjRuX448/PjfddFOn9eeffz6XXHJJjj322NTU1GT27Nn561//2mnPXXfdlZNOOiljx47N6aefnhUrVuz2awMAgJ7SpyJ+wIABqa+vz4MPPpg777wzzz77bObNm1dab2lpycyZMzNp0qQ89NBDufrqq3PdddflnnvuKe25/PLL09jYmCVLlmTZsmVpbW3NnDlzSuuPPvpo5s+fn3nz5uXhhx/O9OnTM2vWrDzzzDN79FoBAOCf1aci/tOf/nTGjBmTQYMG5fWvf32mT5+ehx56qLR+7733pn///jn//PNTVlaWo48+OlOnTs33vve9JC98Cn/HHXfkwgsvzP7775/hw4dnzpw5+clPflKK9B/84Ac54YQTUltbm8GDB2fq1Kk57LDDctttt/XKNQMAwK4a2NsDvJIVK1akqqqq9HVTU1PGjBmT/v3//3uP6urqLF68OEny29/+Nm1tbTnqqKNK66NGjcrQoUOzZs2ajBgxIk1NTTn11FM7fZ/q6uo0NTXt0myNjY3/zCW9ogkTJvT4OaG3PPLII709Qre599ibFOneS9x/7D329L3XZyP+7rvvzuLFi3PzzTeXjrW0tKSioqLTvsrKyrS0tJTWk3TZU1FR0WlPZWVll3OsW7dul+arrq5OWVnZLr0G9iX+YIbe4d6D3tHT915bW9srfmjcpx6n2WHp0qW59NJL09DQkCOPPLJ0vLy8vBTjO2zevDnl5eWl9SRpbm7utKe5ubnTnn9cf/E5AACgr+tzEb948eIsWLAg3/jGN/KOd7yj01pVVVVWr16d9vb20rFVq1aVHrk5+OCDU1ZW1uldy5NPPpktW7aU9lRVVXV5V/PicwAAQF/XpyL+pptuyle+8pXceOONL/lXEieddFK2b9+ehoaGbN26NY8//ngWL16cs846K0kyZMiQTJkyJddee202bNiQTZs2pb6+Pu95z3tywAEHJEnOPPPM/PjHP87999+fbdu25dZbb80TTzyRD37wg3v0WgEA4J/Vp56Jv+yyyzJw4MCcc845nY4vXbo0I0aMSHl5eRYuXJgFCxbkhhtuyOte97p88pOfzCmnnFLaO3/+/Fx22WWZPHlytm/fnne9611ZsGBBab2mpiaXXXZZLrvssjz77LM56KCD0tDQUIp8AADo6/pUxK9du3ane8aMGZPvf//7L7s+ZMiQfOELX8gXvvCFl91z6qmndvkNNQAAUBR96nEaAABg50Q8AAAUjIgHAICCEfEAAFAwIh4AAApGxAMAQMGIeAAAKBgRDwAABSPiAQCgYEQ8AAAUjIgHAICCEfEAAFAwIh4AAApGxAMAQMGIeAAAKBgRDwAABSPiAQCgYEQ8AAAUjIgHAICCEfEAAFAwIh4AAApGxAMAQMGIeAAAKBgRDwAABSPiAQCgYEQ8AAAUjIgHAICCEfEAAFAwIh4AAApGxAMAQMGIeAAAKBgRDwAABSPiAQCgYEQ8AAAUjIgHAICCEfEAAFAwIh4AAApGxAMAQMGIeAAAKBgRDwAABSPiAQCgYEQ8AAAUjIgHAICCEfEAAFAwIh4AAApGxAMAQMGIeAAAKBgRDwAABSPiAQCgYEQ8AAAUjIgHAICCEfEAAFAwIh4AAApGxAMAQMHskxHf3t6eq666KhMnTsz48eMzY8aMPP300709FgAAdMs+GfELFy7MXXfdlZtvvjnLly/PiBEjct5556W9vb23RwMAgJ0a2NsD9IZFixZl5syZOfTQQ5MkF198cSZOnJhHHnkkb3/721/xtR0dHUmSrVu37pbZXv/G/XbLeWFPamtr6+0Rdln5G4b39gjwqhXx3kuSwW+o7O0R4FXZHffejtbc0Z7/qF/Hy63spZqbm3PMMcdk8eLFGTt2bOn45MmT85GPfCTnnHPOTl//xBNP7O4xAQAgo0ePTkVFRZfj+9wn8S0tLUmSysrO7/orKipKa69k2LBhGT16dAYNGpR+/frtlhkBANi3dXR0ZNu2bRk2bNhLru9zEV9eXp7khU/UX6y5ubm09kr69+//ku+GAACgJw0ZMuRl1/a5H2ytqKjIAQcckMbGxtKx5ubmPPXUUzniiCN6cTIAAOiefS7ik6Suri433nhj1q1bl9bW1tTX1+fggw/OhAkTens0AADYqX3ucZokmTlzZpqbmzNt2rRs2bIlEyZMSENDQ/r33yff0wAAUDD73G+nAQCAovPRMwAAFIyIBwCAghHxAABQMCIeAAAKRsQDAEDBiHj2Ce3t7bnqqqsyceLEjB8/PjNmzMjTTz/d22PBXm/p0qWZNm1aampqcvjhh/f2OLDPqK+vz+TJk1NTU5NJkyZl/vz52bhxY2+PRQ8S8ewTFi5cmLvuuis333xzli9fnhEjRuS8885Le3t7b48Ge7XKyspMmzYt8+fP7+1RYJ8yYMCA1NfX58EHH8ydd96ZZ599NvPmzevtsehBfk88+4QTTjghM2fOzLRp05IkmzdvzsSJE/Nf//Vfefvb397L08He78EHH8w555yTtWvX9vYosE9atmxZPvOZz+TRRx/t7VHoIT6JZ6/X3Nycp59+OtXV1aVjlZWVOeigg7JmzZpenAwA9owVK1akqqqqt8egBw3s7QFgd2tpaUnyQri/WEVFRWkNAPZWd999dxYvXpybb765t0ehB/kknr1eeXl5khc+kX+x5ubm0hoA7I2WLl2aSy+9NA0NDTnyyCN7exx6kIhnr1dRUZEDDjggjY2NpWPNzc156qmncsQRR/TiZACw+yxevDgLFizIN77xjbzjHe/o7XHoYSKefUJdXV1uvPHGrFu3Lq2tramvr8/BBx+cCRMm9PZosFfbvn172trasm3btiRJW1tb2tra/GYo2M1uuummfOUrX8mNN97oz7q9lN9Owz6hvb09X/va13LLLbdky5YtmTBhQhYsWJCRI0f29miwV7vtttte8tfa3XTTTTnuuON6YSLYNxx++OEZOHBgBg8e3On40qVLM2LEiF6aip4k4gEAoGA8TgMAAAUj4gEAoGBEPAAAFIyIBwCAghHxAABQMCIeAAAKRsQD8JLmzp2bc889t9v7TzjhhFx//fWv+vv21HkA9mYiHgAACkbEAwBAwYh4AHZq1apVmTlzZt75zndm/Pjx+dCHPpSf/vSnXfa1tbXls5/9bGpqanLcccflqquuSnt7e2l927Zt+frXv54TTjghRx11VCZPnpxFixa94ve+7777MmXKlIwbNy7HHHNMPvzhD2f16tU9fo0ARTKwtwcAoO9raWnJBz7wgcyZMycDBw7MHXfckfPPPz9LlizJIYccUtr3ne98Jx/96Edzyy235PHHH8+///u/5/Wvf30++tGPJkk+97nPZdWqVfn85z+fgw46KL/61a9yySWXZMCAAZk6dWqX7/vcc89l9uzZufDCC/P+978/W7duzerVqzNgwIA9du0AfZGIB2CnjjvuuE5ff+pTn8qyZctyzz33ZNasWaXjVVVVufDCC5Mkhx56aH7zm9/kW9/6Vj760Y/m97//fe64444sXbo0o0aNSpIceOCB+c1vfpObb775ZSN+27ZtOeWUUzJy5MgkKb0WYF8m4gHYqb/85S+59tpr84tf/CJ/+tOfsn379rS1teWZZ57ptO/oo4/u9HVNTU1uuOGGtLS0pLGxMR0dHfnwhz/cac/f//73l/1k/fDDD8+kSZNy2mmnZeLEiTn22GNz0kkn5S1veUvPXiBAwYh4AHZq7ty5+cMf/pCLL744I0eOzJAhQ/KpT30q27Zt6/Y5Ojo6kiT//d//naFDh3Za69ev30u+ZsCAAVm4cGF+9atf5YEHHsi9996br371q7nmmmtSW1v7z18QQMGJeAB2auXKlbn44ovz3ve+N0nS2tqa9evXZ/To0Z32PfbYY52+/t///d/sv//+KS8vz5FHHpkk+cMf/rBLAd6vX7+MHTs2Y8eOzXnnnZcZM2bktttuE/HAPk3EA7BThxxySJYsWZIJEyakvb0911xzTbZv395l35o1a/L1r389p556ahobG3PTTTeVnpE/6KCD8qEPfSif+9znctFFF2X8+PHZsmVLGhsb85e//CUf//jHu5zv0UcfzYoVKzJp0qS88Y1vzO9+97usXbu2yyM5APsaEQ/ATl1xxRW59NJLM3Xq1LzhDW/IjBkz8vzzz3fZN3369DzzzDP50Ic+lEGDBuXss8/OOeecU1r/whe+kG9961v5xje+kfXr12fYsGE57LDDcvbZZ7/k962oqMgvf/nLfO9738umTZvyxje+MaeddlrOP//83XatAEXQr2PHQ4oAAEAh+I89AQBAwYh4AAAoGBEPAAAFI+IBAKBgRDwAABSMiAcAgIIR8QAAUDAiHgAACub/ARl43YPd4wjLAAAAAElFTkSuQmCC\n"
          },
          "metadata": {}
        }
      ],
      "source": [
        "\n",
        "sns.countplot(df.label)\n",
        "plt.xlabel('labels');\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "o66CJq3FCxpw"
      },
      "outputs": [],
      "source": [
        "def text_preprocessing(text):\n",
        "\n",
        "    import re\n",
        "\n",
        "    text = re.sub(r'\\\\(?!u)','',text)\n",
        "\n",
        "    # Remove em dash\n",
        "    text = re.sub(r'—|—', ' ', text)\n",
        "\n",
        "    # Remove all occurances of \"\n",
        "    text = re.sub(r'\\\"|“|”|…',\" \",text)\n",
        "   \n",
        "    # Remove emoticons and emojis\n",
        "\n",
        "    text = re.sub(r':P|:-P|😂|🤣',' joking ',text)\n",
        "\n",
        "    text = re.sub(r'<3|< 3|❤️|💖|😍|💕|😘|💕|💝|💗|💜|💙|🖤|💚|💛|🤍',' love ', text) \n",
        "\n",
        "    text = re.sub(r'[😁😃😄😆😊😸😺😊😀😋🥰☺️]','  Happy . ', text)\n",
        "    text = re.sub(r'(:\\s?\\)|:-\\)|\\(\\s?:|\\(-:|:\\'\\))','  Happy . ', text)\n",
        "    text = re.sub(r'(:\\s?D|:-?D|x-?D)','  Happy . ', text)\n",
        "\n",
        "\n",
        "    text = re.sub(r'[😥😣😓😔😕☹️🙁😖😞😟😢😭😩😵😡😿😫🤢😨😱]','  Sad . ', text)\n",
        "    text = re.sub(r'(:\\(|:-\\(|\\):|\\)-:)','  Sad . ', text)\n",
        "    text = re.sub(r'(:,\\(|:\\'\\(|:\"\\()','  Sad . ', text)\n",
        "\n",
        "    # Remove every pther emoji\n",
        "    text = re.sub('[\\U0001F600-\\U0001FFFF]',\" \", text)\n",
        "    text = re.sub('[\\U0001F300-\\U0001F5FF]',\" \", text)\n",
        "    \n",
        "    # Codecs\n",
        "    text = re.sub(r'\\\\(?!u)','',text)\n",
        "    import codecs\n",
        "    text = codecs.unicode_escape_decode(text)[0]\n",
        "\n",
        "    #remove @usermentions\n",
        "    text = re.sub(r'(@.*?)[\\s]', ' ', text)\n",
        "\n",
        "    # Remove URLs\n",
        "    text = re.sub(r'((https?://[^\\s]+)|(www\\.[^\\s]+))','',text)\n",
        "\n",
        "    # Replace '&amp;' with '&'\n",
        "    text = re.sub(r'&amp;', '&', text)\n",
        "    \n",
        "    # Remove hash (#) symbol\n",
        "    text = re.sub(r'#', ' ', text).strip()\n",
        "\n",
        "    # Remove trailing dots\n",
        "    text= re.sub(\"[.]+\", \". \",text)    \n",
        "\n",
        "    # Remove trailing whitespace\n",
        "    text = re.sub(r'\\s+', ' ', text).strip()\n",
        "\n",
        "    return text"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "text_preprocessing(\"@traitor_thot The novel coronavirus got her... 😔\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "5eyFC1XSpJG8",
        "outputId": "a3b0b659-04b6-4f94-9b46-85b9e2140d0a"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'The novel coronavirus got her. Sad .'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "text_preprocessing(\"#TaiwanCanHelp Feeling😱😰after reading this, 😷 \\\" As Wuhan Virus Spreads, Taiwan Has No Say at WHO. Chinese pressure keeps Taipei out of international organizations—putting everyone at more risk.\\\"😷 https://t.co/DRSW0fvbiU https://t.co/7JJi2ltIxl\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "Aw2xqxFhpRCM",
        "outputId": "48f815ea-7207-42a7-ed93-03676924bce7"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'TaiwanCanHelp Feeling Sad . after reading this, As Wuhan Virus Spreads, Taiwan Has No Say at WHO. Chinese pressure keeps Taipei out of international organizations putting everyone at more risk.'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "*Splitting* data to train, validation, and test"
      ],
      "metadata": {
        "id": "xLSqp--qCdfW"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "id": "Jqr0tgaqFppT"
      },
      "outputs": [],
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "X = df.text.values\n",
        "y = df.label.values\n",
        "\n",
        "# Train: 80% Test: 20%\n",
        "df_train, df_test = train_test_split(\n",
        "  df,\n",
        "  test_size=0.2,\n",
        "  random_state=42\n",
        ")\n",
        "\n",
        "# Test: 50% Validation: 50%\n",
        "df_val, df_test = train_test_split(\n",
        "  df_test,\n",
        "  test_size=0.5,\n",
        "  random_state=42\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "id": "kFPpgNf1Fcr5"
      },
      "outputs": [],
      "source": [
        "X_train = df_train['text'].to_numpy()\n",
        "X_val = df_val['text'].to_numpy()\n",
        "\n",
        "y_train = df_train['label'].to_numpy()\n",
        "y_val = df_val['label'].to_numpy()\n",
        "\n",
        "y_test = df_test['label'].to_numpy()"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Connecting to *GPU*"
      ],
      "metadata": {
        "id": "_xM5BfWJCo8c"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5trbTzqYLt30",
        "outputId": "c8775fd0-31d4-4452-b26b-61b0e294305d"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "There are 1 GPU(s) available.\n",
            "Device name: Tesla T4\n"
          ]
        }
      ],
      "source": [
        "import torch\n",
        "\n",
        "if torch.cuda.is_available():       \n",
        "    device = torch.device(\"cuda\")\n",
        "    print(f'There are {torch.cuda.device_count()} GPU(s) available.')\n",
        "    print('Device name:', torch.cuda.get_device_name(0))\n",
        "\n",
        "else:\n",
        "    print('No GPU available, using the CPU instead.')\n",
        "    device = torch.device(\"cpu\")"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### BERT"
      ],
      "metadata": {
        "id": "t_L8sYWsCu6B"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Tokenizer"
      ],
      "metadata": {
        "id": "HRNKYV9zC75j"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "id": "qrdEXyEEDR2t",
        "outputId": "1df9dfd9-6479-4f6e-a883-1871c6e6b6a1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 113,
          "referenced_widgets": [
            "a8a062ed231e4a0daf3f41c470c23b3b",
            "1069a16a29f04d56951b7a3d01150707",
            "e465bdc6d7b346cb8522acbeff9cefd3",
            "e4bc0477876342faac0df63f9fa303ce",
            "b429ee056980477ab621c2ae950aca7f",
            "00372b91338b4dbbbd231fb02fb61fa1",
            "765632c58ee44919a89bd2a542cb80da",
            "f9098bcf9e774f26894e9d2a672c4a3c",
            "c7e928052f2f47a1b83026e6fdcb224a",
            "330820ecf7c34b128f9b2e83a8412e2a",
            "9f7dc654b9cb4799926c5a93c781159f",
            "1d0287541cb54fc993db00f9b987fbc3",
            "46e59d7920fc406b9f3cbcff6ba5e141",
            "0509afd5dd2b461ea2e9f827a69d529a",
            "36931e3d18df40e4bcbbeb9998b0884b",
            "349f0f7a50694aa3a4417bc721738750",
            "527db8917b9244b08c4a351c5aeddcec",
            "91ae3fcd960f4e178ea31183bbc0fc77",
            "d6226210893942628dcd773c47b28fbf",
            "f44316f1b50c447f8b70492acb049e1e",
            "60c8b82b83504d2495a21bd6cf1fe403",
            "3adad464c0d4424cba51936e8b61fb17",
            "20c081492ba74d22895f3f20d9135451",
            "79240864f44549cd90365eba5006613f",
            "933e00a9ac5d4c39a3fd18020e0790e1",
            "0877fc1a21694589bd38230e3db012fb",
            "6ef711ca8e3d4815886045fe85338be2",
            "82513b3e807b44b5851f6a478d1369e6",
            "a86fe9a63b18453cb975bef570150acc",
            "be072f8e70b346a4ac6b40ed71034522",
            "fc6da1d47e2b43c08bde6bee6aa517ad",
            "47e3880d747f44bb9effddb716b0764c",
            "42e10351fad546ca9596c892e1d9e044"
          ]
        }
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/208k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "a8a062ed231e4a0daf3f41c470c23b3b"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/29.0 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "1d0287541cb54fc993db00f9b987fbc3"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/570 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "20c081492ba74d22895f3f20d9135451"
            }
          },
          "metadata": {}
        }
      ],
      "source": [
        "# Load the BERT tokenizer\n",
        "\n",
        "from transformers import BertTokenizer\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-cased')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pacmJRKQH0ki",
        "outputId": "995b0693-5984-425e-fb04-f71802e66fcd"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Max length:  79\n"
          ]
        }
      ],
      "source": [
        "# Concatenate all train data and test data\n",
        "all_tweets = df.text.values\n",
        "\n",
        "# Encode our concatenated data\n",
        "encoded_tweets = [tokenizer.encode(text_preprocessing(sent), add_special_tokens=True) for sent in all_tweets]\n",
        "\n",
        "# Find the maximum length\n",
        "\n",
        "max_len = max([len(sent) for sent in encoded_tweets])\n",
        "print('Max length: ', max_len)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "id": "W9RrWs1hH_fZ"
      },
      "outputs": [],
      "source": [
        "# Specify max length to trancuate/pad to\n",
        "MAX_LEN = 85 \n",
        "\n",
        "# Create a function to tokenize a set of texts\n",
        "def preprocessing_for_bert(data):\n",
        "    \"\"\"Preprocessing steps for pretrained BERT.\n",
        "    @param    data (np.array): Array of texts to be processed.\n",
        "    @return   input_ids (torch.Tensor): Tensor of token ids to be fed to a model.\n",
        "    @return   attention_masks (torch.Tensor): Tensor of indices specifying which\n",
        "                  tokens should be attended to by the model.\n",
        "    \"\"\"\n",
        "    # Create empty lists to store outputs\n",
        "    input_ids = []\n",
        "    attention_masks = []\n",
        "\n",
        "    # For every sentence...\n",
        "    for sent in data:\n",
        "\n",
        "        # encode_plus will:\n",
        "        #    (1) Tokenize the sentence\n",
        "        #    (2) Add the `[CLS]` and `[SEP]` token to the start and end\n",
        "        #    (3) Truncate/Pad sentence to max length\n",
        "        #    (4) Map tokens to their IDs\n",
        "        #    (5) Create attention mask\n",
        "        #    (6) Return a dictionary of outputs\n",
        "        encoded_sent = tokenizer.encode_plus(\n",
        "            text_preprocessing(r\"\"+sent),   # Preprocess sentence\n",
        "            add_special_tokens=True,        # Add `[CLS]` and `[SEP]`\n",
        "            max_length=MAX_LEN,             # Max length to truncate/pad\n",
        "            pad_to_max_length=True,         # Pad sentence to max length\n",
        "            return_attention_mask=True,     # Return attention mask\n",
        "            \n",
        "            )\n",
        "        \n",
        "        # Add the outputs to the lists\n",
        "        input_ids.append(encoded_sent.get('input_ids'))\n",
        "        attention_masks.append(encoded_sent.get('attention_mask'))\n",
        "\n",
        "    # Convert lists to tensors\n",
        "    input_ids = torch.tensor(input_ids)\n",
        "    attention_masks = torch.tensor(attention_masks)\n",
        "\n",
        "    return input_ids, attention_masks"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ot0WxbeVLzXR",
        "outputId": "ea28801a-2a96-481b-bc36-ad60f03b27a1"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Tokenizing data...\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2307: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ]
        }
      ],
      "source": [
        "# Run function `preprocessing_for_bert` on the data\n",
        "\n",
        "print('Tokenizing data...')\n",
        "\n",
        "train_inputs, train_masks = preprocessing_for_bert(X_train)\n",
        "val_inputs, val_masks = preprocessing_for_bert(X_val)\n",
        "test_inputs, test_masks = preprocessing_for_bert(df_test.text)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "PyTorch Dataloader"
      ],
      "metadata": {
        "id": "gupprHl7DBXA"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {
        "id": "pB_JYUQN1Mjt"
      },
      "outputs": [],
      "source": [
        "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
        "\n",
        "# Convert other data types to torch.Tensor\n",
        "\n",
        "train_labels = torch.tensor(y_train)\n",
        "val_labels = torch.tensor(y_val)\n",
        "test_labels = torch.tensor(y_test)\n",
        "\n",
        "# For fine-tuning BERT, the authors recommend a batch size of 16 or 32.\n",
        "batch_size = 16\n",
        "\n",
        "# Create the DataLoader for our training set\n",
        "train_data = TensorDataset(train_inputs, train_masks, train_labels)\n",
        "train_dataloader = DataLoader(train_data, batch_size=batch_size)\n",
        "\n",
        "# Create the DataLoader for our validation set\n",
        "val_data = TensorDataset(val_inputs, val_masks, val_labels)\n",
        "val_dataloader = DataLoader(val_data, batch_size=batch_size)\n",
        "\n",
        "# Create the DataLoader for our test set\n",
        "test_dataset = TensorDataset(test_inputs, test_masks,test_labels)\n",
        "test_sampler = SequentialSampler(test_dataset)\n",
        "test_dataloader = DataLoader(test_dataset, sampler=test_sampler, batch_size=batch_size)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "BERT Classifier architecture"
      ],
      "metadata": {
        "id": "rUgDI1KWDEKx"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xgph5Ja54Xrw",
        "outputId": "fa556242-60cc-4d03-f46f-e360f3fe71dd"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "CPU times: user 25.3 ms, sys: 1.01 ms, total: 26.3 ms\n",
            "Wall time: 28 ms\n"
          ]
        }
      ],
      "source": [
        "%%time\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "from transformers import BertModel\n",
        "\n",
        "# Create the BertClassfier class\n",
        "\n",
        "# BERT-base consists of 12 transformer layers, each transformer layer takes in a list of token embeddings, and produces the same number of embeddings with the same hidden size on the output. \n",
        "# The output of the final transformer layer of the [CLS] token is used as the features of the sequence to feed a classifier.\n",
        "# We create a single-hidden-layer feed-forward neural network with 0.3 dropout as our classifier.\n",
        "\n",
        "class BertClassifier(nn.Module):\n",
        "    \"\"\"Bert Model for Classification Tasks.\n",
        "    \"\"\"\n",
        "    def __init__(self, freeze_bert=False):\n",
        "        \"\"\"\n",
        "        @param    bert: a BertModel object\n",
        "        @param    classifier: a torch.nn.Module classifier\n",
        "        @param    freeze_bert (bool): Set `False` to fine-tune the BERT model\n",
        "        \"\"\"\n",
        "        super(BertClassifier, self).__init__()\n",
        "        # Specify hidden size of BERT, hidden size of our classifier, and number of labels\n",
        "        D_in, H, D_out = 768, 50, 3\n",
        "\n",
        "        # Instantiate BERT model\n",
        "        self.bert = BertModel.from_pretrained('bert-base-cased',return_dict=False)\n",
        "\n",
        "        # Instantiate an one-layer feed-forward classifier\n",
        "        self.classifier = nn.Sequential(\n",
        "            nn.Linear(D_in, H),\n",
        "            nn.ReLU(),\n",
        "            nn.Dropout(0.3),\n",
        "            nn.Linear(H, D_out)\n",
        "        )\n",
        "\n",
        "\n",
        "    def forward(self, input_ids, attention_mask):\n",
        "        \"\"\"\n",
        "        Feed input to BERT and the classifier to compute logits.\n",
        "        @param    input_ids (torch.Tensor): an input tensor with shape (batch_size,\n",
        "                      max_length)\n",
        "        @param    attention_mask (torch.Tensor): a tensor that hold attention mask\n",
        "                      information with shape (batch_size, max_length)\n",
        "        @return   logits (torch.Tensor): an output tensor with shape (batch_size,\n",
        "                      num_labels)\n",
        "        \"\"\"\n",
        "        # Feed input to BERT\n",
        "        outputs = self.bert(\n",
        "            input_ids=input_ids,\n",
        "            attention_mask=attention_mask,\n",
        "            return_dict=False)\n",
        "        \n",
        "        # Extract the last hidden state of the token `[CLS]` for classification task\n",
        "        last_hidden_state_cls = outputs[0][:, 0, :]\n",
        "\n",
        "        # Feed input to classifier to compute logits\n",
        "        logits = self.classifier(last_hidden_state_cls)\n",
        "\n",
        "        return logits"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Optimizer & Learning Rate Scheduler"
      ],
      "metadata": {
        "id": "RYRGQ48BDOP-"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 26,
      "metadata": {
        "id": "WrrdkNlB41GX"
      },
      "outputs": [],
      "source": [
        "from transformers import AdamW, get_linear_schedule_with_warmup\n",
        "\n",
        "def initialize_model(epochs):\n",
        "    \"\"\"Initialize the Bert Classifier, the optimizer and the learning rate scheduler.\n",
        "    \"\"\"\n",
        "    # Instantiate Bert Classifier\n",
        "    bert_classifier = BertClassifier(freeze_bert=False)\n",
        "\n",
        "    # Tell PyTorch to run the model on GPU\n",
        "    bert_classifier.to(device)\n",
        "\n",
        "    # Create the optimizer\n",
        "    optimizer = AdamW(bert_classifier.parameters(),\n",
        "                      lr=1e-5,    # BERT paper recommends 2e-5 or 3e-5 or 5e-5, but we tested with 1e-5 as well\n",
        "                      eps=1e-8 \n",
        "                      )\n",
        "\n",
        "    # Total number of training steps\n",
        "    total_steps = len(train_dataloader) * epochs\n",
        "\n",
        "    # Set up the learning rate scheduler\n",
        "    scheduler = get_linear_schedule_with_warmup(optimizer,\n",
        "                                                num_warmup_steps=0, # Default value\n",
        "                                                num_training_steps=total_steps)\n",
        "    return bert_classifier, optimizer, scheduler"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Training Loop"
      ],
      "metadata": {
        "id": "pm_lXVxzDXIv"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "epoch_number = 3"
      ],
      "metadata": {
        "id": "XCj5PWHnDV3M"
      },
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 28,
      "metadata": {
        "id": "ZX91ABjv5EwZ"
      },
      "outputs": [],
      "source": [
        "import random\n",
        "import time\n",
        "\n",
        "# Specify loss function\n",
        "loss_fn = nn.CrossEntropyLoss()\n",
        "\n",
        "def set_seed(seed_value=42):\n",
        "    \"\"\"Set seed for reproducibility.\n",
        "    \"\"\"\n",
        "    random.seed(seed_value)\n",
        "    np.random.seed(seed_value)\n",
        "    torch.manual_seed(seed_value)\n",
        "    torch.cuda.manual_seed_all(seed_value)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Training:\n",
        "\n",
        "* Unpack our data from the dataloader and load the data onto the GPU\n",
        "* Zero out gradients calculated in the previous pass\n",
        "* Perform a forward pass to compute logits and loss\n",
        "* Perform a backward pass to compute gradients (loss.backward())\n",
        "* Clip the norm of the gradients to 1.0 to prevent “exploding gradients”\n",
        "* Update the model’s parameters (optimizer.step())\n",
        "* Update the learning rate (scheduler.step())"
      ],
      "metadata": {
        "id": "WT7ljYZ_Dd8a"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "def train(model, train_dataloader, val_dataloader=None, epochs=4, evaluation=False):\n",
        "    \"\"\"Train the BertClassifier model.    \"\"\"\n",
        "    # Start training loop\n",
        "    print(\"Start training...\\n\")\n",
        "    for epoch_i in range(epochs):\n",
        "        #               Training\n",
        "\n",
        "        # Print the header of the result table\n",
        "        print(f\"{'Epoch':^7} | {'Batch':^7} | {'Train Loss':^12} | {'Val Loss':^10} | {'Val Acc':^9} | {'Elapsed':^9}\")\n",
        "        print(\"-\"*70)\n",
        "\n",
        "        # Measure the elapsed time of each epoch\n",
        "        t0_epoch, t0_batch = time.time(), time.time()\n",
        "\n",
        "        # Reset tracking variables at the beginning of each epoch\n",
        "        total_loss, batch_loss, batch_counts = 0, 0, 0\n",
        "\n",
        "        # Put the model into the training mode\n",
        "        model.train()\n",
        "\n",
        "        # For each batch of training data...\n",
        "        for step, batch in enumerate(train_dataloader):\n",
        "            batch_counts +=1\n",
        "            # Load batch to GPU\n",
        "            b_input_ids, b_attn_mask, b_labels = tuple(t.to(device) for t in batch)\n",
        "\n",
        "            # Zero out any previously calculated gradients\n",
        "            model.zero_grad()\n",
        "\n",
        "            # Perform a forward pass. This will return logits.\n",
        "            logits = model(b_input_ids, b_attn_mask)\n",
        "\n",
        "            # Compute loss and accumulate the loss values\n",
        "            loss = loss_fn(logits, b_labels)\n",
        "            batch_loss += loss.item()\n",
        "            total_loss += loss.item()\n",
        "\n",
        "            # Perform a backward pass to calculate gradients\n",
        "            loss.backward()\n",
        "\n",
        "            # Clip the norm of the gradients to 1.0 to prevent \"exploding gradients\"\n",
        "            torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
        "\n",
        "            # Update parameters and the learning rate\n",
        "            optimizer.step()\n",
        "            scheduler.step()\n",
        "\n",
        "            # Print the loss values and time elapsed for every 20 batches\n",
        "            if (step % 20 == 0 and step != 0) or (step == len(train_dataloader) - 1):\n",
        "                # Calculate time elapsed for 20 batches\n",
        "                time_elapsed = time.time() - t0_batch\n",
        "\n",
        "                # Print training results\n",
        "                print(f\"{epoch_i + 1:^7} | {step:^7} | {batch_loss / batch_counts:^12.6f} | {'-':^10} | {'-':^9} | {time_elapsed:^9.2f}\")\n",
        "\n",
        "                # Reset batch tracking variables\n",
        "                batch_loss, batch_counts = 0, 0\n",
        "                t0_batch = time.time()\n",
        "\n",
        "        # Calculate the average loss over the entire training data\n",
        "        avg_train_loss = total_loss / len(train_dataloader)\n",
        "\n",
        "        print(\"-\"*70)\n",
        "        \n",
        "        #               Evaluation\n",
        "        if evaluation == True:\n",
        "            # After the completion of each training epoch, measure the model's performance\n",
        "            # on our validation set.\n",
        "            val_loss, val_accuracy = evaluate(model, val_dataloader)\n",
        "\n",
        "            # Print performance over the entire training data\n",
        "            time_elapsed = time.time() - t0_epoch\n",
        "            \n",
        "            print(f\"{epoch_i + 1:^7} | {'-':^7} | {avg_train_loss:^12.6f} | {val_loss:^10.6f} | {val_accuracy:^9.2f} | {time_elapsed:^9.2f}\")\n",
        "            print(\"-\"*70)\n",
        "        print(\"\\n\")\n",
        "    \n",
        "    print(\"Training complete!\")\n"
      ],
      "metadata": {
        "id": "MyvEEBzUUrbL"
      },
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Evaluation:\n",
        "\n",
        "* Unpack our data and load onto the GPU\n",
        "* Forward pass\n",
        "* Compute loss and accuracy rate over the validation set"
      ],
      "metadata": {
        "id": "_oNqq5bvDpxA"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "def evaluate(model, val_dataloader):\n",
        "    \"\"\"After the completion of each training epoch, measure the model's performance\n",
        "    on our validation set.\n",
        "    \"\"\"\n",
        "    # Put the model into the evaluation mode. The dropout layers are disabled during\n",
        "    # the test time.\n",
        "    model.eval()\n",
        "\n",
        "    # Tracking variables\n",
        "    val_accuracy = []\n",
        "    val_loss = []\n",
        "\n",
        "    # For each batch in our validation set...\n",
        "    for batch in val_dataloader:\n",
        "        # Load batch to GPU\n",
        "        b_input_ids, b_attn_mask, b_labels = tuple(t.to(device) for t in batch)\n",
        "\n",
        "        # Compute logits\n",
        "        with torch.no_grad():\n",
        "            logits = model(b_input_ids, b_attn_mask)\n",
        "\n",
        "        # Compute loss\n",
        "        loss = loss_fn(logits, b_labels)\n",
        "        val_loss.append(loss.item())\n",
        "\n",
        "        # Get the predictions\n",
        "        preds = torch.argmax(logits, dim=1).flatten()\n",
        "\n",
        "        # Calculate the accuracy rate\n",
        "        accuracy = (preds == b_labels).cpu().numpy().mean() * 100\n",
        "        val_accuracy.append(accuracy)\n",
        "\n",
        "    # Compute the average accuracy and loss over the validation set.\n",
        "    val_loss = np.mean(val_loss)\n",
        "    val_accuracy = np.mean(val_accuracy)\n",
        "\n",
        "    return val_loss, val_accuracy"
      ],
      "metadata": {
        "id": "ToSOYw0B4tqh"
      },
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### **Training** ( fine-tuning )"
      ],
      "metadata": {
        "id": "7f5AAk7iDvbE"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 31,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 160,
          "referenced_widgets": [
            "27a2d3884820459e95fe058b89a7d5ea",
            "1d24d87ad6864e66a7a1106ba2ab6390",
            "3d3f45fbf29948bb86a4294d09b57786",
            "fe3e0409926748d383f12b4c2a865a5c",
            "6709b1766d5b481890423c42486ecf08",
            "a976d3c3cf184716b9d7336116496991",
            "b965186f58d040f097eba6654ebad146",
            "0f7ee05a92814c0d9cdc58450d89c268",
            "5d51d3a566ed4f69a7201cf37b96567c",
            "79969b2ef8bc49a68f8d4506428d91f8",
            "a124bc03d6d6427093710457045f33a9"
          ]
        },
        "id": "1IfwTg1L5G-q",
        "outputId": "50ea9c26-e197-4dd3-e6de-2d108eaec140"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/416M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "27a2d3884820459e95fe058b89a7d5ea"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the model checkpoint at bert-base-cased were not used when initializing BertModel: ['cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.predictions.transform.dense.weight', 'cls.predictions.bias', 'cls.predictions.transform.dense.bias']\n",
            "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n"
          ]
        }
      ],
      "source": [
        "set_seed()    # Set seed for reproducibility - default: 42\n",
        "bert_classifier, optimizer, scheduler = initialize_model(epochs=epoch_number)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "whzpTMsb5X56",
        "outputId": "188fe97f-9b20-4836-c91c-8659fce7b6d5"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Start training...\n",
            "\n",
            " Epoch  |  Batch  |  Train Loss  |  Val Loss  |  Val Acc  |  Elapsed \n",
            "----------------------------------------------------------------------\n",
            "   1    |   20    |   1.099823   |     -      |     -     |   5.41   \n",
            "   1    |   40    |   1.079144   |     -      |     -     |   5.17   \n",
            "   1    |   60    |   1.048377   |     -      |     -     |   5.24   \n",
            "   1    |   80    |   1.025588   |     -      |     -     |   5.26   \n",
            "   1    |   100   |   0.939539   |     -      |     -     |   5.32   \n",
            "   1    |   120   |   0.869716   |     -      |     -     |   5.38   \n",
            "   1    |   140   |   0.847265   |     -      |     -     |   5.42   \n",
            "   1    |   160   |   0.844194   |     -      |     -     |   5.43   \n",
            "   1    |   180   |   0.874926   |     -      |     -     |   5.41   \n",
            "   1    |   200   |   0.791333   |     -      |     -     |   5.36   \n",
            "   1    |   220   |   0.816237   |     -      |     -     |   5.31   \n",
            "   1    |   240   |   0.788080   |     -      |     -     |   5.28   \n",
            "   1    |   260   |   0.767887   |     -      |     -     |   5.25   \n",
            "   1    |   280   |   0.770972   |     -      |     -     |   5.24   \n",
            "   1    |   300   |   0.783620   |     -      |     -     |   5.23   \n",
            "   1    |   320   |   0.700860   |     -      |     -     |   5.22   \n",
            "   1    |   340   |   0.691500   |     -      |     -     |   5.24   \n",
            "   1    |   360   |   0.698979   |     -      |     -     |   5.26   \n",
            "   1    |   380   |   0.772991   |     -      |     -     |   5.28   \n",
            "   1    |   400   |   0.751382   |     -      |     -     |   5.30   \n",
            "   1    |   420   |   0.801939   |     -      |     -     |   5.31   \n",
            "   1    |   440   |   0.782262   |     -      |     -     |   5.32   \n",
            "   1    |   460   |   0.715327   |     -      |     -     |   5.32   \n",
            "   1    |   480   |   0.762334   |     -      |     -     |   5.32   \n",
            "   1    |   500   |   0.756834   |     -      |     -     |   5.31   \n",
            "   1    |   520   |   0.682734   |     -      |     -     |   5.30   \n",
            "   1    |   540   |   0.750441   |     -      |     -     |   5.30   \n",
            "   1    |   560   |   0.740409   |     -      |     -     |   5.28   \n",
            "   1    |   580   |   0.678739   |     -      |     -     |   5.28   \n",
            "   1    |   600   |   0.722739   |     -      |     -     |   5.29   \n",
            "   1    |   620   |   0.714764   |     -      |     -     |   5.29   \n",
            "   1    |   640   |   0.693209   |     -      |     -     |   5.29   \n",
            "   1    |   660   |   0.756850   |     -      |     -     |   5.29   \n",
            "   1    |   680   |   0.709908   |     -      |     -     |   5.27   \n",
            "   1    |   700   |   0.657446   |     -      |     -     |   5.29   \n",
            "   1    |   720   |   0.764854   |     -      |     -     |   5.29   \n",
            "   1    |   740   |   0.670649   |     -      |     -     |   5.31   \n",
            "   1    |   760   |   0.720783   |     -      |     -     |   5.30   \n",
            "   1    |   780   |   0.619988   |     -      |     -     |   5.30   \n",
            "   1    |   800   |   0.792447   |     -      |     -     |   5.31   \n",
            "   1    |   820   |   0.707645   |     -      |     -     |   5.30   \n",
            "   1    |   840   |   0.702080   |     -      |     -     |   5.30   \n",
            "   1    |   860   |   0.696957   |     -      |     -     |   5.31   \n",
            "   1    |   880   |   0.693656   |     -      |     -     |   5.29   \n",
            "   1    |   900   |   0.714244   |     -      |     -     |   5.29   \n",
            "   1    |   920   |   0.711043   |     -      |     -     |   5.29   \n",
            "   1    |   940   |   0.708248   |     -      |     -     |   5.28   \n",
            "   1    |   960   |   0.694313   |     -      |     -     |   5.28   \n",
            "   1    |   980   |   0.635022   |     -      |     -     |   5.28   \n",
            "   1    |  1000   |   0.741526   |     -      |     -     |   5.27   \n",
            "   1    |  1020   |   0.685568   |     -      |     -     |   5.28   \n",
            "   1    |  1040   |   0.739290   |     -      |     -     |   5.28   \n",
            "   1    |  1060   |   0.690347   |     -      |     -     |   5.28   \n",
            "   1    |  1080   |   0.673410   |     -      |     -     |   5.28   \n",
            "   1    |  1100   |   0.683365   |     -      |     -     |   5.28   \n",
            "   1    |  1120   |   0.649623   |     -      |     -     |   5.29   \n",
            "   1    |  1140   |   0.783501   |     -      |     -     |   5.29   \n",
            "   1    |  1160   |   0.672488   |     -      |     -     |   5.29   \n",
            "   1    |  1180   |   0.737944   |     -      |     -     |   5.28   \n",
            "   1    |  1200   |   0.602169   |     -      |     -     |   5.29   \n",
            "   1    |  1220   |   0.681209   |     -      |     -     |   5.28   \n",
            "   1    |  1240   |   0.689520   |     -      |     -     |   5.28   \n",
            "   1    |  1260   |   0.637349   |     -      |     -     |   5.28   \n",
            "   1    |  1280   |   0.651936   |     -      |     -     |   5.28   \n",
            "   1    |  1300   |   0.688784   |     -      |     -     |   5.28   \n",
            "   1    |  1320   |   0.654172   |     -      |     -     |   5.28   \n",
            "   1    |  1340   |   0.713363   |     -      |     -     |   5.29   \n",
            "   1    |  1360   |   0.615058   |     -      |     -     |   5.29   \n",
            "   1    |  1380   |   0.739740   |     -      |     -     |   5.29   \n",
            "   1    |  1400   |   0.694336   |     -      |     -     |   5.30   \n",
            "   1    |  1420   |   0.630240   |     -      |     -     |   5.30   \n",
            "   1    |  1440   |   0.700535   |     -      |     -     |   5.28   \n",
            "   1    |  1460   |   0.711524   |     -      |     -     |   5.29   \n",
            "   1    |  1480   |   0.718062   |     -      |     -     |   5.30   \n",
            "   1    |  1500   |   0.710090   |     -      |     -     |   5.29   \n",
            "   1    |  1520   |   0.658110   |     -      |     -     |   5.28   \n",
            "   1    |  1540   |   0.804950   |     -      |     -     |   5.31   \n",
            "   1    |  1560   |   0.649018   |     -      |     -     |   5.29   \n",
            "   1    |  1580   |   0.683138   |     -      |     -     |   5.29   \n",
            "   1    |  1600   |   0.674120   |     -      |     -     |   5.29   \n",
            "   1    |  1620   |   0.660864   |     -      |     -     |   5.28   \n",
            "   1    |  1640   |   0.694012   |     -      |     -     |   5.29   \n",
            "   1    |  1660   |   0.664395   |     -      |     -     |   5.29   \n",
            "   1    |  1680   |   0.700285   |     -      |     -     |   5.30   \n",
            "   1    |  1700   |   0.719539   |     -      |     -     |   5.29   \n",
            "   1    |  1720   |   0.651954   |     -      |     -     |   5.30   \n",
            "   1    |  1740   |   0.624242   |     -      |     -     |   5.28   \n",
            "   1    |  1760   |   0.675462   |     -      |     -     |   5.28   \n",
            "   1    |  1780   |   0.628066   |     -      |     -     |   5.14   \n",
            "----------------------------------------------------------------------\n",
            "   1    |    -    |   0.731708   |  0.618659  |   72.66   |  491.16  \n",
            "----------------------------------------------------------------------\n",
            "\n",
            "\n",
            " Epoch  |  Batch  |  Train Loss  |  Val Loss  |  Val Acc  |  Elapsed \n",
            "----------------------------------------------------------------------\n",
            "   2    |   20    |   0.650271   |     -      |     -     |   5.53   \n",
            "   2    |   40    |   0.629448   |     -      |     -     |   5.29   \n",
            "   2    |   60    |   0.601440   |     -      |     -     |   5.29   \n",
            "   2    |   80    |   0.669895   |     -      |     -     |   5.30   \n",
            "   2    |   100   |   0.569060   |     -      |     -     |   5.29   \n",
            "   2    |   120   |   0.614257   |     -      |     -     |   5.29   \n",
            "   2    |   140   |   0.602587   |     -      |     -     |   5.28   \n",
            "   2    |   160   |   0.623436   |     -      |     -     |   5.30   \n",
            "   2    |   180   |   0.625287   |     -      |     -     |   5.29   \n",
            "   2    |   200   |   0.584293   |     -      |     -     |   5.29   \n",
            "   2    |   220   |   0.595241   |     -      |     -     |   5.28   \n",
            "   2    |   240   |   0.561366   |     -      |     -     |   5.29   \n",
            "   2    |   260   |   0.575508   |     -      |     -     |   5.29   \n",
            "   2    |   280   |   0.540665   |     -      |     -     |   5.29   \n",
            "   2    |   300   |   0.565529   |     -      |     -     |   5.29   \n",
            "   2    |   320   |   0.512541   |     -      |     -     |   5.29   \n",
            "   2    |   340   |   0.512776   |     -      |     -     |   5.27   \n",
            "   2    |   360   |   0.593584   |     -      |     -     |   5.28   \n",
            "   2    |   380   |   0.585119   |     -      |     -     |   5.29   \n",
            "   2    |   400   |   0.620202   |     -      |     -     |   5.28   \n",
            "   2    |   420   |   0.607738   |     -      |     -     |   5.27   \n",
            "   2    |   440   |   0.621791   |     -      |     -     |   5.29   \n",
            "   2    |   460   |   0.549086   |     -      |     -     |   5.28   \n",
            "   2    |   480   |   0.610700   |     -      |     -     |   5.28   \n",
            "   2    |   500   |   0.561470   |     -      |     -     |   5.29   \n",
            "   2    |   520   |   0.490874   |     -      |     -     |   5.28   \n",
            "   2    |   540   |   0.620191   |     -      |     -     |   5.29   \n",
            "   2    |   560   |   0.569207   |     -      |     -     |   5.29   \n",
            "   2    |   580   |   0.538535   |     -      |     -     |   5.28   \n",
            "   2    |   600   |   0.536423   |     -      |     -     |   5.28   \n",
            "   2    |   620   |   0.553486   |     -      |     -     |   5.28   \n",
            "   2    |   640   |   0.542300   |     -      |     -     |   5.28   \n",
            "   2    |   660   |   0.567112   |     -      |     -     |   5.29   \n",
            "   2    |   680   |   0.538856   |     -      |     -     |   5.28   \n",
            "   2    |   700   |   0.489088   |     -      |     -     |   5.27   \n",
            "   2    |   720   |   0.596920   |     -      |     -     |   5.28   \n",
            "   2    |   740   |   0.500119   |     -      |     -     |   5.29   \n",
            "   2    |   760   |   0.527850   |     -      |     -     |   5.28   \n",
            "   2    |   780   |   0.470703   |     -      |     -     |   5.30   \n",
            "   2    |   800   |   0.630774   |     -      |     -     |   5.29   \n",
            "   2    |   820   |   0.564592   |     -      |     -     |   5.29   \n",
            "   2    |   840   |   0.517859   |     -      |     -     |   5.28   \n",
            "   2    |   860   |   0.561119   |     -      |     -     |   5.27   \n",
            "   2    |   880   |   0.538713   |     -      |     -     |   5.28   \n",
            "   2    |   900   |   0.607701   |     -      |     -     |   5.29   \n",
            "   2    |   920   |   0.549761   |     -      |     -     |   5.28   \n",
            "   2    |   940   |   0.592718   |     -      |     -     |   5.28   \n",
            "   2    |   960   |   0.563710   |     -      |     -     |   5.28   \n",
            "   2    |   980   |   0.519742   |     -      |     -     |   5.28   \n",
            "   2    |  1000   |   0.592857   |     -      |     -     |   5.28   \n",
            "   2    |  1020   |   0.581372   |     -      |     -     |   5.28   \n",
            "   2    |  1040   |   0.601433   |     -      |     -     |   5.29   \n",
            "   2    |  1060   |   0.555171   |     -      |     -     |   5.29   \n",
            "   2    |  1080   |   0.530759   |     -      |     -     |   5.29   \n",
            "   2    |  1100   |   0.504630   |     -      |     -     |   5.28   \n",
            "   2    |  1120   |   0.557084   |     -      |     -     |   5.28   \n",
            "   2    |  1140   |   0.647891   |     -      |     -     |   5.28   \n",
            "   2    |  1160   |   0.519627   |     -      |     -     |   5.28   \n",
            "   2    |  1180   |   0.601228   |     -      |     -     |   5.28   \n",
            "   2    |  1200   |   0.504286   |     -      |     -     |   5.29   \n",
            "   2    |  1220   |   0.516182   |     -      |     -     |   5.28   \n",
            "   2    |  1240   |   0.521735   |     -      |     -     |   5.28   \n",
            "   2    |  1260   |   0.518359   |     -      |     -     |   5.28   \n",
            "   2    |  1280   |   0.502718   |     -      |     -     |   5.28   \n",
            "   2    |  1300   |   0.538853   |     -      |     -     |   5.29   \n",
            "   2    |  1320   |   0.517876   |     -      |     -     |   5.29   \n",
            "   2    |  1340   |   0.597043   |     -      |     -     |   5.29   \n",
            "   2    |  1360   |   0.514981   |     -      |     -     |   5.30   \n",
            "   2    |  1380   |   0.576428   |     -      |     -     |   5.28   \n",
            "   2    |  1400   |   0.581045   |     -      |     -     |   5.28   \n",
            "   2    |  1420   |   0.508520   |     -      |     -     |   5.28   \n",
            "   2    |  1440   |   0.549660   |     -      |     -     |   5.30   \n",
            "   2    |  1460   |   0.604823   |     -      |     -     |   5.29   \n",
            "   2    |  1480   |   0.531752   |     -      |     -     |   5.30   \n",
            "   2    |  1500   |   0.564350   |     -      |     -     |   5.29   \n",
            "   2    |  1520   |   0.531104   |     -      |     -     |   5.28   \n",
            "   2    |  1540   |   0.702642   |     -      |     -     |   5.30   \n",
            "   2    |  1560   |   0.519732   |     -      |     -     |   5.28   \n",
            "   2    |  1580   |   0.564932   |     -      |     -     |   5.29   \n",
            "   2    |  1600   |   0.528925   |     -      |     -     |   5.30   \n",
            "   2    |  1620   |   0.497971   |     -      |     -     |   5.28   \n",
            "   2    |  1640   |   0.568314   |     -      |     -     |   5.30   \n",
            "   2    |  1660   |   0.570371   |     -      |     -     |   5.30   \n",
            "   2    |  1680   |   0.557853   |     -      |     -     |   5.30   \n",
            "   2    |  1700   |   0.640382   |     -      |     -     |   5.29   \n",
            "   2    |  1720   |   0.519504   |     -      |     -     |   5.29   \n",
            "   2    |  1740   |   0.493547   |     -      |     -     |   5.29   \n",
            "   2    |  1760   |   0.537521   |     -      |     -     |   5.29   \n",
            "   2    |  1780   |   0.515671   |     -      |     -     |   5.13   \n",
            "----------------------------------------------------------------------\n",
            "   2    |    -    |   0.562530   |  0.649215  |   72.92   |  490.82  \n",
            "----------------------------------------------------------------------\n",
            "\n",
            "\n",
            " Epoch  |  Batch  |  Train Loss  |  Val Loss  |  Val Acc  |  Elapsed \n",
            "----------------------------------------------------------------------\n",
            "   3    |   20    |   0.501193   |     -      |     -     |   5.54   \n",
            "   3    |   40    |   0.485508   |     -      |     -     |   5.28   \n",
            "   3    |   60    |   0.500813   |     -      |     -     |   5.29   \n",
            "   3    |   80    |   0.579275   |     -      |     -     |   5.29   \n",
            "   3    |   100   |   0.484378   |     -      |     -     |   5.29   \n",
            "   3    |   120   |   0.490452   |     -      |     -     |   5.30   \n",
            "   3    |   140   |   0.458560   |     -      |     -     |   5.30   \n",
            "   3    |   160   |   0.534534   |     -      |     -     |   5.30   \n",
            "   3    |   180   |   0.539542   |     -      |     -     |   5.28   \n",
            "   3    |   200   |   0.458074   |     -      |     -     |   5.28   \n",
            "   3    |   220   |   0.478618   |     -      |     -     |   5.29   \n",
            "   3    |   240   |   0.430923   |     -      |     -     |   5.29   \n",
            "   3    |   260   |   0.469829   |     -      |     -     |   5.29   \n",
            "   3    |   280   |   0.463321   |     -      |     -     |   5.30   \n",
            "   3    |   300   |   0.458269   |     -      |     -     |   5.29   \n",
            "   3    |   320   |   0.430684   |     -      |     -     |   5.29   \n",
            "   3    |   340   |   0.412499   |     -      |     -     |   5.28   \n",
            "   3    |   360   |   0.435544   |     -      |     -     |   5.28   \n",
            "   3    |   380   |   0.469272   |     -      |     -     |   5.28   \n",
            "   3    |   400   |   0.479111   |     -      |     -     |   5.28   \n",
            "   3    |   420   |   0.469161   |     -      |     -     |   5.28   \n",
            "   3    |   440   |   0.511360   |     -      |     -     |   5.28   \n",
            "   3    |   460   |   0.423255   |     -      |     -     |   5.29   \n",
            "   3    |   480   |   0.484979   |     -      |     -     |   5.28   \n",
            "   3    |   500   |   0.422885   |     -      |     -     |   5.28   \n",
            "   3    |   520   |   0.396792   |     -      |     -     |   5.29   \n",
            "   3    |   540   |   0.502838   |     -      |     -     |   5.29   \n",
            "   3    |   560   |   0.493527   |     -      |     -     |   5.30   \n",
            "   3    |   580   |   0.443761   |     -      |     -     |   5.29   \n",
            "   3    |   600   |   0.439863   |     -      |     -     |   5.28   \n",
            "   3    |   620   |   0.495849   |     -      |     -     |   5.29   \n",
            "   3    |   640   |   0.424326   |     -      |     -     |   5.29   \n",
            "   3    |   660   |   0.455047   |     -      |     -     |   5.30   \n",
            "   3    |   680   |   0.425348   |     -      |     -     |   5.30   \n",
            "   3    |   700   |   0.383317   |     -      |     -     |   5.28   \n",
            "   3    |   720   |   0.519654   |     -      |     -     |   5.29   \n",
            "   3    |   740   |   0.390760   |     -      |     -     |   5.29   \n",
            "   3    |   760   |   0.431420   |     -      |     -     |   5.29   \n",
            "   3    |   780   |   0.376722   |     -      |     -     |   5.31   \n",
            "   3    |   800   |   0.521519   |     -      |     -     |   5.29   \n",
            "   3    |   820   |   0.441433   |     -      |     -     |   5.29   \n",
            "   3    |   840   |   0.384255   |     -      |     -     |   5.28   \n",
            "   3    |   860   |   0.538335   |     -      |     -     |   5.29   \n",
            "   3    |   880   |   0.447214   |     -      |     -     |   5.31   \n",
            "   3    |   900   |   0.506114   |     -      |     -     |   5.30   \n",
            "   3    |   920   |   0.451751   |     -      |     -     |   5.31   \n",
            "   3    |   940   |   0.528682   |     -      |     -     |   5.28   \n",
            "   3    |   960   |   0.433507   |     -      |     -     |   5.30   \n",
            "   3    |   980   |   0.414629   |     -      |     -     |   5.29   \n",
            "   3    |  1000   |   0.460932   |     -      |     -     |   5.29   \n",
            "   3    |  1020   |   0.484330   |     -      |     -     |   5.29   \n",
            "   3    |  1040   |   0.507607   |     -      |     -     |   5.30   \n",
            "   3    |  1060   |   0.445494   |     -      |     -     |   5.29   \n",
            "   3    |  1080   |   0.429507   |     -      |     -     |   5.29   \n",
            "   3    |  1100   |   0.409080   |     -      |     -     |   5.29   \n",
            "   3    |  1120   |   0.413201   |     -      |     -     |   5.31   \n",
            "   3    |  1140   |   0.562751   |     -      |     -     |   5.30   \n",
            "   3    |  1160   |   0.415479   |     -      |     -     |   5.29   \n",
            "   3    |  1180   |   0.544736   |     -      |     -     |   5.29   \n",
            "   3    |  1200   |   0.420894   |     -      |     -     |   5.28   \n",
            "   3    |  1220   |   0.427657   |     -      |     -     |   5.28   \n",
            "   3    |  1240   |   0.466296   |     -      |     -     |   5.28   \n",
            "   3    |  1260   |   0.443471   |     -      |     -     |   5.29   \n",
            "   3    |  1280   |   0.412292   |     -      |     -     |   5.29   \n",
            "   3    |  1300   |   0.469973   |     -      |     -     |   5.28   \n",
            "   3    |  1320   |   0.441912   |     -      |     -     |   5.28   \n",
            "   3    |  1340   |   0.421665   |     -      |     -     |   5.28   \n",
            "   3    |  1360   |   0.420700   |     -      |     -     |   5.28   \n",
            "   3    |  1380   |   0.509976   |     -      |     -     |   5.28   \n",
            "   3    |  1400   |   0.524249   |     -      |     -     |   5.27   \n",
            "   3    |  1420   |   0.397940   |     -      |     -     |   5.28   \n",
            "   3    |  1440   |   0.501947   |     -      |     -     |   5.30   \n",
            "   3    |  1460   |   0.528134   |     -      |     -     |   5.29   \n",
            "   3    |  1480   |   0.447873   |     -      |     -     |   5.29   \n",
            "   3    |  1500   |   0.473039   |     -      |     -     |   5.29   \n",
            "   3    |  1520   |   0.469285   |     -      |     -     |   5.29   \n",
            "   3    |  1540   |   0.621983   |     -      |     -     |   5.28   \n",
            "   3    |  1560   |   0.435418   |     -      |     -     |   5.29   \n",
            "   3    |  1580   |   0.437430   |     -      |     -     |   5.29   \n",
            "   3    |  1600   |   0.474289   |     -      |     -     |   5.29   \n",
            "   3    |  1620   |   0.412857   |     -      |     -     |   5.29   \n",
            "   3    |  1640   |   0.500926   |     -      |     -     |   5.29   \n",
            "   3    |  1660   |   0.490173   |     -      |     -     |   5.28   \n",
            "   3    |  1680   |   0.452961   |     -      |     -     |   5.29   \n",
            "   3    |  1700   |   0.558276   |     -      |     -     |   5.30   \n",
            "   3    |  1720   |   0.473677   |     -      |     -     |   5.29   \n",
            "   3    |  1740   |   0.410877   |     -      |     -     |   5.29   \n",
            "   3    |  1760   |   0.451762   |     -      |     -     |   5.30   \n",
            "   3    |  1780   |   0.436437   |     -      |     -     |   5.14   \n",
            "----------------------------------------------------------------------\n",
            "   3    |    -    |   0.464360   |  0.684580  |   72.81   |  491.13  \n",
            "----------------------------------------------------------------------\n",
            "\n",
            "\n",
            "Training complete!\n"
          ]
        }
      ],
      "source": [
        "train(bert_classifier, train_dataloader, val_dataloader, epochs=epoch_number, evaluation=True)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Evaluation"
      ],
      "metadata": {
        "id": "fKx_OjTvD4Bk"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 32,
      "metadata": {
        "id": "fdopnSBFYYq4"
      },
      "outputs": [],
      "source": [
        "import torch.nn.functional as F\n",
        "\n",
        "def bert_predict(model, test_dataloader):\n",
        "    \"\"\"Perform a forward pass on the trained BERT model to predict probabilities\n",
        "    on the test set.\n",
        "    \"\"\"\n",
        "    # Put the model into the evaluation mode. The dropout layers are disabled during\n",
        "    # the test time.\n",
        "    model.eval()\n",
        "\n",
        "    all_logits = []\n",
        "\n",
        "    # For each batch in our test set...\n",
        "    for batch in test_dataloader:\n",
        "        # Load batch to GPU\n",
        "        b_input_ids, b_attn_mask = tuple(t.to(device) for t in batch)[:2]\n",
        "\n",
        "        # Compute logits\n",
        "        with torch.no_grad():\n",
        "            logits = model(b_input_ids, b_attn_mask)\n",
        "        all_logits.append(logits)\n",
        "    \n",
        "    # Concatenate logits from each batch\n",
        "    all_logits = torch.cat(all_logits, dim=0)\n",
        "\n",
        "    # Apply softmax to calculate probabilities\n",
        "    probs = F.softmax(all_logits, dim=1).cpu().numpy()\n",
        "\n",
        "    return probs"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 33,
      "metadata": {
        "id": "UBgeNr_TahQI"
      },
      "outputs": [],
      "source": [
        "# Compute predicted probabilities on the test set\n",
        "probs = bert_predict(bert_classifier, test_dataloader)\n",
        "tensor = torch.from_numpy(probs)\n",
        "ytest = torch.max(tensor, dim=1)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.metrics import confusion_matrix, classification_report\n",
        "class_names = ['negative', 'neutral', 'positive']\n",
        "print(classification_report(df_test.label, ytest.indices, target_names=class_names))\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "O6YWmn2SPEe_",
        "outputId": "61a5d2cc-0e71-4a34-8e29-b37b60541e61"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "    negative       0.76      0.77      0.77      1097\n",
            "     neutral       0.68      0.67      0.67      1276\n",
            "    positive       0.79      0.78      0.78      1188\n",
            "\n",
            "    accuracy                           0.74      3561\n",
            "   macro avg       0.74      0.74      0.74      3561\n",
            "weighted avg       0.74      0.74      0.74      3561\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Plot confusion matrix\n",
        "\n",
        "from matplotlib import rcParams\n",
        "rcParams['figure.figsize'] = 12, 8\n",
        "\n",
        "def show_confusion_matrix(confusion_matrix):\n",
        "  hmap = sns.heatmap(confusion_matrix, annot=True, fmt=\"d\", cmap=\"BuPu\")\n",
        "  hmap.yaxis.set_ticklabels(hmap.yaxis.get_ticklabels(), rotation=0, ha='right')\n",
        "  hmap.xaxis.set_ticklabels(hmap.xaxis.get_ticklabels(), rotation=30, ha='right')\n",
        "  plt.ylabel('True sentiment')\n",
        "  plt.xlabel('Predicted sentiment');"
      ],
      "metadata": {
        "id": "wFLX-fCOHzei"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "cm = confusion_matrix(df_test.label,  ytest.indices)\n",
        "df_cm = pd.DataFrame(cm, index=class_names, columns=class_names)\n",
        "show_confusion_matrix(df_cm)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 533
        },
        "id": "_lKq_WwTH1vI",
        "outputId": "582c371e-d2b2-47a8-ec02-96146e0ae96e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 864x576 with 2 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtYAAAIFCAYAAAD2urpHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzde3zO9f/H8ee1sQM7MIfJkEOxNM2GiRRDjhGiUOhcY74kNarfNEU0yilUKodIiUiOfX0np5wiGpvk2Ig5jG3N7HBdvz98XX2vxnbJ5zKX63H/3a5brvf7/fl8Xp/dvje/l9den/fHZLFYLAIAAABwXdyKOwAAAADgVkBiDQAAABiAxBoAAAAwAIk1AAAAYAASawAAAMAAJNYAAACAAUoUdwCuaEjpfxV3CMB1e2bfyOIOATBEzUq+xR0CYAjvEu7FHYI6mx5yyHm/tXznkPMajYo1AAAAYAAq1gAAADCEm4vXbF377gEAAACDULEGAACAIUwmU3GHUKyoWAMAAAAGoGINAAAAQ7h6jzWJNQAAAAzhRisIAAAAgOtFYg0AAABDmOTmkM+1OHfunF577TU1a9ZMYWFhioqK0okTJ6zzmzZtUufOnRUaGqq2bdtq+fLlNsenpaVp8ODBCg8PV0REhGJjY5WTk2PXtUmsAQAAcMsYNmyYzp49q+XLl2vDhg3y9vbWiy++KLPZrJSUFEVFRalPnz7atm2bhg0bpuHDh2vXrl3W44cOHaqsrCwlJCRo6dKlSkxM1JgxY+y6Nok1AAAADOFmMjnkY6+srCytXbtWAwYMkJ+fn0qXLq1BgwYpKSlJO3bs0DfffKPatWurR48e8vDwUGRkpCIjIzV//nxJUkpKijZs2KCYmBj5+/srMDBQgwYN0qJFi3Tx4sUir8/DiwAAADDEtbZt2Cs9PV3p6ekFxv38/OTn52f9brFYbP77v3/eu3evkpOTFRISYnOOkJAQLVu2TJKUnJwsb29v1apVyzpfr149XbhwQYcOHVJwcHChcVKxBgAAwE1t1qxZatWqVYHPrFmzbNaVLl1a9957ryZPnqy0tDRlZGTo/fffl8lk0p9//qnMzEybRFy6lJxnZmZKkjIzM+Xr62szf/n75TWFoWINAAAAQzhqu71+/fqpa9euBcb/niRLUnx8vMaOHavOnTvLZDLpmWee0fr161W2bFn5+PgoIyPDZn16erp8fHwkST4+PgUS6MvrL68pDIk1AAAAbmp/b/koTIUKFTRu3Djr93379mn06NGKiIjQqVOntH79epv1e/bssbZ4BAcHKysrSwcOHLC2gyQmJsrLy0s1atQo8tq0ggAAAMAQbg76v2tx8OBBnT17VhaLRfv379fw4cPVvXt31axZU126dNG+ffu0cOFC5ebm6ocfflBCQoJ69uwpSapSpYqaNWum+Ph4nT9/XqmpqZo0aZK6desmT0/PIq9NxRoAAACGMN0Eb17csWOHJk6cqPT0dJUrV07dunVTVFSUJKlq1aqaNm2a3nnnHcXFxalSpUoaPXq0QkNDrcfHx8crLi5OkZGRcnd3V/v27TVs2DC7rm2y/O9jk7ghhpT+V3GHAFy3Z/aNLO4QAEPUrORb9CLACXiXcC/uENTX43GHnHd2zlyHnNdoVKwBAABgiGtt27jVuPbdAwAAAAahYg0AAABDOGq7PWdBYg0AAABDOOrNi87Cte8eAAAAMAgVawAAABjCzeTaNVvXvnsAAADAIFSsAQAAYAiTXPvhRSrWAAAAgAGoWAMAAMAQrt5jTWINAAAAQ7jRCgIAAADgelGxBgAAgCF4QQwAAACA60bFGgAAAIZwM7l2jzWJNQAAAAzh5uLNEK599wAAAIBBqFgDAADAECYXbwWhYg0AAAAYgIo1AAAADOHqPdYk1gAAADCEq+8K4tr/rAAAAAAMQsUaAAAAhuDNiwAAAACuGxVrAAAAGIIeawAAAADXjYo1AAAADOHqPdYk1gAAADCEm8m1E2vXvnsAAADAIFSsAQAAYAg38fAiAAAAgOtExRoAAACGMLl4jzWJNQAAAAxBKwgAAACA60bFGgAAAIZguz0AAAAA142KNQAAAAxhcvEeaxJrAAAAGMPNtRNrWkEAAAAAA5BYAwAAwBgmk2M+1+D06dN6+eWX1aRJEzVs2FA9e/bUtm3brPObNm1S586dFRoaqrZt22r58uU2x6elpWnw4MEKDw9XRESEYmNjlZOTY9e1SawBAABwy4iLi1NqaqqWLVumLVu2qE2bNnr++eeVnp6ulJQURUVFqU+fPtq2bZuGDRum4cOHa9euXdbjhw4dqqysLCUkJGjp0qVKTEzUmDFj7Lo2iTUAAAAMYXIzOeRzLY4cOaJ27dopICBA7u7ueuyxx5SVlaWjR4/qm2++Ue3atdWjRw95eHgoMjJSkZGRmj9/viQpJSVFGzZsUExMjPz9/RUYGKhBgwZp0aJFunjxYpHXJrEGAADATe1ytfnvn/T09AJrn3vuOa1evVqnTp1Sbm6u5s6dq+rVq6t27dpKTk5WSEiIzfqQkBAlJydLkpKTk+Xt7a1atWpZ5+vVq6cLFy7o0KFDRcbJriAAAAAwxjX2Q9tr1qxZmjJlSoHx6OhoDRw40GYsLCxMixcvVrNmzeTu7q4yZcrogw8+kIeHhzIzM3XHHXfYrPfz81NmZqYkKTMzU76+vjbzl79fXlMYEmsAAAAYw0Hb7fXr109du3YtMO7n52fz3Ww268knn1Tjxo21detWlS5dWmvXrtVzzz2nuXPnysfHRxkZGTbHpKeny8fHR5Lk4+NTIIG+vP7ymsKQWAMAAOCm5ufnVyCJvpLz58/r999/1wcffCB/f39JUuvWrVW1alVt3LhRwcHBWr9+vc0xe/bsUXBwsCQpODhYWVlZOnDggLUdJDExUV5eXqpRo0aR16fHGgAAAMZwMznmY6eyZcuqVq1amjt3rjIzM2U2m7VmzRrt379fd999t7p06aJ9+/Zp4cKFys3N1Q8//KCEhAT17NlTklSlShU1a9ZM8fHxOn/+vFJTUzVp0iR169ZNnp6eRd/+P/7B3QKOHz+usLAwHT9+vLhDAQAAgAGmTp2q06dP68EHH1TDhg01YcIEjRgxQo0bN1bVqlU1bdo0zZw5Uw0aNNCoUaM0evRohYaGWo+Pj4+Xp6enIiMj1bFjR9WtW1fDhg2z69omi8VicdSN3UwWLVqkKVOm6D//+U9xh6Ihpf9V3CE4HZPJpAdj2qph70byr+yvzNOZSlz6i5aNWKqcrEubtjd6IkK9PnyiwLHTHpqi/Qm/Wr9XuKOCuo7vrppNayknK0e7F/+sb4cvtp4H9nlm38jiDsEp7dm9U98umKtDv+3X6dQT6vXUC+rxxNPW+fz8PH274AutWfGtTp08ofIVK+qhbj3VvkuPK54v7ewZvfz8EzqXdlYfzf9W5SsE3qhbuWXUrORb9CIUadoHU/Th1KkFxr9dvkLVbr9dmzZs0PSpH+jokSPKyspSxcBAtWvfQS9ERamkh0cxRHzr8S7hXtwh6O2gtx1y3jeOveGQ8xqNHms4hRaDItViUEvNf3Guft/5uyrWrqie0x9XCc8SWvCvL63r8vPyNbJ2rM2xWWezrH/2KO2hF5dF64/E45rU6n2VKltKPaf1lre/t+Y8OeuG3Q9cV/aFLFW9vYbub9lWn059v8D8/Jkf6/tli/XikOGqXvNO7dv7i6a//45KlCypBzt2sVlrNps1YXSs7gi+W9t/XF/gXMCNVjkoSLPnzrMZKxsQIEkq7eOj3k/00R133qlSpUspOSlJb735pi5cuKBX7KwGwgk46OFFZ3HDWkH69Omjd955Ry+//LLCw8PVvHlzffnlXwnRrl271KdPHzVu3FiRkZGaMGGC8vLyrPO//PKLunfvrrCwMHXr1k0zZ85UnTp1rPNbtmzRY489poiICDVu3Fgvvviifv/9d0nS9u3bNWLECGvrR1hYmP79738rJSVFderUUUpKis6dO6d69epp586dNnHHxMTo1VdftX5fvHixOnfurAYNGqhjx45atmyZo35k+B/V762pX/+TrN1Ldint6Fnt+3eydi74SVUb3l5gbcbJDJtPfm6+dS780YbyKVdanz81S8d3H9NvP+zXwpcWKKxHAwXcHnAjbwkuqkHj+/TEswPULPJBlSxZsEqXsHqZOvforXubtVClykFq3rqdWrfvrK/nziywdsHnn6pEiZLq9EhPxwcO2MHdzU3lK1Sw+bi7X6qihtavr3YdOuiOO+9U5cpBatmqtTp0fEjbt20t5qgB49zQHutFixbp0Ucf1fbt2xUTE6O4uDj9/vvvOnjwoJ588kn17t1bGzdu1Oeff66EhAR9/PHHki5tc/Lcc88pMjJSW7Zs0bhx46xvyLmsRIkSGj58uDZu3KhVq1bJzc1Nr7zyiiSpYcOGiouLU+XKlbVz507t3LlTrVu3tjm+TJkyevDBB7Vw4ULrWGZmplatWqUePXpY4580aZJGjx6tbdu2KS4uTrGxsdq+fbsjf2yQdOjHg6repKZuC6ksSQqoXk53tamrpJV7bNa5l3DX64mxevPAW+q/YqDqtrvbZr5Gkxo6vPWwstOzrWP71iTLnG9WjSY1HX8jQBFyc3IK/Frcw9NLp07+odSTf1jHftm5Xd8vW6yBMbEyOWjfWOBanTx5Um1aRqpNy0gNeOF5/fy3YtX/OnTwoDZuWK9GEY1vYIRwOJPJMR8ncUMT63bt2qlx48Zyc3NThw4d5Ovrq71792revHlq3bq12rdvrxIlSigoKEgvvPCCFi1aJElKSEhQyZIlFRUVJQ8PD9WsWVP9+vWzOXeDBg1Uv359lSxZUmXKlFF0dLR+/vlnXbhwwe74unfvruXLlysr61LrwHfffafAwEA1atRIkvTZZ58pKipKISEhcnNzU8OGDfXQQw/pm2++MegnhKtZO/E/2jB9nYZsfEXx597XG3tG6OCmA1ox8q/fGKT+mqovXpyrmU98qs96faKUXSl6duELatz3Xusav0r+yjhp+5Ymc55ZWWez5Fep6G18AEcLj2ii5d98pSMHf5PFYtGvSYlas3KpJCnt9GlJ0rmzZzTxnTc18NVYlSnLb1pwcwipV09xb4/S5KnTNCY+Xv5lyujpvn3046ZNNuvatIxUo/qh6trpITWKiNCQ/xbBgFvBDe2xrlixos330qVL688//9Thw4e1ZcsWJSQkWOfMZrMuP1d54sQJVa5cWW5uf/07ICgoyOZcSUlJeu+995SUlGRNjC0Wi86ePVtg7dU0adJEZcuW1YoVK/TII4/o66+/1iOPPGKdP3LkiEaPHq2xY8dax/Lz89WwYUM7fwL4p0K71td9z9+v+S/O0/HdKapwZ6AeHttV7Ud01Iq4S8n1ka2HdWTrYesxR7YeVumAUooc0lpbZm8upsiBa/P0gCH6cMJYvfxCH0kmBZQvr9btO2nRF7Nl+m/v4oR3RqhFm/YKbRBRvMEC/+P+B5rbfA9v0FAnT57UrM8+VZOmTa3jn82eowvZF7QvKUkT3ntPZQMC1D964N9PB2fl4j3WN8XDixUqVFCXLl301ltvXXG+UqVKOn78uCwWi/VXnn/fIm/w4MFq2bKlxo8fLz8/P+3du1ddu3a1Juf/m5RfjclkUrdu3bRw4ULVq1dPSUlJmjZtmnW+fPnyeumll9SpU6d/eqv4hzq/00Xrp/6gn77YJkn6Y88fKuldUj2n9db3Y1Yp72LeFY87vPmQwno0sH5PP3FeZaqUtVnjVsJNpQJKKf1E+t8PB244Xz9/DY0drdzcXJ0/d1YB5Spo1dJLv70LvO1SkWD3jm1K/HmHFn85979HXfp77sXeXdWqfSdFDRleHKEDBYSGhurf339vMxZUpYok6Y477pSbm7teHxajp55+Rt6lShVHiDCayaV3cr459rHu1auXVqxYoVWrViknJ0f5+fk6cuSI1q1bJ0mKjIxUTk6Opk+frpycHB0+fFizZ8+2OUdGRoZKly4tHx8fnT59WpMmTbKZL1++vM6ePavz588XGssjjzyin3/+WePHj1fz5s1VoUIF61y/fv00ZcoU/fLLLzKbzcrJydHu3buVmJho0E8CV+NRylMWs9lmzJJvlkwqtL+0Sv2qOpdyzvr90I+HVD2iujx9vaxjdVoFy83dTYd+PGh84MA/VLJkSZWvECg3Nzdt+M9q1b0nTP5lLv2j8P0Z8zT+oznWT9TLr0mSYsdO1GN9ny3OsAEbSXuTVKlSpavOmy1mmc1m5ebm3sCoAMe5KSrW99xzjz755BNNnDhRI0aMUF5enoKCgtSrVy9Jkq+vrz7++GPFxcXpo48+Uo0aNdS1a1eb5HnUqFEaM2aMZsyYoapVq+rJJ5+0aS2599571bx5c7Vp00b5+fkaO3asza4il1WqVEn33Xef1q5da1Otli4l1mXLltWIESN09OhRubu7q3bt2vrXv9iX2tESv9utFoNa6vSB00rZlaKKtSuqfWxHJa9OUm72pb+Q277WXkd/OqJT+1Pl7llCoV3qq/GTTfTN0L8eSN3x1XY9OKytnvisr1bEfadSZUur23s9tHPBTzp75Gxx3R5cyIULWTpxLEWSlJeXq3Nnz+jQb7/Ky9tbtwVV1f7kvTp18g/VvLOOzqelacmCuTp0YL9GTfzQeo7ba9SyOWfG+Uv/eKxctZoCylcQUBzGjR2rB5o3V+WgIGVmZmrR119r84+bNGHyFEnS7JmfqXqNmrq9+u0yyaS9e/ZowvjxahEZKb//vnoazs/k4q0gTvuCmM8//1xz5szRqlWrijuUa8YLYq6dRykPtX29ve55OFR+t/kr81SG9q7YoxUjlykr7VJPfecxXVWv0z3yC/RV7oVcnfw1VT9M+o92L9llc64Kd1ZUt/HdVaNJTeVeyNXuxT9rybBveEHMNeIFMf9M4s8/Kfbl/gXG7w4N11vvTVPSLz9r+oSxOnn8mEqULKG69cL0+DNRur3mHUWekxfE/DO8IMYYw4YO1Y4dPynt7Fn5+Pqqdu3aeua55xVx76UHyD/9+GN9t/RbHT9+XG4mk26rXFntO3bU4336ytvbu5ijvzXcDC+IGVXrXYec9/UDrxa96CbgNIn1xo0bVatWLQUGBioxMVEDBw5U9+7dFR0dXdyhXTMSa9wKSKxxqyCxxq3ipkis74x3yHlf3+8cu8fcFK0g9jhw4IBiYmKUkZGhcuXKqVOnTnr++eeLOywAAABAkhMl1n379lXfvn2LOwwAAABcjRO9zMURnCaxBgAAwE3OxR9evCm22wMAAACcHRVrAAAAGKKwd0u4AirWAAAAgAGoWAMAAMAYLt5jTWINAAAAY9AKAgAAAOB6UbEGAACAMVy8FYSKNQAAAGAAKtYAAAAwhotXrEmsAQAAYAj2sQYAAABw3ahYAwAAwBgu3gpCxRoAAAAwABVrAAAAGIMeawAAAADXi4o1AAAAjOHiPdYk1gAAADAE2+0BAAAAuG5UrAEAAGAMF28FoWINAAAAGICKNQAAAIzh4hVrEmsAAAAYg4cXAQAAAFwvKtYAAAAwhou3glCxBgAAAAxAxRoAAACGcPUXxJBYAwAAwBi0ggAAAAC3ho4dOyosLMz6CQ0NVZ06dfT9999Lkvbu3auePXsqNDRULVq00OzZs22Oz87OVmxsrCIiIhQeHq7Bgwfr3Llzdl2bxBoAAADGMJkc87kGy5Yt086dO62fl19+WWXKlNEDDzygzMxMPfvss2rWrJm2bt2qCRMmaMqUKVq5cqX1+NGjRysxMVFLly5VQkKCsrKyFBMTY9e1SawBAABwy/riiy/UvXt3eXp6avXq1XJzc1P//v3l6emp+vXrq0ePHpo3b56kS9XqxYsXa9CgQQoMDJS/v79iYmK0du1aHT9+vMhr0WMNAAAAYzioxzo9PV3p6ekFxv38/OTn53fV43788UcdPnxYPXv2lCQlJyerbt26cnP7q7YcEhKiBQsWSJIOHz6sixcvql69etb5WrVqydvbW0lJSapcuXKhcZJYAwAA4KY2a9YsTZkypcB4dHS0Bg4ceNXjvvjiC91///2qWrWqJCkzM1O+vr42a/z8/JSZmWmdl1Rgja+vr3WuMCTWAAAAMIaDNgXp16+funbtWmC8sGr1yZMntWbNGn3wwQfWMR8fH505c8ZmXXp6unx8fKzzkpSRkaGAgADrmoyMDOtcYUisAQAAYAwH7WNdVMvHlXz11VeqVKmSHnjgAetYcHCwVqxYIbPZbG0H2bNnj4KDgyVJ1atXl6enpxITE63HHThwQBcuXLCuKQwPLwIAAOCWkpeXp6+++kqPPfaYTT91mzZtlJ+fr2nTpiknJ0e7d+/WggUL1KtXL0mSl5eXunTpokmTJik1NVXnz59XfHy8mjdvrqCgoCKvS2INAAAAQ5jcTA75XKs1a9bo3Llz6t69u824j4+PZsyYoXXr1qlhw4YaOHCgBgwYoPbt21vXvPbaa7rrrrvUsWNHRUZGytPTU++++65992+xWCzXHC2uy5DS/yruEIDr9sy+kcUdAmCImpV8i14EOAHvEu7FHYLebTfLIed9dWU/h5zXaPRYAwAAwBiu/UZzEmsAAAAYxEEPLzoLeqwBAAAAA1CxBgAAgDEc9OZFZ0HFGgAAADAAFWsAAAAYw7UL1iTWAAAAMAgPLwIAAAC4XlSsAQAAYAwXL9m6+O0DAAAAxqBiDQAAAGPQYw0AAADgelGxBgAAgCFMLl6xJrEGAACAMVw7r6YVBAAAADACFWsAAAAYw821S9ZUrAEAAAADULEGAACAMXh4EQAAADCAa+fVtIIAAAAARqBiDQAAAGPw8CIAAACA60XFGgAAAMZw7YI1iTUAAAAM4uK7gtAKAgAAABiAijUAAAAMYXLxhxdJrIvBE4kjijsE4Lp9VPv14g4BMMRz+0YVdwiAIUKqlinuEFweiTUAAACM4doFa3qsAQAAACNQsQYAAIAxXHxXEBJrAAAAGMPFH16kFQQAAAAwABVrAAAAGMO1C9ZUrAEAAAAjULEGAACAMXh4EQAAADCAi/dCuPjtAwAAAMagYg0AAABjuHgrCBVrAAAAwAAk1gAAADCEyWRyyOdabd26Vb1791ZYWJgiIiIUFRVlndu0aZM6d+6s0NBQtW3bVsuXL7c5Ni0tTYMHD1Z4eLgiIiIUGxurnJwcu65LYg0AAABjuDnocw22bdumqKgo9ezZUz/++KM2bNhgTaxTUlIUFRWlPn36aNu2bRo2bJiGDx+uXbt2WY8fOnSosrKylJCQoKVLlyoxMVFjxoyx+/YBAACAW8L48eP16KOPqnPnzvLy8pKHh4fuueceSdI333yj2rVrq0ePHvLw8FBkZKQiIyM1f/58SZcS7w0bNigmJkb+/v4KDAzUoEGDtGjRIl28eLHIa5NYAwAAwBgmk0M+6enpSklJKfBJT0+3uXxWVpa1+tytWzc1btxYjz32mH788UdJUnJyskJCQmyOCQkJUXJysnXe29tbtWrVss7Xq1dPFy5c0KFDh4q8fXYFAQAAwE1t1qxZmjJlSoHx6OhoDRw40Po9PT1dZrNZS5cu1UcffaQ777xT33zzjV588UV99913yszM1B133GFzDj8/P2VmZkqSMjMz5evrazN/+fvlNYUhsQYAAIAxHLTdXr9+/dS1a9cC435+fjbfS5cuLUl65JFHVLduXUnSo48+qlmzZmn9+vXy8fFRRkaGzTHp6eny8fGRJPn4+BRIoC+vv7ymMCTWAAAAMIaDmoz9/PwKJNFX4uvrq6pVqxYYv7yzSHBwsNavX28zt2fPHgUHB1vns7KydODAAWs7SGJiory8vFSjRo0ir0+PNQAAAG4Zjz/+uBYuXKh9+/YpPz9fCxcu1LFjx/TAAw+oS5cu2rdvnxYuXKjc3Fz98MMPSkhIUM+ePSVJVapUUbNmzRQfH6/z588rNTVVkyZNUrdu3eTp6Vnkte1KrPv27VugOVy61GvSt2/fa7xdAAAA3JIc9PDitXjyySfVs2dPPfPMM2rUqJG++uorffjhh6pSpYqqVq2qadOmaebMmWrQoIFGjRql0aNHKzQ01Hp8fHy8PD09FRkZqY4dO6pu3boaNmyYXde2qxVk69atys3NLTCek5Ojn376yc7bBAAAABzLZDIpOjpa0dHRV5xv2rSpli5detXjAwICNHHixH907UIT6+PHj1v/fOLECZv9+/Lz87Vu3TqVL1/+H10YAAAAtxgHPbzoLApNrFu2bGl9lWT37t0LzLu5uWnIkCEOCw4AAABwFoUm1nPnzpXFYtETTzyhqVOnyt/f3zpXsmRJBQUFqVy5cg4PEgAAAE7AxbfFKDSxbtCggSRpzZo1uu222+Tm5uI/LQAAAFwdrSBFCwoKktls1uHDh3X69GlZLBab+UaNGjkkOAAAAMBZ2JVY//LLL3rppZd07NixAkm1yWRSUlKSQ4IDAACAE6FiXbQRI0aoRo0aeu+991SxYkXr22sAAAAAXGJXYn3w4EGNHz/erlc5AgAAwEW5+ON4dt1+zZo1lZaW5uhYAAAA4MxugjcvFie7Eus33nhD77//vpKTkwv0WAMAAACwsxWkT58+MpvN6tq1q0wmU4Ft9xITEx0SHAAAAJyIE1WXHcGuxPrtt992dBwAAACAU7Mrse7atauj4wAAAICz4+FF+6SlpWn27Nl66623rA8y/vzzzzp27JjDggMAAIAT4eHFov36669q37695syZo/nz5yszM1OSlJCQoIkTJzo0QAAAAMAZ2JVYjx07Vg899JBWr14tDw8P6/j999+v7du3Oyw4AAAAOBGTgz5Owq7E+pdfftETTzxR4I2Lt912m06fPu2QwAAAAABnYtfDixaLRXl5eQXG//jjD/n4+BgeFAAAAJyQmxOVlx3Arop1kyZNNG/ePJuxnJwcTZs2Tffdd59DAgMAAACciV0V66FDh6p3797au3evcnNzNXLkSP3222/Kz8/Xl19+6egYAQAA4AycaAcPR7Arsa5WrZqWLFmiefPmyd/fX2azWV26dNETTzyhcuXKOTpGAAAAOAPXzqvtS6wlqS7dd8cAACAASURBVFy5cho4cKAjYwEAAACclt2JdX5+vg4dOqQzZ87IbDbbzDVp0sTwwAAAAOBkXPzhRbsS6+3bt+uVV17RiRMnZLFYbOZMJpOSkpIcEhwAAADgLOxKrEeMGKF69epp6tSpKl++fIH9rAEAAAAeXrTD8ePHNXXqVN1+++2OjgcAAADOyrXzavv2sW7UqJH279/v6FgAAAAAp2VXxfqtt97S8OHDlZKSojp16qhECdvDGjVq5JDgAAAA4ER4eLFoR48e1a+//qpNmzYVmOPhRQAAAMDOxDo2Nlb169fXCy+8oAoVKvDwIgAAAApy8RzRrsT6jz/+0PTp03l4EQAAAFfn2nm1fQ8vNmjQQAcOHHB0LAAAAIDTsqti3bVrV40ZM0bHjh274sOL4eHhDgkOAAAAToSHF4s2dOhQSdKoUaMKzPHwIgAAAGBnYr1mzRpHxwEAAABnx8OLRQsKCnJ0HAAAAIBTu2pivWPHDoWGhsrd3V07duwo9CT0WAMAAMC+bTFuXVdNrHv37q2NGzeqXLly6t27t0wmkywWS4F19FgDAABAEq0gV5tYs2aNAgICrH8GAAAAcHVXTaz/t6/aZDLptttuu+IbF48fP+6YyAAAAOBcXLxibVcnTKtWrXT27NkC42lpaWrVqpXhQQEAAAD/xOTJk3XXXXcpLCzM+hkyZIh1fu/everZs6dCQ0PVokULzZ492+b47OxsxcbGKiIiQuHh4Ro8eLDOnTtn17Xt2hXkSr3Vly/s4eFh14UAAABwi7tJHl5s2LCh5syZU2A8MzNTzz77rHr37q1Zs2YpKSlJzz//vCpWrKh27dpJkkaPHq3ExEQtXbpUXl5eeuWVVxQTE6MPP/ywyOsWmlhPmTJF0qVWkE8++USlSpWyzpnNZu3YsUO1atW6phsFAADALcpBrSDp6elKT08vMO7n5yc/Pz+7z7N69Wq5ubmpf//+cnNzU/369dWjRw/NmzdP7dq1U3Z2thYvXqzJkycrMDBQkhQTE6MOHTro+PHjqly5cqHnLzSx/vbbbyVdqlivWrVK7u7u1rmSJUuqSpUqGjlypN03AwAAAFyrWbNmWQu+/ys6OloDBw4sMJ6YmKh7771X3t7e1naOqlWrKjk5WXXr1pWb21+l9ZCQEC1YsECSdPjwYV28eFH16tWzzteqVUve3t5KSkq6vsR69erVkqQ+ffpoypQp8vf3L/RkrmjYsGGSpDFjxhRzJAAAAMXMQRXrfv36qWvXrgXGr1Stbtu2rbp166bKlSsrNTVV48eP11NPPaUlS5YoMzNTvr6+Bc6RmZkpSdb//n2Nr6+vda4wdvVYX6lHxZlNnjxZW7duveXu61a2dMFcbdu4VsdTjspisajq7TXVpfeTqt/wXuuahFXfacOalTp6+IDycnN1W1BVdejWU81atrWuSTtzWnNnTNHhA/v1R8pR1b0nTK+PmVQctwQXZDKZ1GZYOzXq3Vj+lf315+lM7V66W9/FLlFOVo4kKeKJe/X4R30KHPtBx0n6NWGfJOmO++/UwFWDC6z5ov9cbZ65ybE3AUha/OUcbdmwVsd+PyKLxaJq1Wuq++NPKyyiic26X5MSNXPaBB3cv0+lfX0V2aajej31os1vwNPOnNYnH7ynn7f/KEkKj2iqZwa8LP+yATf0nnBzu5aWj9q1a1v/HBgYqFGjRqlhw4bauXOnfHx8dObMGZv16enp8vHxkSTrfzMyMqzbTl/+fnmuMHYl1pK0efNm/fjjjzp9+rTMZrPN3DvvvGPvaZxGXl6e3N3dr7jFIG68Pbt+UvO2D6lW7bvk6eml/6xcqvjYVxQb/4Hq3H3PpTU/b1eDJver9zP95ePrp20/rtPUcW/J3d1dTZq3liTl5eaqtK+fOnbrqc3r/yNzfn5x3hZcTOSgVmo5qJXmvfC5ft95VBVrB6r3h0+ohGcJfTXwC+u6/Lx8jbjzDZtjs87+WeB87zZ5R+kn/uo5zD5/wXHBA/8j8eef1LJdJ91Rp648vbz07+VLNPqNl/XWe9MUHBIqSTqdelIjYwbq3vsj9eKQ1/THsd81ddzbslikPs8NkHTpea3RbwyRyeSm2LGTJYtFH02K19gRr2rUxI/5/8HO6CZ5ePF/mUwm64sOg4ODtWLFCpnNZms7yJ49exQcHCxJql69ujw9PZWYmKgHHnhAknTgwAFduHDBuqYwdiXWH330kd577z3VrFlTFStWdPj/0Pv06aO6devq9OnTSkhIkK+vr/r376/HHntMkrRr1y6NGzdOv/76q0qVKqWHH35Y0dHRKlGihFJSUtSqVSutWbNGVapUkSRt2bJFffv21b59+/Ttt9/qww8/VH5+vsLCwiRJc+fOVUZGhvr27av33ntPEyZM0B9//KEtW7Zo8eLFmjdvno4fPy4fHx+1bt1ar776qry9vR36M4CtYW+/Z/P98WcHaNf2zdq6ca01sY6OedNmzUOP9FbS7p+1ed1/rIl1hUq36an+l7bcSU7cpbOnUx0fPPBfNZrU1L7/JGvXkp8lSWePntWOr37SnS1qF1ibcbLgQzp/l3k60651gNHeeGeCzfe+zw/Uzm2btXnDWmtivWrpQnmXKq3+L78hNzc3VateU2dPp2rOx1PU44mn5eXtrd07turg/n2a9NlXCqp6uyTpX8Pe1EvP9tKeXTsUUr/BDb83XKeb4B9Dy5cv17333quAgACdOXNG48aNU0BAgDXvGzdunKZNm6bnnntOycnJWrBggd58801JkpeXl7p06aJJkyYpODhYnp6eio+PV/PmzW3e8XI1diXWc+fO1f/93//p8ccf/+d3eY0WLVqkKVOmKD4+XitXrtTQoUPVtGlT5ebm6sknn9To0aP14IMP6uTJk+rfv788PT0VFRVV5Hk7d+6sI0eOFGgF2bJliyRp1apVWrBggby9veXh4aEKFSpo6tSpqlatmg4ePKioqChNnz5dL730ksPuHUUzm826kPWnvLwK/wdO1p8ZqhB42w2KCijcwU0H1HJwa1UOqazjicdVrno53dX2bu1astNmnXsJd/3fnjiV9C6p1F9PKmHiGu1ZkVjgfIP+PUQe3h46ffCUNn6yQdvmbrlRtwLYuPx3sqeXl3Usec9u1W/Q2OYhsbBGTTRj8jgd+m2f7qpXX8l7dqtipcrWpFqSqlWvqXIVKio5cReJNf6Rb7/9ViNHjtSFCxfk5+enRo0a6bPPPrO2csyYMUNxcXH68MMPVbZsWQ0YMEDt27e3Hv/aa69p1KhR6tixo/Lz83X//fcrLi7OrmvblVhnZGRYy+E3Srt27dS4cWNJUocOHRQXF6e9e/dq27Ztat26tfUHEBQUpBdeeEHvv/++XYl1UYYOHaoyZcpYv7dp08b651q1aql3795atmwZiXUxWzx/lrL+zFTL9g9fdc36NSu1P3mP+r5QsBcVKA4JE9eopFdJDd00TLJI7iXdtenTDVoe9511TeqvJzXvhTk69ssxlfAoofrdwvT8wih9EfW5Ns+61IOafuK8Fgyer6M7jspituiuNnXVc0ovVahVQctHfne1ywMOs2jeTP2ZmaE2HbtYx9LOnFbwf3+jeFmZsuUuzZ09Y11TNqBcgfOVKVtOaWdPOzBiOMxNULGePn16ofN169bVl19+edV5Ly8vvfXWW3rrrbeu+dp2JdatW7fW5s2bVbVq1Wu+wD9VsWJFm++lS5fWn3/+qcOHD2vLli1KSEiwzpnN5qu+xOZaXW4fuWzlypX69NNPdeTIEeXl5SkvL0/lyhX8SwA3zuqlC7Xky9kaOuJdlatQ8Yprtv+4Th9PHKMXBg9XjTvr3OAIgSur3zVMzZ6/X/Ne+FzHdqeo4p2B6vruI+r4Zicte3OpJOnw1kM6vPWQ9ZjDWw+pdEBptRryoDWxTt2fqtT9f7Ux/b7jqNxLuCvyX620cvRymfNsn4MBHGnlkq+18IuZGjZynMpVCCzucIBiZVdiXb9+fU2cOFH79+9XcHCwSpYsaTPfqVMnhwR3JRUqVFCXLl2u+q+I0qVLS5IuXPjrIZ7UVNs+2sJ6xP/3V1YnTpzQSy+9pPfff18tW7aUh4eHZs6cWeDVl7hxvvt6nr6eM0NDR7yreuGNrrhm09rvNX38KD03KEb3t25/xTVAcegyppt++GCttn+xVZL0x57jKuldUr2nP6FV76xQ3sW8Kx53aMshhT/asNBzH9p8UO1e6yCfCr5K/+O84bEDV7Lkq8/15eyPNWzkOIU2iLCZK1uuvM6dtd194Vza2Utz/61Sly1XXrt3bCtw3vNpZ1U2oLyDooZD3YQPL95IdiXWl18Cc6WE0mQy3dDEulevXnr66afVrFkzRUZGyt3dXSkpKTpy5IgeeOABlS1bVlWqVNFXX32lmJgYHT9+XJ9++qnNOSpUqKBjx44pJyen0Fey//nnnzKbzSpbtqw8PDyUnJysuXPnOvoWcRULZn+s5d98qVffGq+694Rdcc2aFUs0a+r7ihr6hvWBReBm4VHKQxaz7W/XLPlmyVT4P/ir1K+qcylphZ67Sv2qysnK0Z+ni95nFTDCFzM/1HcL5+v1Ue/r7tDwAvPBd9+jH/5tu/vCz9t+lKeXl2rcUce6ZsGcT3Q85agqV6kmSfr9yEGdPnXS+hAk4EzsSqyTk5MdHYfd7rnnHn3yySeaOHGiRowYoby8PAUFBalXr17WNWPHjlVcXJwaNWqkOnXqqHv37jZviOzQoYNWrlypZs2ayWw2X3U/61q1amnw4MF66aWXlJ2drfr16+vhhx/WokWLHH6fsDVr+gStWb5YA4fFqXKVatYqiIenp0qVvvQwwvJF8zV3xgd6asDLuqtemHVNiZIl5eP7196Xhw/8KknKzEhXdvYF6/fqtQruzAAY6ZfvdqvloFY6ffCUUn7+XRVrB6rDiE5KWr1Xudm5kqR2r3fQ0e1HlLo/VSU8S6h+1zA1ebKpFg5dYD1Pi+hIpf2epj+S/pAsFgW3rqu2w9pr/YfrlJ/LFpJwvE+nvqfvv1uswa+/pcpVb7f2THt4eKr0fx8Qa9vpEa1YskDT3hutTt176+TxFH0x80O1f/hRef13Z617wiNU8846mjTmTT0T/bIsFotmTI5X7btCrpis4+bn6lskmixGNSfDbjsOnSl6EWz0atf0iuMPtO6gqKGX9vsd2LebTqeeKLDmrnphio3/oMhzfbGSF2tci1l3xxZ3CE7Ho5SH2r3eUaFd6sv/Nn9lnsrQnhWJWha3VFlpWZIutYvc0zlUvoF+yr2Qa90V5PIWfZLUcnBr3ftkU5WtUlb5ufmXdgWZsV6bZ/5o2PMmruS5faOKOwSn80jrxlccb9Gmowa++tffDb/u/UUzp0+89IIYHx9Ftn3oKi+IGa+d2zbLZJLCGjXVs9G8IOafCKlapuhFDvbeR47ZnWjI81f+39zNxu7EeuHChZozZ46OHj2qJUuWqGrVqvr4449VrVo1tW3btugTwIrEGrcCEmvcKkiscasgsS5+drWYf/nllxo7dqwefPBB5eX99XBNQECAPv/8c4cFBwAAAOdhMjnm4yzsSqw///xzxcXFacCAATa/vrn77rv122+/OSw4AAAAwFnY9fDikSNHdM899xQYL1WqlDIzeQIdAAAAPLxoV8W6QoUKOnr0aIHxnTt33tCXxgAAAOAm5uagj5OwK9QuXbpozJgxOnTokEwmk7Kzs/XDDz9o3LhxeuSRRxwdIwAAAHDTs6sVJCoqSseOHVOHDh1ksVjUuXNnSVK3bt309NNPOzRAAAAAOAdXbwWxK7EuUaKExowZo+joaCUmJspsNiskJETVqlVzdHwAAACAU7Arsb6sSpUqqlKliqRLr/sGAAAArFy8Ym1Xj/Wnn36q5cuXW7+/+uqratiwoVq2bKkDBw44LDgAAAA4D/axtsMXX3yh8uXLS5K2b9+u77//XuPHj1e9evUUHx/v0AABAAAAZ2BXK8jJkyetLSBr165V27Zt1aFDB915553q06ePQwMEAACAk3Cm8rID2FWxLlWqlDIyMiRJW7duVePGl97X7uXlpezsbMdFBwAAADgJuyrWERERGjt2rMLDw7V3717df//9kqSDBw+qUqVKDg0QAAAAzsHkRsW6SK+//ro8PDz0/fffa+TIkdZ+63Xr1qlp06YODRAAAABwBnZVrAMDAzV9+vQC4//3f/9neEAAAABwUq5dsL62fawBAACAq3H1Ny/a1QoCAAAAoHBUrAEAAGAIFy9YU7EGAAAAjEDFGgAAAMZw8ZK13RXrzZs3Kzo6Wp06ddKJEyckSQsWLNCWLVscFhwAAACch8lkcsjHWdiVWH///fd67rnn5Ofnp8OHDys3N1eSdPHiRc2YMcOhAQIAAADOwK7Eetq0aYqNjdXo0aNVosRf3SNhYWFKSkpyWHAAAABwIm4O+jgJu0I9ePCgmjRpUmDcz89P58+fNzwoAAAAwNnYlVj7+fnp1KlTBcb37dunwMBAw4MCAACA86HH2g5t2rTR+++/r8zMTOvYb7/9pnHjxqlDhw4OCw4AAABOxGRyzMdJ2JVYDxkyRBaLRffdd5+ys7PVvXt3derUSUFBQYqOjnZ0jAAAAMBNz659rEuVKqU5c+Zoy5YtSkxMlNlsVkhIyBX7rgEAAOCanKi47BDX9IKYxo0bq3Hjxo6KBQAAAHBadiXWU6ZMKXSedhAAAAA404OGjmBXYv3tt9/afM/Ly9PJkyfl4eGhihUrklgDAADA5dmVWK9evbrA2JkzZxQTE6OePXsaHhQAAACckBO9zMUR/vHtlytXToMHD1Z8fLyR8QAAAMBJsY/1dShRooRSU1ONigUAAAAwzIABA1SnTh1t2bLFOrZp0yZ17txZoaGhatu2rZYvX25zTFpamgYPHqzw8HBFREQoNjZWOTk5dl3PrlaQHTt22Hy3WCxKTU3VjBkzFBISYteFAAAAcIu7iarLixcvVnZ2ts1YSkqKoqKi9MYbb+jhhx/Wxo0bNXjwYAUFBSk0NFSSNHToULm7uyshIUHZ2dmKiorSmDFjFBsbW+Q17Uqse/fuLZPJJIvFYjMeHh6ut99+2977AwAAABzuxIkTmjBhgubNm6fIyEjr+DfffKPatWurR48ekqTIyEhFRkZq/vz5Cg0NVUpKijZs2KDly5fL399f/v7+GjRokAYNGqSYmBh5enoWel27Eus1a9bYfHdzc1NAQECRJwcAAIDrcFTBOj09Xenp6QXG/fz85OfnZzNmsVj02muvKSoqSpUrV7aZS05OLtBtERISomXLllnnvb29VatWLet8vXr1dOHCBR06dEjBwcGFxllkYp2bm6vx48dr8ODBqlatWlHLAQAA4KoclFnPmjXriu9ViY6O1sCBA23G5s2bJ4vFoscee6zA+szMTN1xxx02Y35+fsrMzLTO+/r62sxf/n55TWGKTKxLliypdevW6aWXXiryZAAAAIDR+vXrp65duxYY/3u1+ujRo5o2bZq+/PLLK57Hx8dHGRkZNmPp6eny8fGxzv89gb68/vKawtjVCtK8eXOtW7dOjz/+uD3LAQAA4IJMbo6pWF+p5eNKtm/frnPnzqlbt2424/3799dDDz2k4OBgrV+/3mZuz5491haP4OBgZWVl6cCBA9Z2kMTERHl5ealGjRpFXt+uxLp+/fqaPHmykpOTVa9ePXl7e9vMd+rUyZ7TAAAAAA7Tvn17NW3a1GasefPmevvtt9W0aVOlp6drxowZWrhwoTp37qxNmzYpISFBs2bNkiRVqVJFzZo1U3x8vMaOHauLFy9q0qRJ6tatm13PFhaaWLdq1Upff/21Ro0aJUlasGCBFixYYLPGZDKRWAMAAKDYd9vz9vYuUACWpICAAOsuH9OmTdM777yjuLg4VapUSaNHj7ZutSdJ8fHxiouLU2RkpNzd3dW+fXsNGzbMrusXmlgfO3ZMZrNZycnJ13hbAAAAcDnFnVlfwb59+2y+N23aVEuXLr3q+oCAAE2cOPEfXcvF3+gOAAAAGKPIHutTp04pLy+v0DWBgYGGBQQAAADnZLoJK9Y3UpGJ9ZW2NrnMYrHIZDIpKSnJ0KAAAAAAZ1NkYj1p0iT5+/vfiFgAAADgzFy7YF10Yh0eHq5y5crdiFgAAAAAp1VoYu3qfTIAAACwn6NeEOMsCk2sLRbLjYoDAAAATs610+oiEmv2rwYAAADsY9crzQEAAICiuHobMS+IAQAAAAxAxRoAAACGcPGCNYk1AAAAjOHqiTWtIAAAAIABqFgDAADAECYX33CPijUAAABgACrWAAAAMISr91iTWAMAAMAQrp5Y0woCAAAAGICKNQAAAAzBmxcBAAAAXDcq1gAAADCEa9erSawBAABgEFpBAAAAAFw3KtbFoG61ssUdAnDdntw7srhDAAzxWrUnijsEwBDfWr4r7hDYbq+4AwAAAABuBVSsAQAAYAgXL1hTsQYAAACMQMUaAAAAhnD1XUFIrAEAAGAIF8+raQUBAAAAjEDFGgAAAIZw9VYQKtYAAACAAahYAwAAwBCuXa8msQYAAIBBXLwThFYQAAAAwAhUrAEAAGAIHl4EAAAAcN2oWAMAAMAQrl2vJrEGAACAQVy8E4RWEAAAAMAIJNYAAAAwhMlkcsjnWkydOlWtW7dWgwYN1LhxYz3zzDNKSkqyzu/du1c9e/ZUaGioWrRoodmzZ9scn52drdjYWEVERCg8PFyDBw/WuXPn7Lo2iTUAAABuGe3bt9fChQv1008/af369brvvvv03HPPyWw2KzMzU88++6yaNWumrVu3asKECZoyZYpWrlxpPX706NFKTEzU0qVLlZCQoKysLMXExNh1bXqsAQAAYAhH9Vinp6crPT29wLifn5/8/PxsxmrUqGHz3c3NTadOnVJGRobWrFkjNzc39e/fX25ubqpfv7569OihefPmqV27dsrOztbixYs1efJkBQYGSpJiYmLUoUMHHT9+XJUrVy40ThJrAAAA3NRmzZqlKVOmFBiPjo7WwIEDC4yvXbtWQ4cOVUZGhkwmk5566in5+/srOTlZdevWlZvbX00bISEhWrBggSTp8OHDunjxourVq2edr1Wrlry9vZWUlERiDQAAgBvD5KAN9/r166euXbsWGP97tfqyFi1aaPv27Tp37pwWL16s2267TZKUmZkpX1/fAufIzMy0zksqsMbX19c6VxgSawAAABjCUa0gV2r5sEeZMmXUt29fNWrUSDVr1pSPj4/OnDljsyY9PV0+Pj6SZP1vRkaGAgICrGsyMjKsc4Xh4UUAAADcssxms/Ly8nTkyBEFBwdr7969MpvN1vk9e/YoODhYklS9enV5enoqMTHROn/gwAFduHDBuqYwJNYAAAAwhMnkmM+1mD17tk6dOiVJOnv2rOLi4uTh4aH69eurTZs2ys/P17Rp05STk6Pdu3drwYIF6tWrlyTJy8tLXbp00aRJk5Samqrz588rPj5ezZs3V1BQUJHXJrEGAADALWPz5s3q0qWL6tevr86dO+v06dOaOXOmypcvLx8fH82YMUPr1q1Tw4YNNXDgQA0YMEDt27e3Hv/aa6/prrvuUseOHRUZGSlPT0+9++67dl3bZLFYLI66MVxZdr656EXATS7p97TiDgEwxIga/Yo7BMAQ31q+K+4QtHrnMYect01Y0dXimwEPLwIAAMAQjnp40VnQCgIAAAAYgIo1AAAADEHFGgAAAMB1o2INAAAAQ5hcvGRNYg0AAABDuHZaTSsIAAAAYAgq1gAAADCEq7eCULEGAAAADEDFGgAAAIZw8YI1FWsAAADACFSsAQAAYAhXr1iTWAMAAMAQJhffcI9WEAAAAMAAVKwBAABgCFdvBaFiDQAAABiAijUAAAAM4eoviCGxBgAAgCFcPK+mFQQAAAAwAhVrAAAAGMLVW0GoWAMAAAAGoGINAAAAQ7h2vZrEGgAAAAZx8U4QWkEAAAAAI1CxBgAAgCF4eBEAAADAdaNiDQAAAEO4eMGaijUAAABgBCrWAAAAMITJxTfcI7EGAACAIWgFAQAAAHDdqFgDAADAEGy3BwAAAOC6UbEGAACAIVy8YE1iDQAAAGO4emJNKwgAAABgACrWAAAAMISr72NNxRoAAAAwABVrAAAAGIIeawAAAMAAJpPJIZ9rER8fr44dOyo8PFzNmjXTa6+9prS0NJs1e/fuVc+ePRUaGqoWLVpo9uzZNvPZ2dmKjY1VRESEwsPDNXjwYJ07d67Ia9/SifX27dsVFhZ23WsAAADgHNzd3RUfH68tW7ZoyZIlOnHihIYPH26dz8zM1LPPPqtmzZpp69atmjBhgqZMmaKVK1da14wePVqJiYlaunSpEhISlJWVpZiYmCKvbbJYLBaH3NVNaPLkydq6davmzJlTrHFk55uL9fq3oi2bN+vFZ59RUFAVfbdqlXX8woUL+nDaVK1avlypqadUNqCsuj/6qF7sP6AYo701JP2eVvQi2Fi6YK62blyr478flcViUdXq/9/efcfXfP7/H39kh0gQkhAjglZKIkZUjFbMojU77FJ86UesGjWqatOaRWt8UHs0NqWlYtRKqSqxajQEETSISGSck98f+eVdETp8Donmeb/d3Oq817mu03N7e53X+3VdV0latOlEhSqBxjGRERdYvXQ+F8+fJTrqCkGvvUH3D4ZkupbZZGLT6mXs2raFG9FR5M7txMs1a9O198Bn2aV/hU+8O2Z1E55LDrkdaPVxa2q+8wqunq5cPXuVlSNXsH/NPgDqvVefoA61KeHnha2DHVd/vcqGKevZvXxXhutYW1vTYmBL6r5XD48SHty7c48Da/Yzq8eXz75Tz7mNqZuzugkcf0r/NvgVy//E5+7cuZP+/ftz5MgRANauXcuUKVPYs2cP1tZpOeaJEydy/PhxFi9ezP3793n55ZeZMWMGtWrVAuD8pAp5hwAAIABJREFU+fM0btyYnTt34unp+dj3Uo21PPdu3rjBx0OHUK16DS5dvGhsN5lM9PzP+9yLi2PYiJGU8Pbmzu3bmR4HiTwr4Ud/IqjBG5Qq8xL2Do7s/HYTnw0fyCeTvqBMufIAJCXep6CbB5UDa7Jl7crHXuvLyWM4ezKcdl2D8Sr1Avfj47kRHfWsuiJC8NxelAksw5fdZ3LtwjUqNw5gwIqBjHo9nqPbf6Z8nfL8uOEgCz/8iriYuwQ2r0bfxR9gSjGx9+sfjOv0WdgXn2o+LPzwKy4cvUAu59x4lHDPwp5JdhQbG0tsbGym7S4uLri4uPzpuQcOHMDHx8d4ffr0acqWLWsE1QC+vr6EhIQAEBERQWJiIn5+fsb+UqVKkStXLk6dOpX9A+sOHTpQpkwZrl69yoEDB3Bzc6Nnz540bdoUgO+//56ZM2cSGRmJh4cH7733Hm+//TaQ9kF/8skn7N+/n+TkZNzc3Pjggw9o2LAhYWFhvPvuu5w5c4aNGzcyZ84cTCaTUfqxbNky7t69axxz7tw5mjZtys6dO/Hw8DDa1759e6pUqUKfPn0wmUwsWrSIkJAQrl+/jpeXFwMHDqRatWrP/oMTzGYzQwd9SKs2bUlKTMwQWG/asIFTJ06w6dvvKFCgAABFihTJqqaKMGTslAyv23UN5pfDB/lx7y4jsC5VpiylypQFYOd3j84+nTj6E/t3fs+nsxZRrERJY7tXqReeUstFMrJzsKPmOzWZ2mEKR78/CsA3MzdToV4F3vnoHY5u/5kpHSZnOGf9lHWUq+VLzXdqGoG1X5Afr7apRW//XkSevGQcG3Hst2fXGbGopzXd3qJFi5g5c2am7T179qRXr16PPW/Lli2EhISwdOlSY1tcXBzOzs4ZjnNxcSEuLs7YD2Q6xtnZ2dj3ONkisAYICQnh888/Z/r06ezdu5eePXvi5eVFamoqffv2Zdq0adSuXZsjR47w/vvvkzdvXho0aMD8+fO5d+8eO3bswMnJiatXr5KQkJDp+k2bNuXixYuZSkHCwsKMv5cuXRo/Pz/Wrl3Lf/7zHyDtV8tPP/3EhAkTAPjyyy8JDQ3lyy+/xMvLix07dtCjRw82bNhA8eLFn/KnJA+bO2sWVlZWdO7aldlffJFh3/fbt+Hr58fyJUvYtHEDtra2VA0MpE+/fuTL9+SPlEQsxWw2Ex9/DwfHXP/ovLC9u3Av7Mnxnw8xacQgkpOTefElX9p360lB90JPqbUif7C1s8Xaxpqk+0kZticmJFGpYWVsbG0wpZgynZcnnxPREdeN19XfrMG1C9eoUK8CwzYMw9bBjjMHTrNgwAJuRt546v2Q50fHjh1p0aJFpu1/lq3+5ptvGDFiBLNmzaJcuXLG9jx58vD7779nODY2NpY8efIY+wHu3r2Lq6urcczdu3eNfY+TbQYv1q5dm6CgIGxtbQkKCqJevXqsWbOGtWvXUrduXerVq4eNjQ1VqlThnXfeYdWqVQDY2dlx+/ZtLly4QGpqKkWKFKF06dJP3I633nqLNWvWkF56vmbNGgIDAylatCgACxcu5MMPP8Tb2xtra2vq169P5cqV2bw56+uacpofw8IIWbWSsRM+feSI4cuRkfx85Ajh4ceZNHUqn4wcxfFjx+jbsxc5aGiBZGPrVy4iPi6Ouo2b/aPzoqOu8Pv1aH74/lu69R1Mn6Gjib1zm9Ef9iIpKfEptVbkDwlxCZzad4p3PnoHdy93rKysqNSwMlWbVcXOwQ6XgpmDnaB2QbwYWIaN0zYY2wqVKoRbcTdqv1uHmf83g4mtPsXFLS9jQ8di52D3LLskFmJl9XT+uLi4ULRo0Ux/HhdYh4SEMHLkSGbPnk1gYGCGfT4+Ppw8eRKz+Y8xbydOnDDKRUqUKIGDgwPh4eHG/vPnz5OQkJChpORRsk1gnR64Pvg6KiqKqKgoihUrlmFf8eLFiYpKqyXs0qULNWrUYNiwYQQGBtKnTx8iIyOfuB2NGzfm999/JywsDJPJxLp164yyk5s3bxIXF0dwcDABAQHGn8OHDxMdHf3E7yn/3K1btxg66ENGjR1HQTe3Rx5jNptJTU3ls0mTKe9fgarVqjFyzFh+PvITp0+dfMYtFslo26Y1rF+5mA+GjaWA2z+rJ01NNZOcnESPgR9TrkJlypTzo89Ho7geHcXRHw88pRaLZDSl/STu3b7H3AvzWJu0ns6TOrN93jYAUs0ZkxdVm1Yl+L+9mNFlOhd+Pm9st7K2xt7RnqnvTuH4ruOcPnCaia0+xd3bg4DGAc+0P2IZ1lZWT+XPP7F48WImTZrE/PnzqVy5cqb9DRo0wGQyMWvWLJKSkjh27BghISG0adMGAEdHR5o3b8706dO5fv06d+7cYeLEidSqVesvS0qzTSnIlStXMr0uVKgQVlZWXL58OcO+S5cuUbhwYQBy5cpFnz596NOnD7dv32bUqFEMGTIkQy1Nur8zD6KTkxONGjVizZo1JCQkkJycTL169YC0X0sODg7MmzePSpUqPWlXxQLOnT3LjevX6dXjP8a29EC6kp8vY8ZPwM3NjeSkJPLmy2ccU+r/P824evUqL5Utl+m6Is/CppDlrF4yj4EjPsOvUpV/fH4+14JYWVlRpHgJY1vefK64uOTlxvVrFmypyOPduHSD4Q0+xj6XA3nyORETFUOnT9/j3p173LlxxzjulVav0mdhX2b+3wx2Ld2Z4Rq3omIwm81cPvVHQuzOjTvcvRmLm5cGMMqTGTt2LLa2trz77rsZtn/zzTd4enqSJ08e5s2bx8iRI5kzZw758+cnODiYRo0aGccOHTqUsWPH8vrrr2MymXjllVcYOXLkX753tgmsd+7cye7du6lZsyZ79+5l+/btRi10hw4dCA0NpVatWvz888+EhIQwZswYAHbs2EHx4sXx9vYmV65cODg4ZBjl+SA3NzeuXLlCUlIS9vb2j23L22+/TadOnbh+/TpNmzY1jrW3t6d169Z89tlnjB07lpIlS5KYmMjx48cpWLAg3t7eFv5U5HHK+fqyesOGDNu+XrGCPbt2M3PObAoVKsyFC+f55ehR7t69awxAiPgtbUCMp6cGMUrW+HrRf9mybhWDxkymbPknm0Pfx9efPdu3EHX5khFc3429Q2zsHdw8CluwtSJ/LSkhkZiERGztbKn+Vg3C1h80yu0adH2NbjO6M63j1AwzgaQ7+cMJ6naqh+eLRbhyJi2J5uzqjHNBF64/UIstz4/ssPLimTNn/vKYsmXLGmXFj+Lo6Mjo0aMZPXr0P3rvbBNYv/XWW6xatYq+fftSsGBBxowZY8zeMWXKFD7//HMGDBiAu7s7AwcOpGHDhgBcvnyZTz/9lBs3bmBvb4+/v78RdD+scePGfPvtt9SsWROz2fzY+awrVqxI0aJFOXjwIEOHDs2wb9CgQSxdupQ+ffoQFRWFg4MDZcuW/VuThovl5M6dmxdeeDHDNlfXAtjZ2RnbW7Vpw8rlyxk2eDA9+/bhfsJ9xo8ZTUCVKvi89FJWNFtyuEWzpvH9lvX0HjISz6LFuR2TNnjG3sGB3E5pA2JSkpO5fCntB2BiQgJxd2OJOP8rtrZ2FPVK+/FeI6g+61csYvaUcXTq8QG2tnYsn/8lhTyLZpgTW+RpqlCvArb2tkSeiqRgMTfajWqHfS57Fg9NW8Guad9mvDexM7ODZxG++zj5PNKeHqYkpRB3K21mhd0r9vD2R63ovaAP/+09h5SkFDp+2omoc1H8tPVwlvVN5ElliwViOnTowMsvv/yn06X8m2iBmKdj1syZfLNpU4YFYk6dPMmkzz7l+C+/4JI3L6+8+ip9+/XPUB4iT0YLxPxzrV+r/sjtr9ZvTI8BwwC4fi2K3h3fzHRMQY9CzFy81ngdHXWFxbM/J/zoTzg4OPBS+Yp06Nabgu4emc6VP6cFYp5MtZbV6TihE27F3UiIS+Dn746weMgiblxKm83jv7/Nx6NE5u/j8V3H+aj2H4seeXh70HVaN8rXKU9ifCIndoczv988bl6++cz68m+RHRaIORN1568PegJlCud9Kte1NAXWWUCBtfwbKLCWfwsF1vJvkR0C61+jMi/iYgkvFv7zRWCyi2wzK4iIiIiIyPMsW9RYP67WWURERESeH9lh8GJWUsZaRERERMQCskXGWkRERESef39nzZB/MwXWIiIiImIROTyuVimIiIiIiIglKGMtIiIiIhaR00tBlLEWEREREbEAZaxFRERExCJydr5aGWsREREREYtQxlpERERELCKn11grsBYRERERi8jhcbVKQURERERELEEZaxERERGxiByesFbGWkRERETEEpSxFhERERHLyOFF1gqsRURERMQicnZYrVIQERERERGLUMZaRERERCwih1eCKGMtIiIiImIJyliLiIiIiEXk8IS1AmsRERERsZAcXguiUhAREREREQtQxlpERERELCJn56uVsRYRERERsQhlrEVERETEInJ4ibUy1iIiIiIilqCMtYiIiIhYSM5OWSuwFhERERGLUCmIiIiIiIj8z5SxFhERERGLyOEJa2WsRUREREQsQRlrEREREbGInF5jrcBaRERERCwkZ0fWKgUREREREbEAZaxFRERExCJyeimIMtYiIiIi8q/xzTff0LZtWypVqkSZMmUy7T958iStW7fG39+foKAgFi9enGH//fv3GT58OC+//DKVKlWib9++3L59+2+9twJrEREREbEIq6f0559wcXGhbdu2DB06NNO+uLg4unbtSs2aNfnxxx+ZNm0aM2fO5NtvvzWOGTduHOHh4WzatImdO3cSHx/PoEGD/tZ7K7AWEREREcvIBpH1K6+8whtvvEGxYsUy7du2bRvW1tb06NEDBwcHKlSowNtvv83y5cuBtGz1+vXr6dOnDx4eHuTNm5dBgwaxa9curl69+pfvrRprEREREcnWYmNjiY2NzbTdxcUFFxeXv32d06dPU7ZsWayt/8gt+/r6EhISAkBERASJiYn4+fkZ+0uVKkWuXLk4deoUnp6ef3p9BdYiIiIiYhFWT2m6vUWLFjFz5sxM23v27EmvXr3+9nXi4uJwdnbOsM3FxYW4uDhjP5DpGGdnZ2Pfn1FgLSIiIiLZWseOHWnRokWm7f8kWw2QJ08efv/99wzbYmNjyZMnj7Ef4O7du7i6uhrH3L1719j3ZxRYi4iIiIhFPK3p9v5pycfj+Pj4sHXrVsxms1EOcuLECXx8fAAoUaIEDg4OhIeH8+qrrwJw/vx5EhISjGP+jAYvioiIiMi/hslkIjExkeTkZAASExNJTEzEbDbToEEDTCYTs2bNIikpiWPHjhESEkKbNm0AcHR0pHnz5kyfPp3r169z584dJk6cSK1atShSpMhfvrcCaxERERGxiGwwKQgbNmygfPnydOnSBYDy5ctTvnx5Dh06RJ48eZg3bx579uwhICCAXr16ERwcTKNGjYzzhw4dyksvvcTrr79O7dq1cXBw4LPPPvt7/U9NTU39h+2V/9F9kzmrmyDyPzsVeSurmyBiEZ94d8zqJohYxMbUzVndBG4lJD+V6+bPZfdUrmtpyliLiIiIiFiABi+KiIiIiEU8pbGLzw1lrEVERERELEAZaxERERGxiKc13d7zQoG1iIiIiFhEDo+rVQoiIiIiImIJyliLiIiIiGXk8FoQZaxFRERERCxAGWsRERERsYicna9WYC0iIiIiFpLDK0FUCiIiIiIiYgnKWIuIiIiIheTslLUy1iIiIiIiFqCMtYiIiIhYRE6vsVZgLSIiIiIWkcPjapWCiIiIiIhYgjLWIiIiImIROb0URBlrERERERELUMZaRERERCwkZ6eslbEWEREREbEAZaxFRERExCJyeo21VWpqampWN0JERERE5HmnUhAREREREQtQYC0iIiIiYgEKrEVERERELECBtYiIiIiIBSiwFhERERGxAAXWIiIiIiIWoMBaRERERMQCFFiLiIiIiFiAAmsREREREQtQYC0ikkXSF77VArgiIv8OCqxFRLLAli1bWLBgAQBWVlZZ3BoREbEEBdYiIs+Y2Wxmz5497N27lxMnTgDKWouI/BsosJbnRmpqKiaTKaubIfI/MZvNWFtb07p1a+zs7Ni8eTOgrLU8f8xms34QijxEgbU8F1JTU7GyssLGxoaoqCh2797NuXPnsrpZIv9Iamoq1tZpt90KFSpQoUIFTp06xa5du4z9Is+D9B+IVlZWnD59mlOnTnHz5k1jv77LklMpsJbnQno2b8KECTRp0oRVq1bRtm1b5s6dy/Xr1wHdyCX7Sn/S8nBW+o033sDZ2ZnvvvuOuLg4Za3luWFtbc2dO3cIDg6mS5cuTJw4kVatWrFnzx4jESKSE9mMGDFiRFY3QuRRHr45b9myhf379zN37lxat25NyZIlWbBgAXFxcVSvXl03csmWzGYzNjY2AISFhXH48GFy585N7ty5cXV1JTY2liNHjmBtbU25cuWyuLUij2Y2mzPcY1NSUvj888+xtrbmq6++omXLlkRFRbFq1SqKFi2Kl5dXFrZWJOsoYy3ZlpWVFREREcTFxQGwdetWAgIC8PT05PDhw0yfPp3cuXPTvHnzLG6pSGbpT1Csra25dOkS7du3Z/DgwSxevJg+ffrw1VdfAdCoUSOKFCnCnj17iIyMzHCuSHaRXsJ04cIFAO7cucOPP/5I586dsbe357vvvmPTpk2UKlWKChUqZGVTRbKUAmvJto4dO0b//v05c+YM9+7dIzk5mVy5ctGrVy969+5Ns2bNWLNmDd7e3pw+fTqrmyuSgZWVFampqURHRzNu3DjKlSvHzp07Wbt2LY0aNWLFihWcOXOGvHnzUrduXeLj49m4caNxrkhWMZvNj9y+cOFCxo0bh8lkIjIyksTERC5fvszbb7/NpEmTGD58OFOnTiUlJcUo0RPJaRRYS5aZOXMmd+7ceez+IkWKcOHCBRwdHXFycsLNzY1x48aRP39+9u/fT+fOnQEYNWoUP/30E0lJSc+q6SKP9GCmefr06fTv3598+fLRqFEjhgwZAsCSJUtYsGABycnJTJ48GYB69epRpEgRDhw4wKVLl7Kk7ZKzJSQkMHfuXCDzD7v0MQJubm4cP34cGxsbKlSoQGpqKt27d6dJkyZs376d1157jaioKCZMmKDvseRYCqwlSyQnJ3Pu3DmSk5ON7EhoaCgRERFGcFKgQAEqVqzIzp07Aejfvz/u7u7Y29uzY8cOQkJCePXVV7l27RoNGjTA3t4+y/ojOcvDpRoPDk4MDw/np59+4vDhw7Rs2RIHBwcaNmzI77//Tvv27Vm7di2zZs3iP//5D4cOHeK7774D4N1332X8+PEUL178mfdH5MSJE5w6dQqTyWTck7du3QpgjBEoXbo0xYsX5/DhwwD069cPBwcHcuXKxenTp5kxYwbNmzcnb968+Pv7Z01HRLKYVaqK+eQZSJ+aKTU1NcOUYwBJSUnY29vTsmVLHB0dadSoER06dOD+/fsMHDgQPz8/unXrBsDhw4fZuXMnMTExREdH06FDB2rXrg1kHuwo8jTdunWLn376iXr16gFpPxZjYmJ47bXXyJcvHyNHjqRWrVrG93LNmjUcPHiQiRMnAjBjxgy+++473NzcjHprkWcl/Z78OJcvX6ZevXq89dZbtGjRgsqVK3PmzBmCg4OZPn06ZcuWBWDZsmX88ssvJCQkcP/+ffr374+Pj8+z6oZItmOb1Q2QnMHa2prExEQcHBwyBL9Lly5ly5YtLF++nPnz57N582amTJmClZUV7du3p3Tp0uzatcsIrAMCAggICDCC8XQmk8nIqog8bYmJiSxcuJDQ0FBeeOEFFixYgJ2dHf369eODDz5g4sSJFCtWDPhjVpBz584RGhrKxYsXWbJkCWfOnGHChAn4+vpmcW8kJ7K2tiY2NhYXFxdjW3JyMgMGDKBcuXJ069aN5cuX8/XXXzNgwABmzpxJuXLljFK89MC6Xbt2tGvXjjt37pA3b17gjxrtPwvcRf6tNN2ePBORkZF07NiR/Pnz88ILL7BixQrs7e0pVKgQ06ZNw8fHh3LlyuHv74+bmxurV6/m4MGD+Pv7c/bsWWrWrEnu3LkzLBQDaQG1tbW1buDyTKQHDLa2tri4uLBt2zYWLFhA7ty56d+/P66urnh6erJr1y6uX79O7dq1jcxgyZIlOXbsGGvXriUhIYGJEydSsmTJLO6R5CTp90uTycQPP/zAsGHDaNSoEQ4ODqxbtw4vLy8uXLjAggUL6NKlC0WKFKFGjRqcPXuWjRs3cu3aNdzd3UlOTqZq1arGAF0rKyscHR2N97CxsdHTQ8mxFI3IM+Hq6kpQUBBTpkyhTp06rFy5EhcXF8qWLUu7du2YOHGiMfjwzTff5KOPPuLGjRsMHjyYqKgo4yb98M1aWWp5ltJXmgO4dOkScXFxFChQgIEDB+Lp6UlKSgru7u4EBwfz9ddfExERga2tLSkpKXh6ejJ79mzmzZvHokWL8PDwyOLeSE5y6dIlAgMDuXz5MjY2NhQoUABXV1eCg4OpVKkSO3bswNHRkfbt21O4cGFGjRoFQK5cuRg9ejTNmzdn48aNLFiwgNu3bxvJDN2TRTJSxlqemgdrnm1sbIxavGrVqrF48WKcnZ0B8PPzY/78+dja2lKpUiUAChcuTK1atUhISKB169Z6XC7Zgtls5pNPPuH+/fs0atSIoKAgrl27xt69e2nYsKERVBQuXJhff/2VzZs389Zbb2FlZYWVlRV2dnbkyZMni3shOZGDgwP169fH29sbSFvgZe7cuZw/f56BAwcyePBgrK2tcXJyIn/+/EyZMoU33niDfPnyYWNjg6+vL6VLl+b27dt06dIFd3f3LO6RSPakwFosLv1x44OZDLPZjLu7Oz4+PuzYsYO6devi4uJCSkoKTk5OODo6Mnv2bJo3b46TkxNmsxknJyeCgoL0uFyyjZSUFHbt2sWiRYvo0qUL+fPn59atWxw6dAhbW1teeuklAOzt7XF3d2fWrFk0bNiQAgUKZHHLJaeztbWlQIECREdHc+HCBUqWLImLiwvOzs5cuXKFBg0aGPfswoULc+rUKXbs2EGLFi2MJEmxYsVo0qQJ7u7uGiwu8hgqBRGLS8/aff3114wZM4YNGzYQHx9P1apVeeWVVyhatChTpkwB/niM2K5dOxwdHZkwYQKQcdDL4xYrEHkWDh06ZEyvZ2dnR/fu3bG3t2fq1KkA1KhRAz8/P1atWsX9+/fZvHkzCxcuxN/fn4MHD1K6dOmsbL7kYOnTQD5o/vz5tGrVCmtra958802qVKnCpUuXWLt2LZD2pDFPnjz06NGDsLAwQkNDHzmvtYJqkUdTxlos7tixY3Tt2pXw8HBcXFzYsmUL+/bto1mzZri4uGBnZ8e6devw9fWlSJEiHDt2DGtra2rXrk3lypVxc3PLcD3dwOVZMZvNGQZk3bp1i9dffz3DvLwuLi44Ojoybdo02rZti5ubG/b29pw8eZLp06ezZ88eWrZsSenSpY0BXSLPUvoPwfQExY4dO4iOjqZYsWJ4e3vz/fffExMTQ7Vq1ciXLx9XrlwhLCyMJk2aYG1tzYkTJyhVqhTe3t5UrVqV3LlzZ7i+BouLPJ7msZb/ycOPA2NiYpg9ezYuLi707NkTgAMHDvDee+8xefJkXn/9daKjo5k6dSo//vgj3t7enDhxgoULFxpzn+oRozxrD07XmJKSkmFWgy+//JLly5ezefNm8uXLB6TNYd26dWsqV65sLPGcmJjI8ePHqVq1apb1Q+RBP//8M5988glJSUn069ePatWq4ezszPLlyxk7dix79+4lf/787N69my+++IK8efNy7do1cufOzfz5843xALoni/x9CqzliTw8b/T27dupX78+8fHxnDp1isqVKwMwevRovvnmGzw9Pbl58yZbt27FycmJuLg4li9fjrW1NV27ds2qbohkMHfuXPbu3UuhQoWoVq0azZs3x8rKijp16lC3bl0++ugjAO7evUvbtm05e/YsK1eupEKFClnccpGMfvnlFwYPHkz79u1p165dhn137tyhS5cuFC5cmBkzZpCYmMiFCxdYuXIlZcuWpVWrVlnUapHnn0pB5ImkPwo8cOAAYWFhDBw4kEaNGuHh4UHBggW5e/cu3bt35+7du8yePZsaNWqwYMECUlNTqV69Ovb29lSuXNmYBSQlJUWPFyXLxMXF0atXL8LDw+nduzfx8fFs377dmEPdw8ODyZMnU65cOUqUKEFYWBj29vYEBQXh5+dnLIwh8qylDxZ/2NGjR7l69Sp9+/bF3t6epUuXcvr0aWJjYyldujSFCxdm2rRpBAYGUrx4cdzc3Khdu7YxA9Pjrisif04rL8oTOX/+PP369cPGxgY/Pz8gLTu9cOFC7O3tOXDgAI6OjkyePJk8efKwc+dOGjRowKFDhzKtmpiamoqtrb6K8mykpKRk+r6dPn0agNWrVwPg5ubG+vXruX37NomJiTRs2JDQ0FDGjx/PmDFjSEpKYvz48dSoUeOZt18kXWpqqvHk8MCBA5hMJkqWLImnpyd2dnbcunWL/v37c+nSJZydnfH09GTq1Kl88cUXvPLKK9SqVYtvv/2WgIAA45rpCxppPmqRJ6NoRv7So+rr1q5di4+PD59++ikpKSlUr16d/v37s2XLFho3bkx0dDS7d+8mJCSEsLAwIiIi+O9//2ss8/wg1e7Js5D+PU5fsOXQoUN4enri5eXFr7/+SkJCAmazmREjRvDtt9/Spk0bPvjgA+Lj4wEYN24cZ8+eJSIigkaNGmVxb0TS7p2RkZEMHDiQW7du4e7uzvXr12nfvj0dOnTA29ubX3/9lYIFCxIQEEBERAQxMTFGJnrKlCk4OTlluKay1CL/GwXW8liPW5o2ISGBTZs2MWDAACBter2goCBatWrF+PHjady3tq1AAAAQnElEQVS4Me+88w5Xrlzh0KFDuLu7M336dCNL/aiMocjTlv493rRpE8OHD6dEiRJcvXqVqVOnYjKZSEhIICAggKCgIDZu3EihQoWIi4sjJCSEpk2bUqBAAV566SVjrmqRZ+3hsS1ms5klS5ZQvnx5hg4dSkJCAitWrGDlypUUKFCAxo0bU7x4cWxtbYmLi2PmzJmYTCaKFy8OYATVD19XRJ6cfprKIz34iHHXrl2sWrXKqM/LlSsXpUqV4ujRo0Dazd3BwQEvLy9u3LjB3LlzAejZsyfTpk1j1KhR2NvbG3OqKqiWrHDu3DlWr17Nvn37mDdvHuvWrePVV19l9uzZ5MmTBxsbG+rUqcOUKVMoVKgQ+/fvp02bNly+fBkHB4esbr7kYOlzDKTfk5OSkgC4ffs269evJzAwEEhbfrxJkyZUrVqVrVu3Amkzg4wfP546depgNpuZM2cOrq6uGa6voFrEchThyCNZWVlx9epVBg0aRHR0NC+++CKrV6+mTJkyjBkzhlq1arFu3TpOnDhBuXLljPPq1avHnDlzaN26NS4uLkDaPwoPBuoiWSE0NJQvv/ySRo0aGbPWjB8/njfeeIMbN27QsGFDDh48SJMmTShcuDBnzpyhR48emiFBslR6zTOkzUc9f/58PD09qVKlChUrVqRKlSpERUUBafdaNzc3HB0djeC7cOHCeHl5sWTJEsqUKQMoQy3yNCmwFuDRN9rNmzfz4osvsmTJEgB2795N9+7dqV+/Pq+99hpnzpyhY8eOdOvWjZMnT3L69Gk++eQToqOj+e6773j77beBtCBdddSS1Tp16kRoaCipqancv38fR0dHbG1t6dy5M0uXLmXgwIG0bNmSGzduEBERQd26dbO6ySJYW1tz9+5dNm7cyKpVq2jdujU3b95k6dKlHDlyhJSUFE6fPs358+cpVaoUkPZUsEiRIgAULVqUtm3bAn+sYqugWuTpUWCdw6UP6Eq/0cbHx5M7d27u3bvH4sWLmTFjBgCzZs1iyZIlNGvWjGrVqmFvb8/48eMpUaIEV65cIVeuXKxbtw47Ozt+//138ufPn5XdEsnE3t6e4OBgZs+ezb59+4zA+Z133mH58uWsXLmS8uXLU6pUKSNAEclq0dHR9OnTh+TkZEaNGmXMmV68eHFCQ0OxsbEhPj6e4OBgPvzwQ3bu3MmuXbuYPHky8Mc9PjU1VQMTRZ4BBdY52MOPGGfPnk3hwoWpV68ederUoXLlyqxfv56hQ4eSO3duZs2ahb+/P5cuXeL27duUL1+e7t27Z7jm+vXrKVCgAN7e3lnRJZE/VbNmTVauXMnu3bvx9fXFw8MDgLFjx2IymTQftWQ7Hh4elCtXjtWrVxsz1ADUrVuXffv28eKLL9KkSRO++uortm7dSmpqqnEfhj8G7eqpocizocA6B7O2tiYuLo4tW7bw9ddf88Ybb3D16lVGjRqFg4MDNjY2bNu2je7du9OpUycATp06xYwZM3jttdfw8/PDysqKa9euceTIEb766iuioqIYMWKEMn6SLVlZWTFgwAD69+/Ptm3b6NChA0CGcQIi2U1wcDCHDx/m+PHjVK9eHQBnZ2diY2OJjIzEy8uLESNGZFgjQHXUIllDgXUOlv6I0dramrFjxxoDW6Kioti+fTsVK1YkOTmZ0NBQChUqxA8//MC2bdvo1KkTzZo1M65TsGBB7t27R+PGjXnvvfeyqjsif4u3tzeVKlXC2dk5q5si8re4urrSsmVLtm3bhru7Oy1atODKlSvExsbSsmVL47j0oNpsNiuoFskiVqnp8/hIjjR69Gg2b97MF198Yay+df78ef7v//6Pbt26UaVKFTZt2kRiYiLx8fH07t3beMSY/tVJr9/To0Z5XjxYBiXyPEhKSqJNmzb8+uuvNGvWjH379lG7dm2GDx+e1U0TkQcosM7hYmJi6Nq1K02bNqVDhw5GlmPy5Mns2LGD8ePH4+/vn+Eck8mEtbW1AmkRkWfowIEDTJ8+nYYNG9KyZUvjqYsSGyLZh1I2OZyrqyvNmjVj//79/PLLL8b24OBg8ufPn2lhjPRHjLqJi4g8W4GBgeTPn5/z58+TnJwMpK1kq/uxSPahwFpo06YN8fHx7N69m7t37wLg6OjIsmXL8PHxyXCsHp+LiGQNKysrBg4cSHh4OJs3bwa0kq1IdqMoSbC3t6dz587s3r2biIiIDPvSlyEXEZGslz74Nn1lWxHJXlRjLUBajd758+cpXbp0VjdFRET+hAbfimRfCqxFRERERCxAP3lFRERERCxAgbWIiIiIiAUosBYRERERsQAF1iIiIiIiFqDAWkRERETEAhRYi0iO06FDBz766KPHvn6WLl++TJkyZTh8+HCWvP9fCQsLo0yZMly7di2rmyIiku0psBaRLDd48GDKlClDmTJlKFu2LLVr12b48OHcunXrmbz/jBkzGDJkyN8+vn79+syYMeMptihrlC1blrVr12bYVrFiRfbu3Yu7u3sWtSqjDRs2UKZMmaxuhojII2ktVBHJFgICApg2bRomk4nw8HCGDRvGtWvXmDt3bqZjU1NTSUlJwc7OziLvnS9fPotc59/I3t4eNze3rG6GiMhzQRlrEckW7OzscHNzo1ChQtSrV4+OHTvyww8/cP/+fdauXUvZsmU5ePAgzZs3x8/Pj/3795OcnMyMGTOoU6cOfn5+vP7666xcuTLDda9cuUKXLl0oX748tWrVYsmSJZne+1GlIMuWLaNx48b4+vpSrVo1evXqZRx76dIlZs6caWTZL1++DMDFixfp1asXAQEBVKlShc6dO3PmzJkM192yZQv169fHz8+P1q1bZ9r/KGfPnqVLly4EBARQoUIFGjVqxPr164399+7dY8yYMbzyyiv4+/vTvHlztm3bZuxPLzfZsmUL3bt3x9/fn7p162bITtepUweTycSQIUOMfkHmUpD017t376ZVq1aUL1+eli1bcvbsWc6ePUubNm3w9/fnrbfe4ty5cxn6ER4eTufOnalYsSKBgYH07NmTK1euGPtnzJhB/fr1+f7772nYsCEVKlSgQ4cOREREGO/94YcfAhhtHDx48F9+fiIiz4oCaxHJlhwdHTGbzaSkpABpyzhPmjSJwYMHs3XrVvz8/Pj444/Ztm0bo0aNYsuWLQQHBzNp0iRCQkKAtMx2z549uX37NkuWLGH27NmEhoZy4sSJP33v6dOnM2nSJNq2bcumTZuYN28eZcuWBdKCvyJFitC5c2f27t3L3r17KVy4MDdv3qRt27a4urqybNkyVq1ahbe3N++++y4xMTEAnDx5kv79+9OwYUM2bNhA586dGTt27F9+Fv369SNfvnysXLmSTZs2MXjwYPLmzWv08f333+fMmTNMnTqVzZs306ZNG/r168eBAwcyXGfy5Mk0a9aMjRs38vrrrzNs2DB+++03AFavXo2NjQ1Dhw41+vVnpk6dSt++fVm7di12dnb069ePESNG0Lt3b2Pbg+U1586do0OHDlSoUIHVq1ezaNEirK2t6dy5M4mJicZxN27cYMWKFUyaNImVK1dy7949hg4dCqSVpQwfPhzAaGNW1caLiDyKSkFEJNs5d+4cy5Ytw9/fnzx58gBpAeTgwYMJCAgAIDIykvXr1/PNN99QqlQpAIoVK8aFCxdYunQpb7/9NgcOHODkyZN8++23eHt7A2nBZVBQ0GPfOz4+nnnz5tGnTx/at29vbC9XrhyQVjZiY2ND7ty5M5RIrFixgiJFijBy5Ehj27Bhw9i9ezcbN26kU6dOLFiwAH9/f/r37w9AyZIluX79OqNHj/7Tz+Pq1au89957lC5d2uhnuh9//JGjR4+yf/9+nJ2dAWjVqhVHjx5lyZIlVKtWzTi2ffv2NG7cGIA+ffqwZMkSwsLC8Pb2xtXVFQBnZ+e/VfrRs2dP49qdOnWib9++TJ8+3djWuXNnevbsyb1793BycmLevHkEBQXRu3dv4xqTJk2iSpUq/PDDD9SrVw+ApKQkJk6caLSna9eu9OvXj8TERBwcHIzvg8pTRCQ7UmAtItnCjz/+SMWKFTGZTCQlJVGtWjVGjRqV4Rg/Pz/j7+Hh4aSmpvLWW29lOCYlJQUbGxsgLUDPnz+/EVQDuLq6Znj9sHPnzpGYmEiNGjX+UfuPHz/OiRMnqFixYobt9+/f5+LFiwCcP3+ewMDADPsrV678l9fu3Lkzw4YNY926dbz88svUqVPHCPSPHz9OcnIyr776aoZzkpOT8fLyyrDNx8fH+LuNjQ0FChTg5s2bf7+TD3hwAGF6kPvgtoIFCwIQExODk5MTx48f5+LFi5k+n8TERKPUA8Dd3d0IqtNfp6am8vvvv+Pp6flEbRUReVYUWItItlC+fHk+/fRTbGxscHd3x97ePsN+GxsbHBwcjNepqalAWqY4V65cGY61srJ6+g1+iNlsJjAw0ChVeFB6JvlJBQcH07RpU/bs2UNYWBhz5syhS5cufPDBB5jNZpydnVm9enWm8x4e3PnwaysrK+Nz/KceNXDU1vaPf1LS/x+YzWbjv82aNaNbt26Zzntw8OjjBqSmX0dEJDtTYC0i2YKjo2OmDOufSc/YRkVFUbt27UceU7p0aW7dukVERAQlSpQA0jKov/32G76+vo88p1SpUjg4OLBv374MGd4H2dnZYTKZMmzz9fVl3bp1FCpUKMMPgIev/fPPP2fYduTIkcf28UHFihWjXbt2tGvXjrlz5zJ//nw++OAD/Pz8iI2NJTExkRdffPFvXetxHtUvS/H19eXMmTMUL178f/rhkx54m0wm48mEiEh2ocGLIvJc8vLy4s033+Tjjz9m/fr1XLx4kdOnT7N69Wpjir5q1arh4+PDwIEDOXbsGKdOnWLgwIEZMqsPc3Jy4r333mPmzJksW7aM3377jdOnTzNnzhzjmKJFi3LkyBGuXr1KTEwMZrOZ9u3bYzKZ6NGjB4cPH+by5cscPnyYqVOnGsFzp06dOHr0KFOnTuW3335j+/btLFiw4E/7ee/ePUaOHMmBAweIjIzk5MmT/PDDD0ZdeWBgINWrV6dXr158//33REZGEh4ezpIlS/j666//0WdatGhRwsLCiI6ONgZcWsr777/P+fPnGTBgAMeOHSMyMpKDBw8yZswYIiMj/1EbAUJDQ4mJieHevXsWbaeIyP9CGWsReW6NHj2aBQsWMHv2bC5fvoyTkxMvvPAC7dq1A9LKEb744guGDx9Ou3btyJ8/P126dCEpKelPr9u3b19cXV1ZvHgx48ePx8XFxRg0CdCrVy+GDx9Ow4YNSUxMZMeOHRQtWpRVq1YxZcoUevbsSVxcHG5ublSuXNmoQfb19WXy5MlMnTqV+fPn89JLLzFkyBCCg4Mf2xZbW1tiY2P56KOPuHHjBnny5KFq1aoMGjTI6OOsWbOYOXMm48aN4/r16+TNmxcfHx+6du36jz7PQYMGMX78eOrWrUtycvLfmgrw7ypVqhQrV65k2rRpdOnShcTERDw8PAgMDPxHpTLly5fn3XffZfjw4cTExNCiRQsmTJhgsXaKiPwvrFKftMBOREREREQMKgUREREREbEABdYiIiIiIhagwFpERERExAIUWIuIiIiIWIACaxERERERC1BgLSIiIiJiAQqsRUREREQsQIG1iIiIiIgFKLAWEREREbGA/wd8PLqnUpnMDAAAAABJRU5ErkJggg==\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Saving and Loading the model"
      ],
      "metadata": {
        "id": "jZBE_-4PEo6Q"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "torch.save(bert_classifier,'/content/drive/MyDrive/NLP/TweetEval/1-3ep0.2(T1nodot)')"
      ],
      "metadata": {
        "id": "pdrnSGF5djSk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# modeLoaded = torch.load('/content/drive/MyDrive/NLP/TweetEval/1-3ep0.2(T1nodot)')"
      ],
      "metadata": {
        "id": "bT67d_-bVUO6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Notebook References"
      ],
      "metadata": {
        "id": "8qzyomLvFff4"
      }
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "25nAA8hzYQfM"
      },
      "source": [
        "ref: \n",
        "\n",
        "https://chriskhanhtran.github.io/_posts/2019-12-25-bert-for-sentiment-analysis/\n",
        "\n",
        "http://mccormickml.com/2019/07/22/BERT-fine-tuning/\n",
        "\n",
        "https://colab.research.google.com/drive/1pTuQhug6Dhl9XalKB0zUGf4FIdYFlpcX#scrollTo=8bwa6Rts-02-\n",
        "\n"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "machine_shape": "hm",
      "name": "3ep-0.2 (arch1) T1 nodot.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "7f5AAk7iDvbE"
      ],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "a8a062ed231e4a0daf3f41c470c23b3b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_1069a16a29f04d56951b7a3d01150707",
              "IPY_MODEL_e465bdc6d7b346cb8522acbeff9cefd3",
              "IPY_MODEL_e4bc0477876342faac0df63f9fa303ce"
            ],
            "layout": "IPY_MODEL_b429ee056980477ab621c2ae950aca7f"
          }
        },
        "1069a16a29f04d56951b7a3d01150707": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_00372b91338b4dbbbd231fb02fb61fa1",
            "placeholder": "​",
            "style": "IPY_MODEL_765632c58ee44919a89bd2a542cb80da",
            "value": "Downloading: 100%"
          }
        },
        "e465bdc6d7b346cb8522acbeff9cefd3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f9098bcf9e774f26894e9d2a672c4a3c",
            "max": 213450,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c7e928052f2f47a1b83026e6fdcb224a",
            "value": 213450
          }
        },
        "e4bc0477876342faac0df63f9fa303ce": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_330820ecf7c34b128f9b2e83a8412e2a",
            "placeholder": "​",
            "style": "IPY_MODEL_9f7dc654b9cb4799926c5a93c781159f",
            "value": " 208k/208k [00:00&lt;00:00, 593kB/s]"
          }
        },
        "b429ee056980477ab621c2ae950aca7f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "00372b91338b4dbbbd231fb02fb61fa1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "765632c58ee44919a89bd2a542cb80da": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f9098bcf9e774f26894e9d2a672c4a3c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c7e928052f2f47a1b83026e6fdcb224a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "330820ecf7c34b128f9b2e83a8412e2a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9f7dc654b9cb4799926c5a93c781159f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1d0287541cb54fc993db00f9b987fbc3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_46e59d7920fc406b9f3cbcff6ba5e141",
              "IPY_MODEL_0509afd5dd2b461ea2e9f827a69d529a",
              "IPY_MODEL_36931e3d18df40e4bcbbeb9998b0884b"
            ],
            "layout": "IPY_MODEL_349f0f7a50694aa3a4417bc721738750"
          }
        },
        "46e59d7920fc406b9f3cbcff6ba5e141": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_527db8917b9244b08c4a351c5aeddcec",
            "placeholder": "​",
            "style": "IPY_MODEL_91ae3fcd960f4e178ea31183bbc0fc77",
            "value": "Downloading: 100%"
          }
        },
        "0509afd5dd2b461ea2e9f827a69d529a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d6226210893942628dcd773c47b28fbf",
            "max": 29,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_f44316f1b50c447f8b70492acb049e1e",
            "value": 29
          }
        },
        "36931e3d18df40e4bcbbeb9998b0884b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_60c8b82b83504d2495a21bd6cf1fe403",
            "placeholder": "​",
            "style": "IPY_MODEL_3adad464c0d4424cba51936e8b61fb17",
            "value": " 29.0/29.0 [00:00&lt;00:00, 1.08kB/s]"
          }
        },
        "349f0f7a50694aa3a4417bc721738750": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "527db8917b9244b08c4a351c5aeddcec": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "91ae3fcd960f4e178ea31183bbc0fc77": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d6226210893942628dcd773c47b28fbf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f44316f1b50c447f8b70492acb049e1e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "60c8b82b83504d2495a21bd6cf1fe403": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3adad464c0d4424cba51936e8b61fb17": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "20c081492ba74d22895f3f20d9135451": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_79240864f44549cd90365eba5006613f",
              "IPY_MODEL_933e00a9ac5d4c39a3fd18020e0790e1",
              "IPY_MODEL_0877fc1a21694589bd38230e3db012fb"
            ],
            "layout": "IPY_MODEL_6ef711ca8e3d4815886045fe85338be2"
          }
        },
        "79240864f44549cd90365eba5006613f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_82513b3e807b44b5851f6a478d1369e6",
            "placeholder": "​",
            "style": "IPY_MODEL_a86fe9a63b18453cb975bef570150acc",
            "value": "Downloading: 100%"
          }
        },
        "933e00a9ac5d4c39a3fd18020e0790e1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_be072f8e70b346a4ac6b40ed71034522",
            "max": 570,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_fc6da1d47e2b43c08bde6bee6aa517ad",
            "value": 570
          }
        },
        "0877fc1a21694589bd38230e3db012fb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_47e3880d747f44bb9effddb716b0764c",
            "placeholder": "​",
            "style": "IPY_MODEL_42e10351fad546ca9596c892e1d9e044",
            "value": " 570/570 [00:00&lt;00:00, 22.3kB/s]"
          }
        },
        "6ef711ca8e3d4815886045fe85338be2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "82513b3e807b44b5851f6a478d1369e6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a86fe9a63b18453cb975bef570150acc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "be072f8e70b346a4ac6b40ed71034522": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fc6da1d47e2b43c08bde6bee6aa517ad": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "47e3880d747f44bb9effddb716b0764c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "42e10351fad546ca9596c892e1d9e044": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "27a2d3884820459e95fe058b89a7d5ea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_1d24d87ad6864e66a7a1106ba2ab6390",
              "IPY_MODEL_3d3f45fbf29948bb86a4294d09b57786",
              "IPY_MODEL_fe3e0409926748d383f12b4c2a865a5c"
            ],
            "layout": "IPY_MODEL_6709b1766d5b481890423c42486ecf08"
          }
        },
        "1d24d87ad6864e66a7a1106ba2ab6390": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a976d3c3cf184716b9d7336116496991",
            "placeholder": "​",
            "style": "IPY_MODEL_b965186f58d040f097eba6654ebad146",
            "value": "Downloading: 100%"
          }
        },
        "3d3f45fbf29948bb86a4294d09b57786": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0f7ee05a92814c0d9cdc58450d89c268",
            "max": 435779157,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_5d51d3a566ed4f69a7201cf37b96567c",
            "value": 435779157
          }
        },
        "fe3e0409926748d383f12b4c2a865a5c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_79969b2ef8bc49a68f8d4506428d91f8",
            "placeholder": "​",
            "style": "IPY_MODEL_a124bc03d6d6427093710457045f33a9",
            "value": " 416M/416M [00:07&lt;00:00, 57.3MB/s]"
          }
        },
        "6709b1766d5b481890423c42486ecf08": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a976d3c3cf184716b9d7336116496991": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b965186f58d040f097eba6654ebad146": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0f7ee05a92814c0d9cdc58450d89c268": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5d51d3a566ed4f69a7201cf37b96567c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "79969b2ef8bc49a68f8d4506428d91f8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a124bc03d6d6427093710457045f33a9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}