fake-covid-zero-shot.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "! pip install transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "V4vJIZJeclwP",
    "outputId": "b20006dd-8811-4d0c-b81e-faad120f1a29",
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cloning into 'space-model'...\n",
      "remote: Enumerating objects: 50, done.\u001b[K\n",
      "remote: Counting objects: 100% (50/50), done.\u001b[K\n",
      "remote: Compressing objects: 100% (37/37), done.\u001b[K\n",
      "remote: Total 50 (delta 17), reused 39 (delta 10), pack-reused 0\u001b[K\n",
      "Unpacking objects: 100% (50/50), 27.83 MiB | 4.41 MiB/s, done.\n"
     ]
    }
   ],
   "source": [
    "! rm -rf space-model\n",
    "! git clone https://github.com/StepanTita/space-model.git"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-31T18:43:02.255921Z",
     "start_time": "2024-01-31T18:43:02.213615Z"
    },
    "id": "LFX6x0OIc6uP",
    "tags": []
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "\n",
    "sys.path.append('space-model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-31T18:43:04.543784Z",
     "start_time": "2024-01-31T18:43:02.255892Z"
    },
    "id": "EzQpePJ4cXPG",
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "\n",
    "from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score\n",
    "\n",
    "from tqdm import tqdm\n",
    "\n",
    "from transformers import AutoModel, AutoTokenizer, AutoModelForSequenceClassification\n",
    "\n",
    "from datasets import Dataset, DatasetDict\n",
    "\n",
    "from space_model.model import *\n",
    "from space_model.loss import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-31T18:43:04.546365Z",
     "start_time": "2024-01-31T18:43:04.546281Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "device_id = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-31T18:43:05.102250Z",
     "start_time": "2024-01-31T18:43:04.549261Z"
    },
    "id": "dPxAI9XscXPH",
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda', index=0)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device = torch.device(f'cuda:{device_id}' if torch.cuda.is_available() else 'cpu')\n",
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-31T18:43:05.102508Z",
     "start_time": "2024-01-31T18:43:05.101798Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "torch.cuda.set_device(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-31T18:43:05.158823Z",
     "start_time": "2024-01-31T18:43:05.101871Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>The CDC currently reports 99031 deaths. In gen...</td>\n",
       "      <td>real</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>States reported 1121 deaths a small rise from ...</td>\n",
       "      <td>real</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Politically Correct Woman (Almost) Uses Pandem...</td>\n",
       "      <td>fake</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>#IndiaFightsCorona: We have 1524 #COVID testin...</td>\n",
       "      <td>real</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Populous states can generate large case counts...</td>\n",
       "      <td>real</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6416</th>\n",
       "      <td>A tiger tested positive for COVID-19 please st...</td>\n",
       "      <td>fake</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6417</th>\n",
       "      <td>???Autopsies prove that COVID-19 is??� a blood...</td>\n",
       "      <td>fake</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6418</th>\n",
       "      <td>_A post claims a COVID-19 vaccine has already ...</td>\n",
       "      <td>fake</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6419</th>\n",
       "      <td>Aamir Khan Donate 250 Cr. In PM Relief Cares Fund</td>\n",
       "      <td>fake</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6420</th>\n",
       "      <td>It has been 93 days since the last case of COV...</td>\n",
       "      <td>real</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6420 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  tweet label\n",
       "id                                                           \n",
       "1     The CDC currently reports 99031 deaths. In gen...  real\n",
       "2     States reported 1121 deaths a small rise from ...  real\n",
       "3     Politically Correct Woman (Almost) Uses Pandem...  fake\n",
       "4     #IndiaFightsCorona: We have 1524 #COVID testin...  real\n",
       "5     Populous states can generate large case counts...  real\n",
       "...                                                 ...   ...\n",
       "6416  A tiger tested positive for COVID-19 please st...  fake\n",
       "6417  ???Autopsies prove that COVID-19 is??� a blood...  fake\n",
       "6418  _A post claims a COVID-19 vaccine has already ...  fake\n",
       "6419  Aamir Khan Donate 250 Cr. In PM Relief Cares Fund  fake\n",
       "6420  It has been 93 days since the last case of COV...  real\n",
       "\n",
       "[6420 rows x 2 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df = pd.read_csv('data/covid_train.csv', index_col=0)\n",
    "train_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-31T18:43:05.351513Z",
     "start_time": "2024-01-31T18:43:05.332943Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Our daily update is published. States reported...</td>\n",
       "      <td>real</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Alfalfa is the only cure for COVID-19.</td>\n",
       "      <td>fake</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>President Trump Asked What He Would Do If He W...</td>\n",
       "      <td>fake</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>States reported 630 deaths. We are still seein...</td>\n",
       "      <td>real</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>This is the sixth time a global health emergen...</td>\n",
       "      <td>real</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2136</th>\n",
       "      <td>#CoronaVirusUpdates: State-wise details of Tot...</td>\n",
       "      <td>real</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2137</th>\n",
       "      <td>Tonight 12(midnight) onwards Disaster Manageme...</td>\n",
       "      <td>fake</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2138</th>\n",
       "      <td>296 new cases of #COVID19Nigeria; Plateau-85 E...</td>\n",
       "      <td>real</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2139</th>\n",
       "      <td>RT @CDCemergency: #DYK? @CDCgov’s One-Stop Sho...</td>\n",
       "      <td>real</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2140</th>\n",
       "      <td>More than half of pregnant women recently admi...</td>\n",
       "      <td>real</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2140 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  tweet label\n",
       "id                                                           \n",
       "1     Our daily update is published. States reported...  real\n",
       "2                Alfalfa is the only cure for COVID-19.  fake\n",
       "3     President Trump Asked What He Would Do If He W...  fake\n",
       "4     States reported 630 deaths. We are still seein...  real\n",
       "5     This is the sixth time a global health emergen...  real\n",
       "...                                                 ...   ...\n",
       "2136  #CoronaVirusUpdates: State-wise details of Tot...  real\n",
       "2137  Tonight 12(midnight) onwards Disaster Manageme...  fake\n",
       "2138  296 new cases of #COVID19Nigeria; Plateau-85 E...  real\n",
       "2139  RT @CDCemergency: #DYK? @CDCgov’s One-Stop Sho...  real\n",
       "2140  More than half of pregnant women recently admi...  real\n",
       "\n",
       "[2140 rows x 2 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df = pd.read_csv('data/covid_test.csv', index_col=0)\n",
    "test_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-31T18:43:06.189120Z",
     "start_time": "2024-01-31T18:43:06.172372Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Chinese converting to Islam after realising th...</td>\n",
       "      <td>fake</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>11 out of 13 people (from the Diamond Princess...</td>\n",
       "      <td>fake</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>COVID-19 Is Caused By A Bacterium, Not Virus A...</td>\n",
       "      <td>fake</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Mike Pence in RNC speech praises Donald Trump’...</td>\n",
       "      <td>fake</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6/10 Sky's @EdConwaySky explains the latest #C...</td>\n",
       "      <td>real</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2136</th>\n",
       "      <td>Donald Trump wrongly claimed that New Zealand ...</td>\n",
       "      <td>fake</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2137</th>\n",
       "      <td>Current understanding is #COVID19 spreads most...</td>\n",
       "      <td>real</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2138</th>\n",
       "      <td>Nothing screams “I am sat around doing fuck al...</td>\n",
       "      <td>fake</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2139</th>\n",
       "      <td>Birx says COVID-19 outbreak not under control ...</td>\n",
       "      <td>fake</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2140</th>\n",
       "      <td>Another 4422 new coronavirus cases have been c...</td>\n",
       "      <td>real</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2140 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  tweet label\n",
       "id                                                           \n",
       "1     Chinese converting to Islam after realising th...  fake\n",
       "2     11 out of 13 people (from the Diamond Princess...  fake\n",
       "3     COVID-19 Is Caused By A Bacterium, Not Virus A...  fake\n",
       "4     Mike Pence in RNC speech praises Donald Trump’...  fake\n",
       "5     6/10 Sky's @EdConwaySky explains the latest #C...  real\n",
       "...                                                 ...   ...\n",
       "2136  Donald Trump wrongly claimed that New Zealand ...  fake\n",
       "2137  Current understanding is #COVID19 spreads most...  real\n",
       "2138  Nothing screams “I am sat around doing fuck al...  fake\n",
       "2139  Birx says COVID-19 outbreak not under control ...  fake\n",
       "2140  Another 4422 new coronavirus cases have been c...  real\n",
       "\n",
       "[2140 rows x 2 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_df = pd.read_csv('data/covid_val.csv', index_col=0)\n",
    "val_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-31T18:43:08.909098Z",
     "start_time": "2024-01-31T18:43:08.908903Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def encode_label(s: str):\n",
    "    if s == 'fake':\n",
    "        return 1\n",
    "    else:\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-31T18:43:09.428038Z",
     "start_time": "2024-01-31T18:43:09.384155Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>The CDC currently reports 99031 deaths. In gen...</td>\n",
       "      <td>real</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>States reported 1121 deaths a small rise from ...</td>\n",
       "      <td>real</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Politically Correct Woman (Almost) Uses Pandem...</td>\n",
       "      <td>fake</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>#IndiaFightsCorona: We have 1524 #COVID testin...</td>\n",
       "      <td>real</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Populous states can generate large case counts...</td>\n",
       "      <td>real</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10695</th>\n",
       "      <td>#CoronaVirusUpdates: State-wise details of Tot...</td>\n",
       "      <td>real</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10696</th>\n",
       "      <td>Tonight 12(midnight) onwards Disaster Manageme...</td>\n",
       "      <td>fake</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10697</th>\n",
       "      <td>296 new cases of #COVID19Nigeria; Plateau-85 E...</td>\n",
       "      <td>real</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10698</th>\n",
       "      <td>RT @CDCemergency: #DYK? @CDCgov’s One-Stop Sho...</td>\n",
       "      <td>real</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10699</th>\n",
       "      <td>More than half of pregnant women recently admi...</td>\n",
       "      <td>real</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10700 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   tweet label\n",
       "0      The CDC currently reports 99031 deaths. In gen...  real\n",
       "1      States reported 1121 deaths a small rise from ...  real\n",
       "2      Politically Correct Woman (Almost) Uses Pandem...  fake\n",
       "3      #IndiaFightsCorona: We have 1524 #COVID testin...  real\n",
       "4      Populous states can generate large case counts...  real\n",
       "...                                                  ...   ...\n",
       "10695  #CoronaVirusUpdates: State-wise details of Tot...  real\n",
       "10696  Tonight 12(midnight) onwards Disaster Manageme...  fake\n",
       "10697  296 new cases of #COVID19Nigeria; Plateau-85 E...  real\n",
       "10698  RT @CDCemergency: #DYK? @CDCgov’s One-Stop Sho...  real\n",
       "10699  More than half of pregnant women recently admi...  real\n",
       "\n",
       "[10700 rows x 2 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df = pd.concat([train_df, val_df, test_df], axis=0, ignore_index=True)\n",
    "train_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-31T18:43:10.835070Z",
     "start_time": "2024-01-31T18:43:10.833588Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>The CDC currently reports 99031 deaths. In gen...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>States reported 1121 deaths a small rise from ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Politically Correct Woman (Almost) Uses Pandem...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>#IndiaFightsCorona: We have 1524 #COVID testin...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Populous states can generate large case counts...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10695</th>\n",
       "      <td>#CoronaVirusUpdates: State-wise details of Tot...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10696</th>\n",
       "      <td>Tonight 12(midnight) onwards Disaster Manageme...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10697</th>\n",
       "      <td>296 new cases of #COVID19Nigeria; Plateau-85 E...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10698</th>\n",
       "      <td>RT @CDCemergency: #DYK? @CDCgov’s One-Stop Sho...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10699</th>\n",
       "      <td>More than half of pregnant women recently admi...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10700 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   tweet  label\n",
       "0      The CDC currently reports 99031 deaths. In gen...      0\n",
       "1      States reported 1121 deaths a small rise from ...      0\n",
       "2      Politically Correct Woman (Almost) Uses Pandem...      1\n",
       "3      #IndiaFightsCorona: We have 1524 #COVID testin...      0\n",
       "4      Populous states can generate large case counts...      0\n",
       "...                                                  ...    ...\n",
       "10695  #CoronaVirusUpdates: State-wise details of Tot...      0\n",
       "10696  Tonight 12(midnight) onwards Disaster Manageme...      1\n",
       "10697  296 new cases of #COVID19Nigeria; Plateau-85 E...      0\n",
       "10698  RT @CDCemergency: #DYK? @CDCgov’s One-Stop Sho...      0\n",
       "10699  More than half of pregnant women recently admi...      0\n",
       "\n",
       "[10700 rows x 2 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df['label'] = train_df['label'].apply(encode_label)\n",
    "train_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-31T18:43:12.445871Z",
     "start_time": "2024-01-31T18:43:12.357151Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: >"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAGFCAYAAAASI+9IAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/OQEPoAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAojUlEQVR4nO3deXxcZaH/8c8smcmeNFuT7gttuu87a5GlbAoCooIgolcUcUGviF5AL16Bi15BQVEE8SerAiIgFKWVlrWFLtBC071NmjR7mj2z//6YMqVAS5tM8sw55/t+vfJqO07H7yT0fOd5zjnP44rFYjFEREQAt+kAIiKSOlQKIiKSoFIQEZEElYKIiCSoFEREJEGlICIiCSoFERFJUCmIiEiCSkFERBJUCiIikqBSEBGRBJWCiIgkqBRERCRBpSAiIgkqBRERSVApiIhIgkpBREQSVAoiIpKgUhARkQSVgoiIJKgUREQkQaUgIiIJKgUREUlQKYiISIJKQUREElQKIiKSoFIQy7vrrrsYNWoU6enpzJ8/n9WrV5uOJGJZKgWxtEcffZRrrrmGG2+8kbVr1zJ9+nROP/106uvrTUcTsSRXLBaLmQ4h0lvz589n7ty53HnnnQBEo1GGDx/O1VdfzQ9+8APD6USsRyMFsaxgMMiaNWs45ZRTEo+53W5OOeUUXnvtNYPJRKxLpSCW1djYSCQSYfDgwQc9PnjwYGpraw2lErE2lYKIiCSoFMSyioqK8Hg81NXVHfR4XV0dpaWlhlKJWJtKQSzL5/Mxe/Zsli1blngsGo2ybNkyFi5caDCZiHV5TQcQ6YtrrrmGyy67jDlz5jBv3jxuv/12Ojs7ufzyy01HE7EklYJY2kUXXURDQwM33HADtbW1zJgxg6VLl37o5LOIHBndpyAiIgk6pyAiIgkqBRERSVApiIhIgkpBREQSVAoiIpKgUhARkQSVgoiIJKgUREQkQXc0i+10ByM0dQZo7gzS0hWitTtEW3eItp4QPcEIoWiMUDhKOBojGIkmfh+JxkjzuPGnufF73fi8bvxeD35v/M856V4KsvwUZKXt/9VHXkaa6bcrklQqBbGc+vYe9rR0s6elm6rmrv2/76K6pZu9rT10hyIDliXN4yI/00dRtp+h+RmMKMhkREEGIwozGVGQybBBmaSneQYsj0hfaZkLSVktnUEqatupqG1jc207m2rb2VrXTldw4A76feVyQUmOn3ElOUwsy2FiWS4Ty3IZV5KN16PZW0k9KgVJCR2BMGt3t/Dm7hbeqtpHRW0bdW0B07H6jc/j5piSbCYNyWXKkFzmjCpgYlkuHrfLdDRxOJWCGFGzr5s3d7fw5q5m3tzVwua6diJRZ/+nmOP3MnPkIOaPLmDuqAKmD8/D79XUkwwslYIMiO5ghNd2NPLvigZWbGmgsrnLdKSU5/O6mT4sj+PHFXPyhBKmDM0zHUkcQKUg/WZHQwf/3tzAi5vrWb2zmUA4ajqSpQ3O9bO4vISTJ5Rw3LgiMn26TkSST6UgSbVpbxtPv1XDsxv2sqtJo4H+4vO6WTCmkFMnDebMKaUUZvtNRxKbUClIn21v6ODpt2p45u29bKvvMB3HcTxuF4vGFnL2tDKWTCnTvRPSJyoF6ZX69h6eWFvNU+treHdvm+k4sp/P6+YTE0o4d+ZQFpeX4PPqslc5OioFOWKxWIyXtjby0KpKXthUR9jhVwulusIsHxfOGc7F80cwvCDTdByxCJWCfKyG9gB/ebOKR96opKq523QcOUpuF5wwvphL5o/k5AkluHUvhByGSkEOaW1lC/e+tJN/vltLKKL/TOxgaH4Gn5s3nIvmjqA4Ryen5cNUCnKQWCzGsk31/G7ldt7Y1WI6jvQTv9fNBbOH8dUTxjKiUFNLcoBKQQAIRaI8ua6a36/cwVZdQeQYHreLs6aW8bWTxjKxLNd0HEkBKgWH6w5GeHDVbv7w0k5q23pMxxGDFpcX8/XFxzB3VIHpKGKQSsGhAuEID62q5Dcvbqeh3b4Lz8nRWzimkO8vKWfmiEGmo4gBKgWHiURjPLamijte2EpNq0YGcmhLJpfyn0vKGVucbTqKDCCVgoMs3VjLz/+5WXcdyxHzuF1cOHsY3z5lPKV56abjyABQKTjAOzWt/Pipd3Q1kfRaepqbLy4azVWLx5KTrmU07EylYGOtXSFu+2cFD62qRDcfSzIU5/j50ZkTOXfmUNNRpJ+oFGwoGo3xyBtV/Pyfm2nuDJqOIzY0f3QBN507hfGDc0xHkSRTKdjMusoWbnzqHd7e02o6itic1+3i8mNH8e1TxpPl194OdqFSsImuYJhbnqvgz6/vRj9RGUiluenceM4kzphaZjqKJIFKwQZe297EtY+/rS0uxaizp5Vx06emMCjLZzqK9IFKwcK6gmFufraCB1ZpdCCpoSjbz82fnsqpkwabjiK9pFKwqNe2N/H9x9/SUtaSkj49ayg3njNZu8BZkErBYoLhKLc8V8EfX92p0YGktNLcdG69YBonji82HUWOgkrBQnY3dfKNh9axoVpXFok1uFzwlePH8P3Ty/F6tDWoFagULOKZt2u47vENtAfCpqOIHLXZIwdx5+dnUpaXYTqKfAyVQorrCUX4ydPv8vDqStNRRPqkIMvHLz4zncXlJaajyGGoFFLYjoYOvv7gWipq201HEUkKlwuuPHEs3zutHI/2ik5JKoUU9eLmeq5+eB3tPZouEvuZN7qA3148i8Js7ROdalQKKeielTu4ZWkFEa1iJzY2bFAG9142l/JSrZ+USlQKKSQYjvLDv23gsTV7TEcRGRDZfi+/+twMTp6gm91ShUohRTS0B7jygTWs2a09D8RZ3C744ZkT+fLxY0xHEVQKKWHT3ja+/Kc3qd6nu5PFuT47dzg3nTuFNN3PYJRKwbDVO5u54k9v6ISyCLBwTCG/v3S2dnczSKVg0Avv1vGNh9fSE4qajiKSMiaV5XL/l+ZSkqM9oU1QKRjy1zeruO6JDYR1hZHIh4woyOTPV8xjZGGW6SiOo1Iw4Pcrt3PzcxVa0E7kMEpy/Dzw5fna8nOAqRQG2M3PbeJ3K3aYjiFiCYMy0/jTl+YxbVi+6SiOoVIYQDc98y73vrzTdAwRS8nxe/nj5XOZM6rAdBRH0LVfA+TmZzepEER6oT0Q5vI/vsH6qn2moziCSmEA3Lq0gt+t1JSRSG+1B8Jceu8qNmovkX6nUuhnv/jnZn774nbTMUQsr60nzBfuXUVFbZvpKLamUuhHv/zXFn69fJvpGCK20dIV4pI/rGJbvZaT7y8qhX7yh5d2cMeyraZjiNhOY0eQz9+zip2Nnaaj2JJKoR889VYN//PsJtMxRGyrvj3ApfetoqE9YDqK7agUkuzV7Y187y9v6cY0kX5W1dzNFX96g+5gxHQUW1EpJFFFbRtf/fMaghGtZSQyEN7e08rVD6/VhlRJpFJIkpp93XzxPq12KjLQXthUz4+fesd0DNtQKSRBa3eIL/5xNbVtPaajiDjSn1/fzd0rdOl3MqgU+igSjXH1w+vYUtdhOoqIo926tIJn3q4xHcPyvKYDWN0tz21i5ZYG0zFsb9/LD9L6ysMHPeYtGMbQr9xNpLud1pcfpHvXOiJtDbgz8sgcv4D84y/B7T/00sv7Xn6Qzk0vEWlvwOX24is9hvwTLsU/pPxDz42FQ+z98zWE6ndS9sVf4RusrSNTTSwG//nXtxlTlM2kIbmm41iWSqEPnli7h3te0npGAyWtaASDL/qfAw+44wPdSEcTkY5mBi3+EmmFIwi31dP8/F1E2psoPu+Hh369gqEUnHol3vxSYqEA7W/+nbpHr2foV+/Bk5l30HNbXrwPT3YBoXr9vFNZdyjClQ+s4elvHEdepnZv6w1NH/XSxupWrntig+kYzuL24MkedOBr/4HbVzyK4vN+SOYx80kbVEbGyOnkn3ApXdtXE4se+nLFrEknkTFqBmn5pfiKRzLo5C8TC3YR/MCBv3v7m3TvXMegxVf069uT5Khs7uKbj6wjqiuSekWl0AstnUGufGANgbAuPR1I4ZYa9tx1KdV3X0HD07cRbqs/5HOjgU7cvkxcbs8RvXYsEqJ9/VJc/ix8JaMTj0c6W2ha+muKzv4u7jR/n9+DDIwVWxr45QtbTMewJE0fHaVoNMY3H1nHnpZu01EcxV9WTuGZ3yGtYCiRjmZaX3mY2gevZciX7sLtzzzouZGuVlpffYTsGUs+9nW7tq2m8an/JRYK4MkexOCLbkqMQGKxGI3/uJ3smWfgLxtHuLWuX96b9I87/72NacPyOXXSYNNRLEUjhaP02xXbeWlro+kYjpMxdg5ZE47DVzKajDGzKbnwx0R7OumsePmg50UDXdQ/9hPSCkeQf+znP/Z100dMo+zyX1F6yW2kj55Nw99vJdK5D4D2NU8TC3aTt+DC/nhL0s9iMbjm0fXs0hpJR0U7rx2F9VX7uOC3rxLWXGVK2Pun75A+ajqDTvwisL8Q/nIDrjQ/JRfciMvrO+rXrP79V8ieeip5Cz9D/RM/pXvb6oOfEIuCy03W5JMoOuuaJLwL6W/Th+Xx+NcW4fXoM/CR0PTREeoIhPnWI+tUCCkiGuwmvG8vnqzF8T8Huqj7y/W4PGkUn399rwoBgFiMWCQEQMEp/0H0+EsS/1Oko5n6v9xA0aeuxV/24ctWJTW9taeV21/YyvdO18/sSKgUjtANT25kd1OX6RiO1bL8XjKOmYc3r4RwezOtLz8Y/8Q+6cR4ITx6PbFwgKKzv0cs0E0kED/n487MTZxsrr7nSgadeCmZ4xcRDfbQ+tqjZB4zH092AZHuNtrXPkO4vYnM8uMA8OaWHJQh7MsAIC2/DG9u0QC+e+mr37y4jRPLi5mrfZ4/lkrhCPx9fTVPrKs2HcPRwu2NND59G5HuNjwZefiHTaL0C7/Ak5lHT+XbBPduBqDm91856O8NvfJevHnxE43h5j1EA/Fid7ndhJr30PDksv2vmYuvdBylF9+Kr3jkwL456XfRGHz7kfU89+3jyU3X/QuHo3MKH6OquYsz73iJ9oAWuhOxunNnDOH2z840HSOl6czLx/j+Y2+rEERs4sn1Nfx9vUb9h6NSOIxHVlfy2o4m0zFEJIl+/NQ7NHVox7ZDUSkcQn1bDz/TlpoittPSFeKmZ941HSNlqRQO4fq/b6RNG+aI2NKT62tYodWNP5JK4SM8t2Evz7+jJQ1E7Oy/ntyg/Z0/gkrhA1q7Qtygrf1EbK+quVuL5n0ElcIH3LK0goZ2nYQScYJ7X97JxupW0zFSikrhfd6paeXRNypNxxCRARKJxvjR3zag27UOUCm8z38//S5a2kjEWd7a08qTunchQaWw33Mb9rJqZ7PpGCJiwG1LN9MT0klnUCkAEAhH+NlzuidBxKlqWnv4/codpmOkBJUC8IeXdlLVrJ3URJzs7hXbqW/rMR3DOMeXQn17D7/59zbTMUTEsK5ghNue32w6hnGOL4VfL9tGp25gERHg8bV7HH+JqqNLoWZfN4++UWU6hoikiGgM/u9fzr6hzdGl8Ovl2whGoqZjiEgKWV5Rz9t79pmOYYxjS6GquYvH1miUICIfdvsLW01HMMaxpXDn8m2EIrpTTUQ+zMmjBUeWQmVTF4+v3WM6hoiksDscOlpwZCn8avlWwlrPQkQOY1lFPRv2OO9KJMeVQl1bj/ZoFZEjcscy512J5LhSuP/VXTqXICJHZFlFPdvqO0zHGFCOKoXuYISHVmlpbBE5MrFYfM8FJ3FUKTy2porW7pDpGCJiIX9bt4fmzqDpGAPGMaUQi8W475VdpmOIiMX0hKI8vNo5MwyOKYUXNtWzs7HTdAwRsaAHXt9N2CGrHzimFO5z2LygiCTP3tYenn+nznSMAeGIUtjZ2MlrO5pMxxARC/vTa7tMRxgQjiiFv76pNY5EpG9W72xmlwOmoG1fCpFoTEtaiEhSPLbG/scS25fCii311LUFTMcQERt4Yu0eojZfIsf2paBNdEQkWWpae3hle6PpGP3K1qXQ2BFgeUW96RgiYiN2n0KydSk8ua5a6xyJSFI9/04t7T32XRnB1qXw9Nt7TUcQEZvpCUV5xsbHFtuWQvW+bt6q2mc6hojY0LMbVAqW85yNf2giYtbrO5po7bLnFJJtS2HpxlrTEUTEpkKRGMsq7LnshS1Loa6thzWVLaZjiIiN2fWDpy1LYenGWmK66EhE+tHKrQ10ByOmYySdLUvhuY06nyAi/asnFGXFFvvdB2W7UmjtDvHGLk0diUj/s+Ny2rYrhVe2NRKx+dokIpIaVmxpIGazuWrblcLKLQ2mI4iIQzR3Bnmnps10jKRSKYiI9MGrNlsgz1alsKOhg5rWHtMxRMRBXtlmr10dbVUKr2y31w9HRFLfG7uaCUWipmMkja1K4dVt9hrGiUjq6wpGWLvbPlc82qoUVu9sNh1BRBzITrMUtimF3U2dNHUGTccQEQd6fYdKIeWs1zLZImLIxupW29wfZZtSWFe5z3QEEXGormCEbfUdpmMkhX1KQSMFETHILpt62aIUAuEIm2x2V6GIWMtbe/aZjpAUtiiFd2vaCNroOmERsZ6397SajpAUtigFnWQWEdMqatsIhK2/v4ItSqFib7vpCCLicKFIjHdtMI1ti1LY1mCPs/4iYm1b6qz/AdUepWCTS8FExNp2NHSajtBnli+F+rYeWrtDpmOIiLDdBrMWli+FrRoliEiK2K6RgnmaOhKRVFHV3EUwbO3L4y1fClvrrX9iR0TsIRyNsbvJ2qMFy5fCrsYu0xFERBKsfl7B8qVQ09ptOoKISMLuJmt/ULV8KdRqT2YRSSG1bdY+Jlm6FFq7QnQFrX9buYjYR51KwZy9bZo6EpHUUtcWMB2hT6xdCpo6EpEUo5GCQTqfICKppl4jBXM0UhCRVBOMRGnuDJqO0WuWLoXWLut+40XEvqw8hWTpUmjrCZuOICLyIVZepNPSpdDeY91vvIjYV4eFP7BauhQ0UhCRVNQZtO6xydKl0K5SEJEU1BGw7rHJ4qWg6SMRST2dKgUzNFIQkVTUEbDu8juWLoUuC8/biYh9aaRgSCQaMx1BRORDrPyB1dKloEoQkVRk5Q+s1i4F637fRcTGLNwJ1i2FqJW/6yJia1Y+PnlNB+itqIYJ0ktZ3ggL81qZld3EBF89I2N7KQ5Vkxa29obrkjqCOacBM0zH6BULl4LpBJLKPK4os/PamZvTwmR/A6Ndexkc2kNu12487dW4OqOgDpB+kjF8hukIvWbZUojpNLMAE7K7mJ/bzLSMRsa6aymLVJPfXYmvrRJXTwCsu1ilWJnbYzpBr1m2FNLclj0dIkepLD3IsXnNTM9sZLy3gaHRagp7Kklv34Ur2AHNphOKfIDbsodW65aC2+0iPc1NTyhqOookQY43zKL8VmZlNVHuq2dkrIai4B6y2nfh7m6EVuJfIlagUjAj0+elJ6SNdqwizR1jbl4bc3KamORrYJSrlsGhPeR07sLTXoOrIwYdplOKJIFKwYyMNOvO29nZlJwO5uW2MCU9Ps8/JFJNXtdu0toqcXWHoNt0QpF+pnMKZmT5rfuNt7ph6QEW5bcwPbORcZ46hkarKeip2j/P3wlNphOKGOTLMp2g1yxdChk+S8dPeYPSwizI38esrEbK0xoYEauhKFBFZscu3N3NsI/4l4gcLKvYdIJes/RRNVPTR33md0eZl9/GnOxmJvrqGbn/ev6cjt24O/biao9Bu+mUIhaTU2o6Qa9ZuhSy/JaOP2BcrhjTcjqZl9PMlPRGxrj3UhauJrdrN2ntVbi6wtBlOqWIjWQPNp2g1yx9VC3O8ZmOkFJGZ3SzIL+FaRlNjPPUMjRaw6DuSvztu3EFuzTPLzJQsktMJ+g1a5dCtt90hAFX6AuxMG8fM7MbKffWMzy2l6JAJRntu3H3tEAL8S8RMUcjBTOKc9NNR+gXGZ4I8/LamJPTzIS0eka59lISrCK7Yzeeztr4HL/m+UVSU3o+eK37gdXapWDhkYLLFWNmbgfzcpqZnN7IaNeBeX5vWxWurojm+UWsyMKjBLB6KeSkfikck9nNgrxmpmY0cYynlqGRavJ7KvG3VeIKdEPAdEIRSSoLn08Ai5dCSYqUQok/xML8FmZkNlHurWNYrIbCnioy23fhCrRqnl/ESTRSMGcgRwpZnijz8/cx673r+YnP82d17MLTWa8F20QkzsL3KIDFSyE9zUNhlo+mzuQsiudxRZmV28HcnGYmpzcw2lVL6UEbs0S0MYuIHJ6mj8waUZh51KVQntXFgrwWpqY3MtYTX7AtvjHLblyBgOb5RaT3NH1k1qjCLNZV7vvQ46X+IIv2z/OP89YxPFZDQU8lGe27cAXatTGLiPQPjRTMOq4syvjmrUzw1cU3YN8/z+/u0sYsImJAwVjTCfrE8qVwfsFOqLvRdAwREUjPg0EjTafoE+tvdFw8wXQCEZG40mmmE/SZ9UuhcBy4tIS2iKQAlUIK8PqgYLTpFCIiUDrVdII+s34pABSVm04gIgJlGimkhmKVgogY5k23xQdUe5RCySTTCUTE6YongMfyF3TapBSGzzWdQESczgZTR2CXUhg0CnKHmU4hIk5mgyuPwC6lADDqWNMJRMTJyqabTpAU9imFkSoFETHE5YbBk02nSAr7lMKo40wnEBGnKhgLvizTKZLCPqVQOBayrb25hYhY1JAZphMkjX1KAXReQUTMGLPYdIKksVcp6LyCiAw4F4w71XSIpLFXKei8gogMtLLplt9Y5/3sVQrF5ZBVbDqFiDjJuNNMJ0gqe5UCwMhFphOIiJOoFFLcSE0hicgAySyEobNNp0gqG5aCRgoiMkDGfgLc9jqM2uvdQPyuwpwhplOIiBPYbOoI7FgKLhdMPs90ChGxO5cbjvmE6RRJZ79SAJh6vukEImJ3Q+dAZoHpFElnz1IYOhsKxphOISJ2ZsOpI7BrKQBMucB0AhGxMxvdxfx+9i2FqSoFEekn2aW22T/hg+xbCsXlMHiq6RQiYkfTLoxf1GJD9i0F0AlnEekfsy4znaDf2LsUppwP2LPNRcSQEYugaJzpFP3G3qWQPwKGzzOdQkTsZNalphP0K3uXAugqJBFJnvQ8mHyu6RT9yv6lMPk8cHlMpxARO5h6IaRlmE7Rr+xfCtnFMOZE0ylExA5sfIL5PfYvBYAZF5tOICJWVzYDyqaZTtHvnFEKk86FvBGmU4iIlc22/ygBnFIKHi8s+JrpFCJiVWlZ8fMJDuCMUoD4ZWTpeaZTiIgVTT4P/DmmUwwI55SCPxvmXGE6hYhYkUOmjsBJpQAw/0rw+E2nEBErKZnsqJtgnVUKOYNh2mdMpxARKznhu6YTDChnlQLAoqvRekgickSKJ8IkZ23v67xSKC6H8aebTiEiVnDSteB21mHSWe/2PYu+aTqBiKS6wVPi9zg5jDNLYdSx8X2cRUQO5cRrbbuRzuE4sxRAowURObTSqTDxHNMpjHBuKUz8JBSMMZ1CRFLRSdc5cpQATi4Ftxs+cYPpFCKSasqmw4SzTKcwxrmlAPFb10ceazqFiKSSk64zncAoZ5cCwJKbwaVvg4gAQ2ZC+RmmUxilo2HZdO23ICJxJ/3QdALjVAoQP7fgc8YKiCJyCENnw/jTTKcwTqUAkF3iuPVNROR9XG5YcqvpFClBpfCeBV+HQaNMpxARE+Z8CYbPNZ0iJagU3uP1w2k/NZ1CRAZaThl84kbTKVKGSuH9Jp4Do443nUJEBtIZt0J6rukUKUOl8EFLbgGXx3QKERkI48+ASZ8ynSKlqBQ+qHRKfD9nEbG3tCw48zbTKVKOSuGjnPxfkJ5vOoWI9KeTfwT5w02nSDkqhY+SVQRn/cJ0ChHpL2XT43u2y4eoFA5l6gUwVfs5i9iOywPn3AFunTv8KCqFwznr55A3wnQKEUmm+V+Nr3EkH8kVi8VipkOktN2vwv1nQSxqOon0wS0vB7huWYBvzfdx+5J0du2LMvqOjo987l8uyODCyWmHfK1NDRGufSHAit1hwlGYVOzm8c9kMiLvwGes16rC/Gh5gFXVETwumFHq4flLMslIc+Ya/SkjdxhctQr82aaTpCyv6QApb+QiOPbb8PL/mU4ivfRGdYTfrQkybfCBg/bwXBd7v3vwgeH3a0Lc9mqAM8Yd+p/F9uYox/2xiytmpvGTk7LI9bt4pyFC+vv+ymtVYZY82MV1x/n59RnpeN3wVl0Ut/rAvLN+oUL4GCqFI7H4h7B9OexdbzqJHKWOYIyLn+jmnnMy+OnKQOJxj9tFafbBR+m/VYT4zKQ0sn2HPnr/aHkPZ47z8r+npiceG1tw8Czsd54P8M15Pn5wnD/xWHmR5q+NW3AVlC8xnSLl6ZzCkfCkwfl/gLRM00nkKF31bA9njfNyypjDf/5ZUxNhfW2UK2YdetooGovxj61hxhe4Of2BTkpua2f+Hzp4siKUeE59Z5RV1RFKstwsureTwT9v58T7O3m5Mpy09yS9MHwBnPrfplNYgkrhSBWN09pIFvPIxhBr90a4+RT/xz733nVBJha5WTT80OVR3xmjIwi3vBJgyVgv//xCJudNSOPTj3azYlf8oL+jJX7u6ccrAnxlVhpLL85kVqmHT/y/LrY2RZLzxuToZBXDhfeDRxMjR0KlcDTmXgHjNfy0gqrWKN9a2sODn84g3Xv4yfzuUIyHNoS4YuahRwkA0f2XZHyq3Mt3FvqZUerhB8f5OXu8l7vXBA96zldnp3H5TB8zyzz8ckk65YVu7lsXOsQrS79xueOj/Nwy00ksQ9V5tD55J/x2IXQ2mE4ih7Fmb4T6zhizfteZeCwSg5W7I9y5Okjgv3Lw7D/z+9i7IbpCcOn0w5dCUaYLrxsmFR98fmBikZuXq+KjgLLs+OesScUHf96aWOymsk1XsA24k34IY04yncJSVApHK7sYPnUXPKQb21LZJ0Z72fC1rIMeu/zv3Uwo8nDtsb5EIQDcuy7EJ8u9FGcdfuDs87iYO8TD5qaDD+5bmqOMzIu/3qh8F0NyXGxu/MBzmqKccYz+uQ2ocafBCd8zncJyNH3UG+NPhxO+bzqFHEaO38WUEs9BX1lpLgoz4o+/Z1tzlJW7I3x5lu8jX2fCnR38bdOBaZ//XOTj0Y0h7lkTZFtzlDtXB3l6c5ivz43/fZfLxX8u8vGr1UEeezfEtuYo1y/voaIxyhUzP/r/Q/pB3gg473fg0nXAR0sfXXrr5B9B8w7Y+JjpJNIH960LMizXxWljP/qS0c1NUVoDB+7vPG9iGnefHePml4N8c2kP5YVuHv9MBseNOPBP6dsL/PSE4TvP99DcHWP6YA//+kLmhy5dlX7i8cFn7ofMAtNJLEl3NPdFOAB/+iRUvW46iYi858yfw7yvmE5hWfro0hdeP3z2IRg02nQSEQGYcoEKoY9UCn2VVQgX/1X7L4iYVjwRPvkr0yksT6WQDEXj4KI/g/vwlzSKSD/JHQqXPAa+rI9/rhyWSiFZRp8QX6NdRAZWej5c8gTkDTOdxBZUCsk082I4/rumU4g4hzcdPvcIlEwwncQ2VArJdvL1MPk80ylE7M/lgfPvhZELTSexFZVCsrlccO7dMGye6SQi9nbWz2Hi2aZT2I5KoT+kpcPnHobCcaaTiNjTKT+BOV8yncKWVAr9JasILn8WSiaZTiJiLydeC8d923QK21Ip9KfsEvjiP6B0mukkIvaw6JvxnRCl36gU+ltmAVz2FAydbTqJiLXN/QqcdpPpFLanUhgIGYPgC0/GtwQUkaM38xI48zbTKRxBpTBQ0nPhC0/AqONNJxGxlgVXxTe30jLYA0KrpA60UDc8cjFsX2Y6iUiKc8X3RV/0DdNBHEWlYEI4AH+5DLY8ZzqJSGry+OC8u2HK+aaTOI5KwZRICB77Emx6ynQSkdTiz4PPPhBfT0wGnErBpGgE/vZV2PBX00lEUkPOkPhqp4Mnm07iWCoF06JR+Nf18NqdppOImFU8AS55XKudGqZSSBXrHoBnvgORoOkkIgNvxML40jAZg0wncTyVQiqpfD1+ZVJXo+kkIgNn4ifh0/fE1wwT41QKqWZfJTz8OajbaDqJSP+b9x+w5FZw65apVKFSSEXBzvgJ6E1Pm04i0j+8GbDkZ1rpNAWpFFLZK3fACz+BWMR0EpHkKZ4AF/wRBmsF4VSkUkh1u16Gv14OnfWmk4j03axL49NFvkzTSeQQVApW0LYX/vpFqHrddBKR3vHnwtm/hKkXmE4iH0OlYBWRcPx+htd/YzqJyNEZMgsuuA8KRptOIkdApWA125fDU9+C1krTSUQ+hgsWXgWn/Bg8aabDyBFSKVhRoANe+DG88QdAPz5JQZlF8QXtxp1qOokcJZWCle16BZ76BjTvMJ1E5IBRx8dvRsstM51EekGlYHWhblj+0/i5hljUdBpxMl82nPSD+KY4uhnNslQKdrHnTfj7VdBQYTqJONHkT8Pp/wO5Q0wnkT5SKdhJOAArbo3f9BYNm04jTlBUHt87ecyJppNIkqgU7GjvW/DkVVC3wXQSsStfNpz4fVjwdV1ZZDMqBbuKhODVX8HLt0OgzXQasZPJ58HpP9NUkU2pFOyuqxle+kX88tVwj+k0YmVF4/dPFZ1kOon0I5WCU7RWw4s3w/qHtMCeHJ20rPhU0cKrNFXkACoFp2ncCstvgnefQje+yWG502D6Z+Gk6yBvqOk0MkBUCk5VvRaW/Tfs+LfpJJJqvBkw+zJYdLX2S3YglYLT7VgBy34C1WtMJxHT/Lkw94r4zWfZxabTiCEqBYnb9HT8zmjd/OY8GQWw4GvxrTEz8k2nEcNUCnJANArbXohfqbTtX1o2w+6yS2HRN+JbYvqyTKeRFKFSkI/WsgvevA/WPQBdTabTSDLlj4BjvwUzvwBev+k0kmJUCnJ44QBsfCI+eqh+03Qa6S2XO7566cxL4usUebymE0mKUinIkatZD2/cAxseh3C36TRyJArHwYzPwbSLdCWRHBGVghy97hZY92B8eql5u+k08kHp+TDlfJjxeRg2x3QasRiVgvReLAY7V8avXNr8LLRVm07kXG4vHHNqfFQwfonOFUivqRQkeWrWQcWzUPEPqH/HdBpnKJ0K0z8PUy/UvQWSFCoF6R8tuw4UROVrWm8pWXw5MOpYGLMYxp4MxeNNJxKbUSlI/+tqhi1L4wWxfTmEukwnsg63F4bOjpfAmJNg2FxdOST9SqUgAyvUHT8PUbUqvrRG9ToItJpOlVqKxh8ogVHHQXqu6UTiICoFMSsWg6Zt+wti/1ftBogETScbGO40KBwLZdNh9InxItCKpGKQSkFSTzgYL4b3F0XTNiy91LfLAwWjoXgClEyCkv2/Fh6jPQokpagUxBp6WqFhC7RWxS99bd1z4KutGjobTCfczwWDRkLxRCjZ/1U8IT4llJZuOpzIx1IpiD2Eeg6UxftLo60aAh0QDcWnpCLhj/l9iMSIxOUGfw748+Lz+v6c+PLS6bmQVQxZRZBVAtkl8T+/96vuERALUymIfFAkDNFw/ODucplOIzKgVAoiIpLgNh1ARERSh0pBREQSVAoiIpKgUhARkQSVgoiIJKgUREQkQaUgIiIJKgWRPli5ciXnnHMOQ4YMweVy8eSTT5qOJNInKgWRPujs7GT69OncddddpqOIJIV26xDpgzPOOIMzzjjDdAyRpNFIQUREElQKIiKSoFIQEZEElYKIiCSoFEREJEFXH4n0QUdHB9u2bUv8eefOnaxfv56CggJGjBhhMJlI72iTHZE+ePHFF1m8ePGHHr/sssu4//77Bz6QSB+pFEREJEHnFEREJEGlICIiCSoFERFJUCmIiEiCSkFERBJUCiIikqBSEBGRBJWCiIgkqBRERCRBpSAiIgkqBRERSVApiIhIgkpBREQSVAoiIpKgUhARkQSVgoiIJKgUREQkQaUgIiIJKgUREUlQKYiISIJKQUREElQKIiKSoFIQEZEElYKIiCSoFEREJEGlICIiCSoFERFJ+P9vlgbAdjbLtAAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_df.groupby('label').size().plot(kind='pie', autopct='%.2f')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-31T19:38:35.209565Z",
     "start_time": "2024-01-31T19:38:35.208532Z"
    },
    "id": "5b6HjLMEcXPJ",
    "tags": []
   },
   "outputs": [],
   "source": [
    "EXPERIMENT_NAME = 'inter-loss'\n",
    "MODEL_NAME = 'bert-base-cased'\n",
    "MODELS_PATH = 'models'\n",
    "DATASET_NAME = 'fake-news-net-gossip'\n",
    "\n",
    "# SEED = \n",
    "NUM_EPOCHS = 10\n",
    "BATCH_SIZE = 256\n",
    "MAX_SEQ_LEN = 512\n",
    "N_LATENT = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-31T18:43:14.835275Z",
     "start_time": "2024-01-31T18:43:14.808837Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    test: Dataset({\n",
       "        features: ['tweet', 'label'],\n",
       "        num_rows: 10700\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_dataset = DatasetDict({\n",
    "    'test': Dataset.from_pandas(train_df[['tweet', 'label']]),\n",
    "})\n",
    "val_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-31T18:43:15.901447Z",
     "start_time": "2024-01-31T18:43:15.748305Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "7CdJl4UEcXPJ",
    "outputId": "ee737554-14a9-4e2c-86c9-35c190b1ec64",
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BertTokenizerFast(name_or_path='bert-base-cased', vocab_size=28996, model_max_length=512, is_fast=True, padding_side='right', truncation_side='right', special_tokens={'unk_token': '[UNK]', 'sep_token': '[SEP]', 'pad_token': '[PAD]', 'cls_token': '[CLS]', 'mask_token': '[MASK]'}, clean_up_tokenization_spaces=True),  added_tokens_decoder={\n",
       "\t0: AddedToken(\"[PAD]\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       "\t100: AddedToken(\"[UNK]\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       "\t101: AddedToken(\"[CLS]\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       "\t102: AddedToken(\"[SEP]\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       "\t103: AddedToken(\"[MASK]\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       "}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)\n",
    "tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-31T18:43:22.023281Z",
     "start_time": "2024-01-31T18:43:18.175120Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ca3299a3e5df4d8d90d6d7b164228894",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/10700 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    test: Dataset({\n",
       "        features: ['tweet', 'label', 'input_ids', 'token_type_ids', 'attention_mask'],\n",
       "        num_rows: 10700\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_dataset = val_dataset.map(lambda x: tokenizer(x['tweet'], truncation=True, padding='max_length', max_length=MAX_SEQ_LEN, return_tensors='pt'), batched=True)\n",
    "val_dataset.set_format('torch', device=device)\n",
    "val_dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Base Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-24T18:21:01.009619Z",
     "start_time": "2024-01-24T18:21:00.746135Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "state_dict = torch.load(f'{MODELS_PATH}/{DATASET_NAME}_{MODEL_NAME}_{NUM_EPOCHS}.bin')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-24T18:07:38.274348Z",
     "start_time": "2024-01-24T18:07:37.768723Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "7LtAcsA8cXPK",
    "outputId": "363eca6c-f4a9-42fa-d817-05d9d2362878",
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BertForSequenceClassification(\n",
       "  (bert): BertModel(\n",
       "    (embeddings): BertEmbeddings(\n",
       "      (word_embeddings): Embedding(28996, 768, padding_idx=0)\n",
       "      (position_embeddings): Embedding(512, 768)\n",
       "      (token_type_embeddings): Embedding(2, 768)\n",
       "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): BertEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0-11): 12 x BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (pooler): BertPooler(\n",
       "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "      (activation): Tanh()\n",
       "    )\n",
       "  )\n",
       "  (dropout): Dropout(p=0.1, inplace=False)\n",
       "  (classifier): Linear(in_features=768, out_features=2, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base_model = AutoModelForSequenceClassification.from_pretrained(MODEL_NAME, state_dict=state_dict).to(device)\n",
    "base_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-31T18:43:25.254625Z",
     "start_time": "2024-01-31T18:43:25.253363Z"
    },
    "id": "cgyRO91Ng17v"
   },
   "outputs": [],
   "source": [
    "def eval(f):\n",
    "    def wrapper(model, *args, **kwargs):\n",
    "        model.eval()\n",
    "        return f(model, *args, **kwargs)\n",
    "    return wrapper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-31T18:43:25.439654Z",
     "start_time": "2024-01-31T18:43:25.427143Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def concept_space_to_preds(concept_spaces):\n",
    "    tensor_concept_spaces = torch.cat([cs.unsqueeze(0) for cs in concept_spaces], dim=0)\n",
    "    concept_space_dist = tensor_concept_spaces.permute(1, 0, 2, 3).mean(dim=(2, 3)) # (B, n)\n",
    "    return torch.argmax(concept_space_dist, dim=1).detach().cpu().tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-31T18:43:25.882370Z",
     "start_time": "2024-01-31T18:43:25.880694Z"
    }
   },
   "outputs": [],
   "source": [
    "@eval\n",
    "def eval_epoch(model, val_dataloader):\n",
    "    val_loss = 0.0\n",
    "    val_preds = []\n",
    "    cs_val_preds = []\n",
    "    val_labels = []\n",
    "\n",
    "    with torch.no_grad():\n",
    "\n",
    "        for step, batch in enumerate(tqdm(val_dataloader, total=len(val_dataloader))):\n",
    "            ids = batch['input_ids'].to(device, dtype = torch.long)\n",
    "            mask = batch['attention_mask'].to(device, dtype = torch.long)\n",
    "            targets = batch['label'].to(device, dtype = torch.long)\n",
    "\n",
    "            outputs = model(input_ids=ids, attention_mask=mask, labels=targets)\n",
    "\n",
    "            loss, logits = outputs.loss, outputs.logits\n",
    "\n",
    "            probs = F.softmax(logits, dim=-1).cpu()\n",
    "            pred = torch.argmax(probs, dim=-1) # (B)\n",
    "            val_preds += pred.detach().tolist()\n",
    "            val_labels += [l.item() for l in targets.cpu()]\n",
    "\n",
    "            ### Distance Based Classification\n",
    "            # out.concept_spaces (n, B, seq_len, n_latent)\n",
    "            if hasattr(outputs, 'concept_spaces'):\n",
    "                cs_val_preds += concept_space_to_preds(outputs.concept_spaces)\n",
    "            ### END\n",
    "\n",
    "            val_loss += loss.item()\n",
    "    return val_loss, val_preds, val_labels, cs_val_preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-31T18:43:26.972769Z",
     "start_time": "2024-01-31T18:43:26.972647Z"
    }
   },
   "outputs": [],
   "source": [
    "val_dataloader = torch.utils.data.DataLoader(val_dataset['test'], batch_size=2 * BATCH_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-24T17:23:41.084245Z",
     "start_time": "2024-01-24T17:20:35.281063Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 21/21 [03:07<00:00,  8.91s/it]\n"
     ]
    }
   ],
   "source": [
    "val_loss, val_preds, val_labels, _ = eval_epoch(base_model, val_dataloader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-24T18:10:49.191475Z",
     "start_time": "2024-01-24T18:10:49.154637Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "val_acc = accuracy_score(val_labels, val_preds)\n",
    "val_f1 = f1_score(val_labels, val_preds, average='macro')\n",
    "val_precision = precision_score(val_labels, val_preds, average='macro')\n",
    "val_recall = recall_score(val_labels, val_preds, average='macro')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-24T18:10:52.128865Z",
     "start_time": "2024-01-24T18:10:52.122540Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val loss: 0.8171130248478481\n",
      "Val acc: 0.5233644859813084\n",
      "Val f1: 0.3435582822085889\n",
      "Val precision: 0.2616822429906542\n",
      "Val recall: 0.5\n"
     ]
    }
   ],
   "source": [
    "print(f'Val loss: {val_loss / len(val_dataloader)}')\n",
    "print(f'Val acc: {val_acc}')\n",
    "print(f'Val f1: {val_f1}')\n",
    "print(f'Val precision: {val_precision}')\n",
    "print(f'Val recall: {val_recall}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Space Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-31T19:38:39.376242Z",
     "start_time": "2024-01-31T19:38:39.189989Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BertModel(\n",
       "  (embeddings): BertEmbeddings(\n",
       "    (word_embeddings): Embedding(28996, 768, padding_idx=0)\n",
       "    (position_embeddings): Embedding(512, 768)\n",
       "    (token_type_embeddings): Embedding(2, 768)\n",
       "    (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "    (dropout): Dropout(p=0.1, inplace=False)\n",
       "  )\n",
       "  (encoder): BertEncoder(\n",
       "    (layer): ModuleList(\n",
       "      (0-11): 12 x BertLayer(\n",
       "        (attention): BertAttention(\n",
       "          (self): BertSelfAttention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (output): BertSelfOutput(\n",
       "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (intermediate): BertIntermediate(\n",
       "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          (intermediate_act_fn): GELUActivation()\n",
       "        )\n",
       "        (output): BertOutput(\n",
       "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (pooler): BertPooler(\n",
       "    (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "    (activation): Tanh()\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base_model = AutoModel.from_pretrained(MODEL_NAME)\n",
    "base_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-31T19:38:41.164691Z",
     "start_time": "2024-01-31T19:38:39.872802Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SpaceModelForSequenceClassification(\n",
       "  (base_model): BertModel(\n",
       "    (embeddings): BertEmbeddings(\n",
       "      (word_embeddings): Embedding(28996, 768, padding_idx=0)\n",
       "      (position_embeddings): Embedding(512, 768)\n",
       "      (token_type_embeddings): Embedding(2, 768)\n",
       "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): BertEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0-11): 12 x BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (pooler): BertPooler(\n",
       "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "      (activation): Tanh()\n",
       "    )\n",
       "  )\n",
       "  (space_model): SpaceModel(\n",
       "    (concept_spaces): ModuleList(\n",
       "      (0-1): 2 x Sequential(\n",
       "        (0): Linear(in_features=768, out_features=3, bias=False)\n",
       "        (1): Tanh()\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (classifier): Linear(in_features=6, out_features=2, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "space_model = SpaceModelForSequenceClassification(base_model, n_embed=768, n_latent=N_LATENT, n_concept_spaces=2, l1=0, l2=0).to(device)\n",
    "space_model.load_state_dict(torch.load(f'{MODELS_PATH}/{EXPERIMENT_NAME}/{DATASET_NAME}_space-{MODEL_NAME}-({N_LATENT})_{NUM_EPOCHS}.bin'))\n",
    "space_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-31T19:41:56.072384Z",
     "start_time": "2024-01-31T19:38:41.168894Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 21/21 [03:14<00:00,  9.28s/it]\n"
     ]
    }
   ],
   "source": [
    "val_loss, val_preds, val_labels, cs_val_preds = eval_epoch(space_model, val_dataloader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-31T19:41:56.140324Z",
     "start_time": "2024-01-31T19:41:56.077162Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "cs_val_acc = accuracy_score(val_labels, cs_val_preds)\n",
    "cs_val_f1 = f1_score(val_labels, cs_val_preds, average='macro')\n",
    "\n",
    "val_acc = accuracy_score(val_labels, val_preds)\n",
    "val_f1 = f1_score(val_labels, val_preds, average='macro')\n",
    "val_precision = precision_score(val_labels, val_preds, average='macro')\n",
    "val_recall = recall_score(val_labels, val_preds, average='macro')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-31T19:41:56.140535Z",
     "start_time": "2024-01-31T19:41:56.139846Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val loss: 0.7022485051836286\n",
      "Val acc: 0.5233644859813084\n",
      "CS Val acc: 0.5288785046728972\n",
      "Val f1: 0.3435582822085889\n",
      "CS Val f1: 0.3597856329276285\n",
      "Val precision: 0.2616822429906542\n",
      "Val recall: 0.5\n"
     ]
    }
   ],
   "source": [
    "print(f'Val loss: {val_loss / len(val_dataloader)}')\n",
    "print(f'Val acc: {val_acc}')\n",
    "print(f'CS Val acc: {cs_val_acc}')\n",
    "print(f'Val f1: {val_f1}')\n",
    "print(f'CS Val f1: {cs_val_f1}')\n",
    "print(f'Val precision: {val_precision}')\n",
    "print(f'Val recall: {val_recall}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "gpuType": "T4",
   "provenance": []
  },
  "environment": {
   "kernel": "python3",
   "name": ".m114",
   "type": "gcloud",
   "uri": "gcr.io/deeplearning-platform-release/:m114"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}