LSTM for text classification-using pytorch.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "LSTM in Pytorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: torch in c:\\users\\simmu\\anaconda3\\lib\\site-packages (1.9.0)\n",
      "Requirement already satisfied: typing-extensions in c:\\users\\simmu\\anaconda3\\lib\\site-packages (from torch) (3.10.0.0)\n",
      "Requirement already satisfied: spacy in c:\\users\\simmu\\anaconda3\\lib\\site-packages (3.1.0)\n",
      "Requirement already satisfied: typer<0.4.0,>=0.3.0 in c:\\users\\simmu\\anaconda3\\lib\\site-packages (from spacy) (0.3.2)\n",
      "Requirement already satisfied: catalogue<2.1.0,>=2.0.4 in c:\\users\\simmu\\anaconda3\\lib\\site-packages (from spacy) (2.0.4)\n",
      "Requirement already satisfied: requests<3.0.0,>=2.13.0 in c:\\users\\simmu\\anaconda3\\lib\\site-packages (from spacy) (2.24.0)\n",
      "Requirement already satisfied: packaging>=20.0 in c:\\users\\simmu\\anaconda3\\lib\\site-packages (from spacy) (20.4)\n",
      "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in c:\\users\\simmu\\anaconda3\\lib\\site-packages (from spacy) (2.0.5)\n",
      "Requirement already satisfied: blis<0.8.0,>=0.4.0 in c:\\users\\simmu\\anaconda3\\lib\\site-packages (from spacy) (0.7.4)\n",
      "Requirement already satisfied: numpy>=1.15.0 in c:\\users\\simmu\\anaconda3\\lib\\site-packages (from spacy) (1.19.5)\n",
      "Requirement already satisfied: setuptools in c:\\users\\simmu\\anaconda3\\lib\\site-packages (from spacy) (49.2.0.post20200714)\n",
      "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.9.0,>=1.7.4 in c:\\users\\simmu\\anaconda3\\lib\\site-packages (from spacy) (1.8.2)\n",
      "Requirement already satisfied: srsly<3.0.0,>=2.4.1 in c:\\users\\simmu\\anaconda3\\lib\\site-packages (from spacy) (2.4.1)\n",
      "Requirement already satisfied: pathy>=0.3.5 in c:\\users\\simmu\\anaconda3\\lib\\site-packages (from spacy) (0.6.0)\n",
      "Requirement already satisfied: jinja2 in c:\\users\\simmu\\anaconda3\\lib\\site-packages (from spacy) (2.11.2)\n",
      "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in c:\\users\\simmu\\anaconda3\\lib\\site-packages (from spacy) (4.47.0)\n",
      "Requirement already satisfied: wasabi<1.1.0,>=0.8.1 in c:\\users\\simmu\\anaconda3\\lib\\site-packages (from spacy) (0.8.2)\n",
      "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in c:\\users\\simmu\\anaconda3\\lib\\site-packages (from spacy) (3.0.5)\n",
      "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in c:\\users\\simmu\\anaconda3\\lib\\site-packages (from spacy) (1.0.5)\n",
      "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.7 in c:\\users\\simmu\\anaconda3\\lib\\site-packages (from spacy) (3.0.8)\n",
      "Requirement already satisfied: thinc<8.1.0,>=8.0.7 in c:\\users\\simmu\\anaconda3\\lib\\site-packages (from spacy) (8.0.7)\n",
      "Requirement already satisfied: click<7.2.0,>=7.1.1 in c:\\users\\simmu\\anaconda3\\lib\\site-packages (from typer<0.4.0,>=0.3.0->spacy) (7.1.2)\n",
      "Requirement already satisfied: idna<3,>=2.5 in c:\\users\\simmu\\anaconda3\\lib\\site-packages (from requests<3.0.0,>=2.13.0->spacy) (2.10)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\simmu\\anaconda3\\lib\\site-packages (from requests<3.0.0,>=2.13.0->spacy) (2021.5.30)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in c:\\users\\simmu\\anaconda3\\lib\\site-packages (from requests<3.0.0,>=2.13.0->spacy) (1.25.9)\n",
      "Requirement already satisfied: chardet<4,>=3.0.2 in c:\\users\\simmu\\anaconda3\\lib\\site-packages (from requests<3.0.0,>=2.13.0->spacy) (3.0.4)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in c:\\users\\simmu\\anaconda3\\lib\\site-packages (from packaging>=20.0->spacy) (2.4.7)\n",
      "Requirement already satisfied: six in c:\\users\\simmu\\anaconda3\\lib\\site-packages (from packaging>=20.0->spacy) (1.15.0)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in c:\\users\\simmu\\anaconda3\\lib\\site-packages (from pydantic!=1.8,!=1.8.1,<1.9.0,>=1.7.4->spacy) (3.10.0.0)\n",
      "Requirement already satisfied: smart-open<6.0.0,>=5.0.0 in c:\\users\\simmu\\anaconda3\\lib\\site-packages (from pathy>=0.3.5->spacy) (5.1.0)\n",
      "Requirement already satisfied: MarkupSafe>=0.23 in c:\\users\\simmu\\anaconda3\\lib\\site-packages (from jinja2->spacy) (1.1.1)\n",
      "Collecting jovian\n",
      "  Downloading jovian-0.2.41-py2.py3-none-any.whl (68 kB)\n",
      "Collecting uuid\n",
      "  Downloading uuid-1.30.tar.gz (5.8 kB)\n",
      "Requirement already satisfied: requests in c:\\users\\simmu\\anaconda3\\lib\\site-packages (from jovian) (2.24.0)\n",
      "Requirement already satisfied: pyyaml in c:\\users\\simmu\\anaconda3\\lib\\site-packages (from jovian) (5.3.1)\n",
      "Requirement already satisfied: click in c:\\users\\simmu\\anaconda3\\lib\\site-packages (from jovian) (7.1.2)\n",
      "Requirement already satisfied: idna<3,>=2.5 in c:\\users\\simmu\\anaconda3\\lib\\site-packages (from requests->jovian) (2.10)\n",
      "Requirement already satisfied: chardet<4,>=3.0.2 in c:\\users\\simmu\\anaconda3\\lib\\site-packages (from requests->jovian) (3.0.4)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\simmu\\anaconda3\\lib\\site-packages (from requests->jovian) (2021.5.30)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in c:\\users\\simmu\\anaconda3\\lib\\site-packages (from requests->jovian) (1.25.9)\n",
      "Building wheels for collected packages: uuid\n",
      "  Building wheel for uuid (setup.py): started\n",
      "  Building wheel for uuid (setup.py): finished with status 'done'\n",
      "  Created wheel for uuid: filename=uuid-1.30-py3-none-any.whl size=6501 sha256=5dc0e8310cbd5c6c668739a16bb26b790f0ed0f581993e97031ab78755f075e6\n",
      "  Stored in directory: c:\\users\\simmu\\appdata\\local\\pip\\cache\\wheels\\1b\\6c\\cb\\f9aae2bc97333c3d6e060826c1ee9e44e46306a178e5783505\n",
      "Successfully built uuid\n",
      "Installing collected packages: uuid, jovian\n",
      "Successfully installed jovian-0.2.41 uuid-1.30\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "if (window.IPython && IPython.notebook.kernel) IPython.notebook.kernel.execute('jovian.utils.jupyter.get_notebook_name_saved = lambda: \"' + IPython.notebook.notebook_name + '\"')"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#library imports\n",
    "!pip install torch\n",
    "!pip install spacy\n",
    "!pip install jovian\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "import spacy\n",
    "import jovian\n",
    "from collections import Counter\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "import torch.nn.functional as F\n",
    "import string\n",
    "from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence\n",
    "from sklearn.metrics import mean_squared_error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#input (Basic LSTM in PyTorch with Random numbers)\n",
    "x = torch.tensor([[1,2, 12,34, 56,78, 90,80],\n",
    "                 [12,45, 99,67, 6,23, 77,82],\n",
    "                 [3,24, 6,99, 12,56, 21,22]])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "using two different models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "model1 = nn.Embedding(100, 7, padding_idx=0)\n",
    "model2 = nn.LSTM(input_size=7, hidden_size=3, num_layers=1, batch_first=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "out1 = model1(x)\n",
    "out2 = model2(out1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([3, 8, 7])\n",
      "tensor([[[-0.1138, -1.3844, -0.4573, -0.4441, -1.1819,  0.2450,  0.1639],\n",
      "         [ 0.5921, -1.3083,  2.2689, -2.0425,  0.4900, -0.4369, -1.4076],\n",
      "         [-0.7947,  1.3762,  0.9369, -2.0566, -1.0512, -0.3015, -0.2779],\n",
      "         [ 0.1303,  1.7352, -0.3041, -0.1496, -2.0450,  0.9286,  0.1745],\n",
      "         [-0.8471, -0.3860, -1.1694, -0.7194, -0.6510, -0.0369, -1.1985],\n",
      "         [ 1.5536, -0.7190,  0.0508,  1.2089,  0.4229,  0.2796, -0.2056],\n",
      "         [-1.5812,  0.6476, -1.2037,  0.5257,  1.3810, -0.0458,  0.6029],\n",
      "         [-2.6046,  0.1465, -0.4745,  0.8798,  0.1657,  0.5388,  0.1244]],\n",
      "\n",
      "        [[-0.7947,  1.3762,  0.9369, -2.0566, -1.0512, -0.3015, -0.2779],\n",
      "         [ 0.0282,  0.6131,  0.6507, -1.5690, -0.6792, -0.1663,  0.9858],\n",
      "         [ 0.5950, -1.4406, -1.5375, -0.5230,  1.2359,  0.8157, -1.5023],\n",
      "         [-0.4815,  1.3777,  2.3236, -0.2003, -0.1235,  0.7554,  0.9372],\n",
      "         [-0.9159,  0.1368,  0.1068,  0.1013,  0.8645, -0.8717, -0.5511],\n",
      "         [-0.6171,  1.1255, -1.5057,  0.2491,  0.4841,  0.5325,  1.0668],\n",
      "         [ 0.4613, -0.4444,  1.1426, -0.3817,  0.5012,  2.1782, -2.7210],\n",
      "         [ 1.0997, -0.7412,  0.0798,  0.2433, -1.2987, -0.8602,  0.3271]],\n",
      "\n",
      "        [[ 0.2827,  1.1019,  0.6050,  0.9001, -0.3414, -0.9864,  0.5806],\n",
      "         [-0.8800, -1.4849,  0.5403, -0.5742,  0.8243,  0.3181, -1.6267],\n",
      "         [-0.9159,  0.1368,  0.1068,  0.1013,  0.8645, -0.8717, -0.5511],\n",
      "         [ 0.5950, -1.4406, -1.5375, -0.5230,  1.2359,  0.8157, -1.5023],\n",
      "         [-0.7947,  1.3762,  0.9369, -2.0566, -1.0512, -0.3015, -0.2779],\n",
      "         [-0.8471, -0.3860, -1.1694, -0.7194, -0.6510, -0.0369, -1.1985],\n",
      "         [-0.0638, -1.2498,  2.1575,  0.3677,  0.4811,  1.3797,  0.3971],\n",
      "         [ 0.2604, -1.3889,  3.4009, -0.1270,  2.1333,  1.7345, -0.3822]]],\n",
      "       grad_fn=<EmbeddingBackward>)\n"
     ]
    }
   ],
   "source": [
    "print(out1.shape)\n",
    "print(out1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[[-0.1589, -0.0241, -0.0439],\n",
      "         [ 0.1044, -0.1058,  0.0247],\n",
      "         [-0.2264,  0.2256,  0.2958]]], grad_fn=<StackBackward>)\n"
     ]
    }
   ],
   "source": [
    "out, (ht, ct) = model2(out1)\n",
    "print(ht)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "using nn.sequential"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "model3 = nn.Sequential(nn.Embedding(100, 7, padding_idx=0),\n",
    "                        nn.LSTM(input_size=7, hidden_size=3, num_layers=1, batch_first=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[[ 0.0582,  0.0995,  0.0592],\n",
      "         [ 0.2026,  0.3528, -0.3194],\n",
      "         [ 0.2184,  0.2725, -0.5097],\n",
      "         [ 0.3748, -0.1079, -0.3855],\n",
      "         [ 0.3830, -0.0189, -0.3125],\n",
      "         [ 0.2868, -0.1447, -0.2994],\n",
      "         [ 0.1497,  0.0120, -0.3394],\n",
      "         [ 0.2409,  0.0047, -0.0993]],\n",
      "\n",
      "        [[ 0.1437,  0.0394, -0.2014],\n",
      "         [ 0.1259,  0.1149,  0.0271],\n",
      "         [ 0.1465,  0.2317,  0.0763],\n",
      "         [ 0.2318, -0.0348, -0.0095],\n",
      "         [ 0.3190,  0.0747, -0.3237],\n",
      "         [ 0.2444, -0.1477, -0.2047],\n",
      "         [ 0.1595, -0.0876, -0.0358],\n",
      "         [ 0.0333, -0.0167, -0.0888]],\n",
      "\n",
      "        [[ 0.1608,  0.1862, -0.1811],\n",
      "         [ 0.0418, -0.0486,  0.0225],\n",
      "         [ 0.2414,  0.0597, -0.2811],\n",
      "         [ 0.1673,  0.1486,  0.0118],\n",
      "         [ 0.2229,  0.0901, -0.2066],\n",
      "         [ 0.3826,  0.0506, -0.2140],\n",
      "         [ 0.2935, -0.3682, -0.0411],\n",
      "         [ 0.1645, -0.1411, -0.0221]]], grad_fn=<TransposeBackward0>)\n"
     ]
    }
   ],
   "source": [
    "out, (ht, ct) = model3(x)\n",
    "print(out)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Multiclass Text Classification\n",
    "\n",
    "We are going to predict item ratings based on customer reviews based on a e commerce store of clothing dataset from Kaggle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(23486, 11)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>Clothing ID</th>\n",
       "      <th>Age</th>\n",
       "      <th>Title</th>\n",
       "      <th>Review Text</th>\n",
       "      <th>Rating</th>\n",
       "      <th>Recommended IND</th>\n",
       "      <th>Positive Feedback Count</th>\n",
       "      <th>Division Name</th>\n",
       "      <th>Department Name</th>\n",
       "      <th>Class Name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>767</td>\n",
       "      <td>33</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Absolutely wonderful - silky and sexy and comf...</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Initmates</td>\n",
       "      <td>Intimate</td>\n",
       "      <td>Intimates</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1080</td>\n",
       "      <td>34</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Love this dress!  it's sooo pretty.  i happene...</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>General</td>\n",
       "      <td>Dresses</td>\n",
       "      <td>Dresses</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>1077</td>\n",
       "      <td>60</td>\n",
       "      <td>Some major design flaws</td>\n",
       "      <td>I had such high hopes for this dress and reall...</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>General</td>\n",
       "      <td>Dresses</td>\n",
       "      <td>Dresses</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>1049</td>\n",
       "      <td>50</td>\n",
       "      <td>My favorite buy!</td>\n",
       "      <td>I love, love, love this jumpsuit. it's fun, fl...</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>General Petite</td>\n",
       "      <td>Bottoms</td>\n",
       "      <td>Pants</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>847</td>\n",
       "      <td>47</td>\n",
       "      <td>Flattering shirt</td>\n",
       "      <td>This shirt is very flattering to all due to th...</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>General</td>\n",
       "      <td>Tops</td>\n",
       "      <td>Blouses</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  Clothing ID  Age                    Title  \\\n",
       "0           0          767   33                      NaN   \n",
       "1           1         1080   34                      NaN   \n",
       "2           2         1077   60  Some major design flaws   \n",
       "3           3         1049   50         My favorite buy!   \n",
       "4           4          847   47         Flattering shirt   \n",
       "\n",
       "                                         Review Text  Rating  Recommended IND  \\\n",
       "0  Absolutely wonderful - silky and sexy and comf...       4                1   \n",
       "1  Love this dress!  it's sooo pretty.  i happene...       5                1   \n",
       "2  I had such high hopes for this dress and reall...       3                0   \n",
       "3  I love, love, love this jumpsuit. it's fun, fl...       5                1   \n",
       "4  This shirt is very flattering to all due to th...       5                1   \n",
       "\n",
       "   Positive Feedback Count   Division Name Department Name Class Name  \n",
       "0                        0       Initmates        Intimate  Intimates  \n",
       "1                        4         General         Dresses    Dresses  \n",
       "2                        0         General         Dresses    Dresses  \n",
       "3                        0  General Petite         Bottoms      Pants  \n",
       "4                        6         General            Tops    Blouses  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#loading the data\n",
    "reviews = pd.read_csv(\"C:/Users/simmu/Downloads/archive (1)/Womens Clothing E-Commerce Reviews.csv\")\n",
    "print(reviews.shape)\n",
    "reviews.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "reviews['Title'] = reviews['Title'].fillna('')\n",
    "reviews['Review Text'] = reviews['Review Text'].fillna('')\n",
    "reviews['review'] = reviews['Title'] + ' ' + reviews['Review Text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>review</th>\n",
       "      <th>rating</th>\n",
       "      <th>review_length</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Absolutely wonderful - silky and sexy and com...</td>\n",
       "      <td>4</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Love this dress!  it's sooo pretty.  i happen...</td>\n",
       "      <td>5</td>\n",
       "      <td>62</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Some major design flaws I had such high hopes ...</td>\n",
       "      <td>3</td>\n",
       "      <td>102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>My favorite buy! I love, love, love this jumps...</td>\n",
       "      <td>5</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Flattering shirt This shirt is very flattering...</td>\n",
       "      <td>5</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              review  rating  review_length\n",
       "0   Absolutely wonderful - silky and sexy and com...       4              8\n",
       "1   Love this dress!  it's sooo pretty.  i happen...       5             62\n",
       "2  Some major design flaws I had such high hopes ...       3            102\n",
       "3  My favorite buy! I love, love, love this jumps...       5             25\n",
       "4  Flattering shirt This shirt is very flattering...       5             38"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#keeping only relevant columns and calculating sentence lengths\n",
    "reviews = reviews[['review', 'Rating']]\n",
    "reviews.columns = ['review', 'rating']\n",
    "reviews['review_length'] = reviews['review'].apply(lambda x: len(x.split()))\n",
    "reviews.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "#changing ratings to 0-numbering\n",
    "zero_numbering = {1:0, 2:1, 3:2, 4:3, 5:4}\n",
    "reviews['rating'] = reviews['rating'].apply(lambda x: zero_numbering[x])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "60.832921740611425"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#mean sentence length\n",
    "np.mean(reviews['review_length'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "✔ Download and installation successful\n",
      "You can now load the package via spacy.load('en_core_web_sm')\n"
     ]
    }
   ],
   "source": [
    "#tokenization\n",
    "from spacy.cli.download import download\n",
    "download(model=\"en_core_web_sm\")\n",
    "tok = spacy.load('en_core_web_sm')\n",
    "def tokenize (text):\n",
    "    text = re.sub(r\"[^\\x00-\\x7F]+\", \" \", text)\n",
    "    regex = re.compile('[' + re.escape(string.punctuation) + '0-9\\\\r\\\\t\\\\n]') # remove punctuation and numbers\n",
    "    nopunct = regex.sub(\" \", text.lower())\n",
    "    return [token.text for token in tok.tokenizer(nopunct)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "#count number of occurences of each word\n",
    "counts = Counter()\n",
    "for index, row in reviews.iterrows():\n",
    "    counts.update(tokenize(row['review']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num_words before: 14137\n",
      "num_words after: 8262\n"
     ]
    }
   ],
   "source": [
    "#deleting infrequent words\n",
    "print(\"num_words before:\",len(counts.keys()))\n",
    "for word in list(counts):\n",
    "    if counts[word] < 2:\n",
    "        del counts[word]\n",
    "print(\"num_words after:\",len(counts.keys()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "#creating vocabulary\n",
    "vocab2index = {\"\":0, \"UNK\":1}\n",
    "words = [\"\", \"UNK\"]\n",
    "for word in counts:\n",
    "    vocab2index[word] = len(words)\n",
    "    words.append(word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "def encode_sentence(text, vocab2index, N=70):\n",
    "    tokenized = tokenize(text)\n",
    "    encoded = np.zeros(N, dtype=int)\n",
    "    enc1 = np.array([vocab2index.get(word, vocab2index[\"UNK\"]) for word in tokenized])\n",
    "    length = min(N, len(enc1))\n",
    "    encoded[:length] = enc1[:length]\n",
    "    return encoded, length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>review</th>\n",
       "      <th>rating</th>\n",
       "      <th>review_length</th>\n",
       "      <th>encoded</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Absolutely wonderful - silky and sexy and com...</td>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>[[2, 3, 4, 5, 6, 7, 8, 7, 9, 0, 0, 0, 0, 0, 0,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Love this dress!  it's sooo pretty.  i happen...</td>\n",
       "      <td>4</td>\n",
       "      <td>62</td>\n",
       "      <td>[[2, 10, 11, 12, 5, 13, 14, 15, 16, 5, 17, 18,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Some major design flaws I had such high hopes ...</td>\n",
       "      <td>2</td>\n",
       "      <td>102</td>\n",
       "      <td>[[54, 55, 56, 57, 17, 58, 59, 60, 61, 62, 11, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>My favorite buy! I love, love, love this jumps...</td>\n",
       "      <td>4</td>\n",
       "      <td>25</td>\n",
       "      <td>[[68, 109, 110, 2, 17, 10, 2, 10, 2, 10, 11, 1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Flattering shirt This shirt is very flattering...</td>\n",
       "      <td>4</td>\n",
       "      <td>38</td>\n",
       "      <td>[[122, 123, 11, 123, 52, 92, 122, 19, 124, 125...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              review  rating  review_length  \\\n",
       "0   Absolutely wonderful - silky and sexy and com...       3              8   \n",
       "1   Love this dress!  it's sooo pretty.  i happen...       4             62   \n",
       "2  Some major design flaws I had such high hopes ...       2            102   \n",
       "3  My favorite buy! I love, love, love this jumps...       4             25   \n",
       "4  Flattering shirt This shirt is very flattering...       4             38   \n",
       "\n",
       "                                             encoded  \n",
       "0  [[2, 3, 4, 5, 6, 7, 8, 7, 9, 0, 0, 0, 0, 0, 0,...  \n",
       "1  [[2, 10, 11, 12, 5, 13, 14, 15, 16, 5, 17, 18,...  \n",
       "2  [[54, 55, 56, 57, 17, 58, 59, 60, 61, 62, 11, ...  \n",
       "3  [[68, 109, 110, 2, 17, 10, 2, 10, 2, 10, 11, 1...  \n",
       "4  [[122, 123, 11, 123, 52, 92, 122, 19, 124, 125...  "
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reviews['encoded'] = reviews['review'].apply(lambda x: np.array(encode_sentence(x,vocab2index )))\n",
    "reviews.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({3: 5077, 4: 13131, 2: 2871, 1: 1565, 0: 842})"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#check how balanced the dataset is\n",
    "Counter(reviews['rating'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = list(reviews['encoded'])\n",
    "y = list(reviews['rating'])\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ReviewsDataset(Dataset):\n",
    "    def __init__(self, X, Y):\n",
    "        self.X = X\n",
    "        self.y = Y\n",
    "        \n",
    "    def __len__(self):\n",
    "        return len(self.y)\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        return torch.from_numpy(self.X[idx][0].astype(np.int32)), self.y[idx], self.X[idx][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_ds = ReviewsDataset(X_train, y_train)\n",
    "valid_ds = ReviewsDataset(X_valid, y_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model(model, epochs=10, lr=0.001):\n",
    "    parameters = filter(lambda p: p.requires_grad, model.parameters())\n",
    "    optimizer = torch.optim.Adam(parameters, lr=lr)\n",
    "    for i in range(epochs):\n",
    "        model.train()\n",
    "        sum_loss = 0.0\n",
    "        total = 0\n",
    "        for x, y, l in train_dl:\n",
    "            x = x.long()\n",
    "            y = y.long()\n",
    "            y_pred = model(x, l)\n",
    "            optimizer.zero_grad()\n",
    "            loss = F.cross_entropy(y_pred, y)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            sum_loss += loss.item()*y.shape[0]\n",
    "            total += y.shape[0]\n",
    "        val_loss, val_acc, val_rmse = validation_metrics(model, val_dl)\n",
    "        if i % 5 == 1:\n",
    "            print(\"train loss %.3f, val loss %.3f, val accuracy %.3f, and val rmse %.3f\" % (sum_loss/total, val_loss, val_acc, val_rmse))\n",
    "\n",
    "def validation_metrics (model, valid_dl):\n",
    "    model.eval()\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    sum_loss = 0.0\n",
    "    sum_rmse = 0.0\n",
    "    for x, y, l in valid_dl:\n",
    "        x = x.long()\n",
    "        y = y.long()\n",
    "        y_hat = model(x, l)\n",
    "        loss = F.cross_entropy(y_hat, y)\n",
    "        pred = torch.max(y_hat, 1)[1]\n",
    "        correct += (pred == y).float().sum()\n",
    "        total += y.shape[0]\n",
    "        sum_loss += loss.item()*y.shape[0]\n",
    "        sum_rmse += np.sqrt(mean_squared_error(pred, y.unsqueeze(-1)))*y.shape[0]\n",
    "    return sum_loss/total, correct/total, sum_rmse/total"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 5000\n",
    "vocab_size = len(words)\n",
    "train_dl = DataLoader(train_ds, batch_size=batch_size, shuffle=True)\n",
    "val_dl = DataLoader(valid_ds, batch_size=batch_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "LSTM with fixed length input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LSTM_fixed_len(torch.nn.Module) :\n",
    "    def __init__(self, vocab_size, embedding_dim, hidden_dim) :\n",
    "        super().__init__()\n",
    "        self.embeddings = nn.Embedding(vocab_size, embedding_dim, padding_idx=0)\n",
    "        self.lstm = nn.LSTM(embedding_dim, hidden_dim, batch_first=True)\n",
    "        self.linear = nn.Linear(hidden_dim, 5)\n",
    "        self.dropout = nn.Dropout(0.2)\n",
    "        \n",
    "    def forward(self, x, l):\n",
    "        x = self.embeddings(x)\n",
    "        x = self.dropout(x)\n",
    "        lstm_out, (ht, ct) = self.lstm(x)\n",
    "        return self.linear(ht[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_fixed =  LSTM_fixed_len(vocab_size, 50, 50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_model(model_fixed, epochs=30, lr=0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_model(model_fixed, epochs=30, lr=0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_model(model_fixed, epochs=30, lr=0.01)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}