cnn_glove_66_score.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchtext import data\n",
    "import pandas as pd\n",
    "import string\n",
    "from sklearn.model_selection import train_test_split\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "import spacy\n",
    "nlp = spacy.load('en')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "TEXT = data.Field(tokenize='spacy')\n",
    "LABEL = data.LabelField()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "fields = [('text', TEXT), ('topic', LABEL)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data, valid_data = data.TabularDataset.splits(\n",
    "                                        path = 'data',\n",
    "                                        train = 'train.csv',\n",
    "                                        validation = 'val.csv',\n",
    "                                        format = 'csv',\n",
    "                                        fields = fields,\n",
    "                                        skip_header = True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_VOCAB_SIZE = 25_000\n",
    "\n",
    "TEXT.build_vocab(train_data, \n",
    "                 max_size = MAX_VOCAB_SIZE, \n",
    "                 vectors = \"glove.6B.300d\", \n",
    "                 unk_init = torch.Tensor.normal_)\n",
    "\n",
    "LABEL.build_vocab(train_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "defaultdict(None, {'Bad Taste/Flavor': 0, 'Quality/Contaminated': 1, 'Not Effective': 2, 'Allergic': 3, 'Packaging': 4, 'Texture': 5, 'Shipment and delivery': 6, 'Customer Service': 7, 'Color and texture': 8, 'Too big to swallow': 9, 'Ingredients': 10, 'Expiry': 11, 'Smells Bad': 12, 'Pricing': 13, 'Wrong Product received': 14, 'Too Sweet': 15, 'Inferior to competitors': 16, 'False Advertisement': 17, \"Didn't Like\": 18, 'Customer Issues': 19, 'Hard to Chew': 20})\n"
     ]
    }
   ],
   "source": [
    "print(LABEL.vocab.stoi)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE = 64\n",
    "\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "\n",
    "train_iterator, valid_iterator = data.BucketIterator.splits(\n",
    "    (train_data, valid_data),\n",
    "    sort_key=lambda x: len(x.text), #sort by s attribute (quote)\n",
    "    sort_within_batch = True,\n",
    "    batch_size=BATCH_SIZE,\n",
    "    device=device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CNN(nn.Module):\n",
    "    def __init__(self, vocab_size, embedding_dim, n_filters, filter_sizes, output_dim, \n",
    "                 dropout, pad_idx):\n",
    "        \n",
    "        super().__init__()\n",
    "        \n",
    "        self.embedding = nn.Embedding(vocab_size, embedding_dim)\n",
    "        \n",
    "        self.convs = nn.ModuleList([\n",
    "                                    nn.Conv2d(in_channels = 1, \n",
    "                                              out_channels = n_filters, \n",
    "                                              kernel_size = (fs, embedding_dim)) \n",
    "                                    for fs in filter_sizes\n",
    "                                    ])\n",
    "        \n",
    "        self.fc = nn.Linear(len(filter_sizes) * n_filters, output_dim)\n",
    "        \n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "        \n",
    "    def forward(self, text):\n",
    "        \n",
    "        #text = [sent len, batch size]\n",
    "        \n",
    "        text = text.permute(1, 0)\n",
    "                \n",
    "        #text = [batch size, sent len]\n",
    "        \n",
    "        embedded = self.embedding(text)\n",
    "                \n",
    "        #embedded = [batch size, sent len, emb dim]\n",
    "        \n",
    "        embedded = embedded.unsqueeze(1)\n",
    "        \n",
    "        #embedded = [batch size, 1, sent len, emb dim]\n",
    "        \n",
    "        conved = [F.relu(conv(embedded)).squeeze(3) for conv in self.convs]\n",
    "            \n",
    "        #conv_n = [batch size, n_filters, sent len - filter_sizes[n]]\n",
    "        \n",
    "        pooled = [F.max_pool1d(conv, conv.shape[2]).squeeze(2) for conv in conved]\n",
    "        \n",
    "        #pooled_n = [batch size, n_filters]\n",
    "        \n",
    "        cat = self.dropout(torch.cat(pooled, dim = 1))\n",
    "\n",
    "        #cat = [batch size, n_filters * len(filter_sizes)]\n",
    "            \n",
    "        return self.fc(cat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "INPUT_DIM = len(TEXT.vocab)\n",
    "EMBEDDING_DIM = 300\n",
    "N_FILTERS = 170\n",
    "FILTER_SIZES = [2,3,4]\n",
    "OUTPUT_DIM = len(LABEL.vocab)\n",
    "DROPOUT = 0.5\n",
    "WEIGHT_DECAY = 0.0001\n",
    "PAD_IDX = TEXT.vocab.stoi[TEXT.pad_token]\n",
    "UNK_IDX = TEXT.vocab.stoi[TEXT.unk_token]\n",
    "\n",
    "model = CNN(INPUT_DIM, EMBEDDING_DIM, N_FILTERS, FILTER_SIZES, OUTPUT_DIM, DROPOUT, PAD_IDX)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 2.1185, -0.4615,  0.6739,  ..., -0.6159,  0.0302, -0.0682],\n",
       "        [-0.4318, -1.0575, -0.2638,  ...,  0.3831, -1.2745,  0.1936],\n",
       "        [-0.1256,  0.0136,  0.1031,  ..., -0.3422, -0.0224,  0.1368],\n",
       "        ...,\n",
       "        [-1.2150, -1.0141, -0.5998,  ...,  0.3420, -0.0995,  0.3069],\n",
       "        [-0.5372, -0.9808, -0.6435,  ...,  0.3492,  0.3977,  0.0348],\n",
       "        [-0.3239,  1.6213, -0.7122,  ...,  0.1654, -0.4450, -0.0763]])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pretrained_embeddings = TEXT.vocab.vectors\n",
    "model.embedding.weight.data.copy_(pretrained_embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.embedding.weight.data[UNK_IDX] = torch.zeros(EMBEDDING_DIM)\n",
    "model.embedding.weight.data[PAD_IDX] = torch.zeros(EMBEDDING_DIM)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = optim.Adam(model.parameters(), weight_decay=WEIGHT_DECAY)\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "model = model.to(device)\n",
    "criterion = criterion.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def categorical_accuracy(preds, y):\n",
    "    \"\"\"\n",
    "    Returns accuracy per batch, i.e. if you get 8/10 right, this returns 0.8, NOT 8\n",
    "    \"\"\"\n",
    "    max_preds = preds.argmax(dim = 1, keepdim = True) # get the index of the max probability\n",
    "    correct = max_preds.squeeze(1).eq(y)\n",
    "    return correct.sum() / torch.FloatTensor([y.shape[0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model, iterator, optimizer, criterion):\n",
    "    \n",
    "    epoch_loss = 0\n",
    "    epoch_acc = 0\n",
    "    \n",
    "    model.train()\n",
    "    \n",
    "    for batch in iterator:\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        predictions = model(batch.text)\n",
    "        \n",
    "        loss = criterion(predictions, batch.topic)\n",
    "        \n",
    "        acc = categorical_accuracy(predictions, batch.topic)\n",
    "        \n",
    "        loss.backward()\n",
    "        \n",
    "        optimizer.step()\n",
    "        \n",
    "        epoch_loss += loss.item()\n",
    "        epoch_acc += acc.item()\n",
    "        \n",
    "    return epoch_loss / len(iterator), epoch_acc / len(iterator)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(model, iterator, criterion):\n",
    "    \n",
    "    epoch_loss = 0\n",
    "    epoch_acc = 0\n",
    "    \n",
    "    model.eval()\n",
    "    \n",
    "    with torch.no_grad():\n",
    "    \n",
    "        for batch in iterator:\n",
    "\n",
    "            predictions = model(batch.text)\n",
    "            \n",
    "            loss = criterion(predictions, batch.topic)\n",
    "            \n",
    "            acc = categorical_accuracy(predictions, batch.topic)\n",
    "\n",
    "            epoch_loss += loss.item()\n",
    "            epoch_acc += acc.item()\n",
    "        \n",
    "    return epoch_loss / len(iterator), epoch_acc / len(iterator)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "\n",
    "def epoch_time(start_time, end_time):\n",
    "    elapsed_time = end_time - start_time\n",
    "    elapsed_mins = int(elapsed_time / 60)\n",
    "    elapsed_secs = int(elapsed_time - (elapsed_mins * 60))\n",
    "    return elapsed_mins, elapsed_secs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 01 | Epoch Time: 0m 1s\n",
      "\tTrain Loss: 2.383 | Train Acc: 29.86%\n",
      "\t Val. Loss: 1.930 |  Val. Acc: 44.26%\n",
      "Epoch: 02 | Epoch Time: 0m 1s\n",
      "\tTrain Loss: 1.652 | Train Acc: 51.44%\n",
      "\t Val. Loss: 1.475 |  Val. Acc: 54.31%\n",
      "Epoch: 03 | Epoch Time: 0m 1s\n",
      "\tTrain Loss: 1.288 | Train Acc: 60.93%\n",
      "\t Val. Loss: 1.284 |  Val. Acc: 57.94%\n",
      "Epoch: 04 | Epoch Time: 0m 1s\n",
      "\tTrain Loss: 1.077 | Train Acc: 66.21%\n",
      "\t Val. Loss: 1.255 |  Val. Acc: 55.87%\n",
      "Epoch: 05 | Epoch Time: 0m 1s\n",
      "\tTrain Loss: 0.965 | Train Acc: 68.11%\n",
      "\t Val. Loss: 1.229 |  Val. Acc: 56.83%\n",
      "Epoch: 06 | Epoch Time: 0m 1s\n",
      "\tTrain Loss: 0.906 | Train Acc: 68.51%\n",
      "\t Val. Loss: 1.205 |  Val. Acc: 56.45%\n",
      "Epoch: 07 | Epoch Time: 0m 1s\n",
      "\tTrain Loss: 0.845 | Train Acc: 69.06%\n",
      "\t Val. Loss: 1.216 |  Val. Acc: 55.49%\n",
      "Epoch: 08 | Epoch Time: 0m 1s\n",
      "\tTrain Loss: 0.820 | Train Acc: 69.69%\n",
      "\t Val. Loss: 1.227 |  Val. Acc: 54.64%\n",
      "Epoch: 09 | Epoch Time: 0m 1s\n",
      "\tTrain Loss: 0.792 | Train Acc: 70.27%\n",
      "\t Val. Loss: 1.258 |  Val. Acc: 54.72%\n",
      "Epoch: 10 | Epoch Time: 0m 1s\n",
      "\tTrain Loss: 0.751 | Train Acc: 71.69%\n",
      "\t Val. Loss: 1.271 |  Val. Acc: 54.92%\n"
     ]
    }
   ],
   "source": [
    "N_EPOCHS = 10\n",
    "\n",
    "best_valid_loss = float('inf')\n",
    "\n",
    "for epoch in range(N_EPOCHS):\n",
    "\n",
    "    start_time = time.time()\n",
    "    \n",
    "    train_loss, train_acc = train(model, train_iterator, optimizer, criterion)\n",
    "    valid_loss, valid_acc = evaluate(model, valid_iterator, criterion)\n",
    "    \n",
    "    end_time = time.time()\n",
    "\n",
    "    epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n",
    "    \n",
    "    if valid_loss < best_valid_loss:\n",
    "        best_valid_loss = train_loss\n",
    "        torch.save(model.state_dict(), 'model.pt')\n",
    "    \n",
    "    print(f'Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s')\n",
    "    print(f'\\tTrain Loss: {train_loss:.3f} | Train Acc: {train_acc*100:.2f}%')\n",
    "    print(f'\\t Val. Loss: {valid_loss:.3f} |  Val. Acc: {valid_acc*100:.2f}%')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load_state_dict(torch.load('model.pt'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_class(model, sentence, min_len = 4):\n",
    "    model.eval()\n",
    "    tokenized = [tok.text for tok in nlp.tokenizer(sentence)]\n",
    "    if len(tokenized) < min_len:\n",
    "        tokenized += ['<pad>'] * (min_len - len(tokenized))\n",
    "    indexed = [TEXT.vocab.stoi[t] for t in tokenized]\n",
    "    tensor = torch.LongTensor(indexed).to(device)\n",
    "    tensor = tensor.unsqueeze(1)\n",
    "    preds = model(tensor)\n",
    "    max_preds = preds.argmax()\n",
    "    return max_preds.item()\n",
    "    return tensor, length_tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = []\n",
    "df = pd.read_csv('./data/test.csv')\n",
    "for i, x in enumerate(df.text_data.values):\n",
    "    predictions.append(LABEL.vocab.itos[predict_class(model, x)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1037,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data_df = pd.read_csv('test.csv')\n",
    "test_data_df['topic'] = predictions\n",
    "test_data_df.to_csv('submission.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}