CNN_Text_classification.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "CNN Text classification",
      "provenance": [],
      "authorship_tag": "ABX9TyPEvVbblKTRTV5G7YlGCnPX",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/sayarghoshroy/Intro_to_DL_tutorial/blob/master/CNN_Text_classification.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lUgxNO4aFcWj"
      },
      "source": [
        "# Using Convolutional Neural Networks for sentence Classification\n",
        "\n",
        "This code uses a version of the model described in ) [CNN For Sentence Classification (Yoon Kim, 2014)](https://www.aclweb.org/anthology/D14-1181/\n",
        ") in order to perform sentence classification on the IMDB dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NudorOZQ9FTA",
        "outputId": "d897b564-2e79-4f62-858f-a04355c97440"
      },
      "source": [
        "!pip install -q torchtext==0.2.3\n",
        "import torchtext\n",
        "print (torchtext.__version__)\n"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.2.3\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IcReTQg-8Qa1",
        "outputId": "93c65f37-9b47-475b-b564-2a7eb0e0d5d8"
      },
      "source": [
        "!pip install -q torch==0.4.1 \n",
        "import torch\n",
        "print (torch.__version__)"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.4.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PTR2rKr35UXP",
        "outputId": "d15b28af-8d7a-4799-b07f-5190b2890bc3"
      },
      "source": [
        "import os\n",
        "import sys\n",
        "import time\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "from torch.autograd import Variable\n",
        "import torch.optim as optim\n",
        "import numpy as np\n",
        "import torchtext\n",
        "from torchtext import data\n",
        "from torchtext import datasets\n",
        "from torchtext.vocab import Vectors, GloVe\n",
        "print (torch.__version__)\n",
        "print (torchtext.__version__)"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.4.1\n",
            "0.2.3\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9ZTQj_UjGYkU"
      },
      "source": [
        "The function to load the dataset, tokenize the sentences and also find out the GloVe embeddings for the vocabulary"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IDWE5AlY5XQA"
      },
      "source": [
        "def load_dataset(test_sen=None):\n",
        "\n",
        "    \"\"\"\n",
        "    tokenizer : Breaks sentences into a list of words. If sequential=False, no tokenization is applied\n",
        "    Field : A class that stores information about the way of preprocessing\n",
        "    fix_length : An important property of TorchText is that we can let the input to be variable length, and TorchText will\n",
        "                 dynamically pad each sequence to the longest sequence in that \"batch\". But here we are using fi_length which\n",
        "                 will pad each sequence to have a fix length of 200.\n",
        "                 \n",
        "    build_vocab : It will first make a vocabulary or dictionary mapping all the unique words present in the train_data to an\n",
        "                  idx and then after it will use GloVe word embedding to map the index to the corresponding word embedding.\n",
        "                  \n",
        "    vocab.vectors : This returns a torch tensor of shape (vocab_size x embedding_dim) containing the pre-trained word embeddings.\n",
        "    BucketIterator : Defines an iterator that batches examples of similar lengths together to minimize the amount of padding needed.\n",
        "    \n",
        "    \"\"\"\n",
        "    \n",
        "    tokenize = lambda x: x.split()\n",
        "    TEXT = data.Field(sequential=True, tokenize=tokenize, lower=True, include_lengths=True, batch_first=True, fix_length=200)\n",
        "    LABEL = data.LabelField()\n",
        "    train_data, test_data = datasets.IMDB.splits(TEXT, LABEL)\n",
        "    TEXT.build_vocab(train_data, vectors=GloVe(name='6B', dim=300))\n",
        "    LABEL.build_vocab(train_data)\n",
        "\n",
        "    word_embeddings = TEXT.vocab.vectors\n",
        "    print (\"Length of Text Vocabulary: \" + str(len(TEXT.vocab)))\n",
        "    print (\"Vector size of Text Vocabulary: \", TEXT.vocab.vectors.size())\n",
        "    print (\"Label Length: \" + str(len(LABEL.vocab)))\n",
        "\n",
        "    train_data, valid_data = train_data.split() # Further splitting of training_data to create new training_data & validation_data\n",
        "    train_iter, valid_iter, test_iter = data.BucketIterator.splits((train_data, valid_data, test_data), batch_size=32, sort_key=lambda x: len(x.text), repeat=False, shuffle=True)\n",
        "\n",
        "    '''Alternatively we can also use the default configurations'''\n",
        "    # train_iter, test_iter = datasets.IMDB.iters(batch_size=32)\n",
        "\n",
        "    vocab_size = len(TEXT.vocab)\n",
        "\n",
        "    return TEXT, vocab_size, word_embeddings, train_iter, valid_iter, test_iter"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UpUkOqkF5afM",
        "outputId": "4f689868-eff9-4b26-d7b8-db474690a003"
      },
      "source": [
        "TEXT, vocab_size, word_embeddings, train_iter, valid_iter, test_iter = load_dataset()"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "downloading aclImdb_v1.tar.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ".vector_cache/glove.6B.zip: 862MB [02:41, 5.35MB/s]                          \n",
            "100%|██████████| 400000/400000 [00:37<00:00, 10768.76it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Length of Text Vocabulary: 251639\n",
            "Vector size of Text Vocabulary:  torch.Size([251639, 300])\n",
            "Label Length: 2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kPnsfFJtHDnY"
      },
      "source": [
        "Clipping the gradient to a maximum and minimum possible value to prevent the exploding gradient problem"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gsQLjnVd5dBW"
      },
      "source": [
        "def clip_gradient(model, clip_value):\n",
        "    params = list(filter(lambda p: p.grad is not None, model.parameters()))\n",
        "    for p in params:\n",
        "        p.grad.data.clamp_(-clip_value, clip_value)"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "g8hCKhKGHOpa"
      },
      "source": [
        "This function is used to train the given model using an Adam optimiser "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T1Auu6r75e2j"
      },
      "source": [
        "def train_model(model, train_iter, epoch):\n",
        "    total_epoch_loss = 0\n",
        "    total_epoch_acc = 0\n",
        "    model.cuda()\n",
        "    optim = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()))\n",
        "    steps = 0\n",
        "    model.train()\n",
        "    for idx, batch in enumerate(train_iter):\n",
        "        text = batch.text[0]\n",
        "        target = batch.label\n",
        "        target = torch.autograd.Variable(target).long()\n",
        "        if torch.cuda.is_available():\n",
        "            text = text.cuda()\n",
        "            target = target.cuda()\n",
        "        if (text.size()[0] is not 32):# One of the batch returned by BucketIterator has length different than 32.\n",
        "            continue\n",
        "        optim.zero_grad()\n",
        "        prediction = model(text)\n",
        "        loss = loss_fn(prediction, target)\n",
        "        num_corrects = (torch.max(prediction, 1)[1].view(target.size()).data == target.data).float().sum()\n",
        "        acc = 100.0 * num_corrects/len(batch)\n",
        "        loss.backward()\n",
        "        clip_gradient(model, 1e-1)\n",
        "        optim.step()\n",
        "        steps += 1\n",
        "        \n",
        "        if steps % 100 == 0:\n",
        "            print (f'Epoch: {epoch+1}, Idx: {idx+1}, Training Loss: {loss.item():.4f}, Training Accuracy: {acc.item(): .2f}%')\n",
        "        \n",
        "        total_epoch_loss += loss.item()\n",
        "        total_epoch_acc += acc.item()\n",
        "        \n",
        "    return total_epoch_loss/len(train_iter), total_epoch_acc/len(train_iter)"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xtgF3JW45haa"
      },
      "source": [
        "def eval_model(model, val_iter):\n",
        "    total_epoch_loss = 0\n",
        "    total_epoch_acc = 0\n",
        "    model.eval()\n",
        "    with torch.no_grad():\n",
        "        for idx, batch in enumerate(val_iter):\n",
        "            text = batch.text[0]\n",
        "            if (text.size()[0] is not 32):\n",
        "                continue\n",
        "            target = batch.label\n",
        "            target = torch.autograd.Variable(target).long()\n",
        "            if torch.cuda.is_available():\n",
        "                text = text.cuda()\n",
        "                target = target.cuda()\n",
        "            prediction = model(text)\n",
        "            loss = loss_fn(prediction, target)\n",
        "            num_corrects = (torch.max(prediction, 1)[1].view(target.size()).data == target.data).sum()\n",
        "            acc = 100.0 * num_corrects/len(batch)\n",
        "            total_epoch_loss += loss.item()\n",
        "            total_epoch_acc += acc.item()\n",
        "\n",
        "    return total_epoch_loss/len(val_iter), total_epoch_acc/len(val_iter)"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5dPb1zy4H2mz"
      },
      "source": [
        "Defining the model class"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LIQApSTW5klv"
      },
      "source": [
        "class CNN(nn.Module):\n",
        "\tdef __init__(self, batch_size, output_size, in_channels, out_channels, kernel_heights, stride, padding, keep_probab, vocab_size, embedding_length, weights):\n",
        "\t\tsuper(CNN, self).__init__()\n",
        "\t\t\n",
        "\t\t\"\"\"\n",
        "\t\tArguments\n",
        "\t\t---------\n",
        "\t\tbatch_size : Size of each batch which is same as the batch_size of the data returned by the TorchText BucketIterator\n",
        "\t\toutput_size : 2 = (pos, neg)\n",
        "\t\tin_channels : Number of input channels. Here it is 1 as the input data has dimension = (batch_size, num_seq, embedding_length)\n",
        "\t\tout_channels : Number of output channels after convolution operation performed on the input matrix\n",
        "\t\tkernel_heights : A list consisting of 3 different kernel_heights. Convolution will be performed 3 times and finally results from each kernel_height will be concatenated.\n",
        "\t\tkeep_probab : Probability of retaining an activation node during dropout operation\n",
        "\t\tvocab_size : Size of the vocabulary containing unique words\n",
        "\t\tembedding_length : Embedding dimension of GloVe word embeddings\n",
        "\t\tweights : Pre-trained GloVe word_embeddings which we will use to create our word_embedding look-up table\n",
        "\t\t--------\n",
        "\t\t\n",
        "\t\t\"\"\"\n",
        "\t\tself.batch_size = batch_size\n",
        "\t\tself.output_size = output_size\n",
        "\t\tself.in_channels = in_channels\n",
        "\t\tself.out_channels = out_channels\n",
        "\t\tself.kernel_heights = kernel_heights\n",
        "\t\tself.stride = stride\n",
        "\t\tself.padding = padding\n",
        "\t\tself.vocab_size = vocab_size\n",
        "\t\tself.embedding_length = embedding_length\n",
        "\t\t\n",
        "\t\tself.word_embeddings = nn.Embedding(vocab_size, embedding_length)\n",
        "\t\tself.word_embeddings.weight = nn.Parameter(weights, requires_grad=False)\n",
        "\t\tself.conv1 = nn.Conv2d(in_channels, out_channels, (kernel_heights[0], embedding_length), stride, padding)\n",
        "\t\tself.conv2 = nn.Conv2d(in_channels, out_channels, (kernel_heights[1], embedding_length), stride, padding)\n",
        "\t\tself.conv3 = nn.Conv2d(in_channels, out_channels, (kernel_heights[2], embedding_length), stride, padding)\n",
        "\t\tself.dropout = nn.Dropout(keep_probab)\n",
        "\t\tself.label = nn.Linear(len(kernel_heights)*out_channels, output_size)\n",
        "\t\n",
        "\tdef conv_block(self, input, conv_layer):\n",
        "\t\tconv_out = conv_layer(input)# conv_out.size() = (batch_size, out_channels, dim, 1)\n",
        "\t\tactivation = F.relu(conv_out.squeeze(3))# activation.size() = (batch_size, out_channels, dim1)\n",
        "\t\tmax_out = F.max_pool1d(activation, activation.size()[2]).squeeze(2)# maxpool_out.size() = (batch_size, out_channels)\n",
        "\t\t\n",
        "\t\treturn max_out\n",
        "\t\n",
        "\tdef forward(self, input_sentences, batch_size=None):\n",
        "\t\t\n",
        "\t\t\"\"\"\n",
        "\t\tThe idea of the Convolutional Neural Netwok for Text Classification is very simple. We perform convolution operation on the embedding matrix \n",
        "\t\twhose shape for each batch is (num_seq, embedding_length) with kernel of varying height but constant width which is same as the embedding_length.\n",
        "\t\tWe will be using ReLU activation after the convolution operation and then for each kernel height, we will use max_pool operation on each tensor \n",
        "\t\tand will filter all the maximum activation for every channel and then we will concatenate the resulting tensors. This output is then fully connected\n",
        "\t\tto the output layers consisting two units which basically gives us the logits for both positive and negative classes.\n",
        "\t\t\n",
        "\t\tParameters\n",
        "\t\t----------\n",
        "\t\tinput_sentences: input_sentences of shape = (batch_size, num_sequences)\n",
        "\t\tbatch_size : default = None. Used only for prediction on a single sentence after training (batch_size = 1)\n",
        "\t\t\n",
        "\t\tReturns\n",
        "\t\t-------\n",
        "\t\tOutput of the linear layer containing logits for pos & neg class.\n",
        "\t\tlogits.size() = (batch_size, output_size)\n",
        "\t\t\n",
        "\t\t\"\"\"\n",
        "\t\t\n",
        "\t\tinput = self.word_embeddings(input_sentences)\n",
        "\t\t# input.size() = (batch_size, num_seq, embedding_length)\n",
        "\t\tinput = input.unsqueeze(1)\n",
        "\t\t# input.size() = (batch_size, 1, num_seq, embedding_length)\n",
        "\t\tmax_out1 = self.conv_block(input, self.conv1)\n",
        "\t\tmax_out2 = self.conv_block(input, self.conv2)\n",
        "\t\tmax_out3 = self.conv_block(input, self.conv3)\n",
        "\t\t\n",
        "\t\tall_out = torch.cat((max_out1, max_out2, max_out3), 1)\n",
        "\t\t# all_out.size() = (batch_size, num_kernels*out_channels)\n",
        "\t\tfc_in = self.dropout(all_out)\n",
        "\t\t# fc_in.size()) = (batch_size, num_kernels*out_channels)\n",
        "\t\tlogits = self.label(fc_in)\n",
        "\t\t\n",
        "\t\treturn logits"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rHvNfrkqH6R5"
      },
      "source": [
        "Listing all Hyper-parameters"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k7Bfyz1l5m8y"
      },
      "source": [
        "\n",
        "learning_rate = 2e-5\n",
        "batch_size = 32\n",
        "output_size = 2\n",
        "in_channels = 1\n",
        "out_channels = 100\n",
        "kernel_heights = [2,3,4]\n",
        "keep_probab =0.5\n",
        "stride =1\n",
        "padding =0\n",
        "vocab_size = len(TEXT.vocab)\n",
        "embedding_length = 300\n",
        "word_embeddings =  TEXT.vocab.vectors\n",
        "\n",
        "loss_fn = F.cross_entropy\n",
        "\n",
        "\n",
        "model = CNN(batch_size, output_size, in_channels, out_channels, kernel_heights, stride, padding, keep_probab, vocab_size, embedding_length, word_embeddings)"
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YyhIsWJRIBkL"
      },
      "source": [
        "Training the model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uu-DpGZ15rIs",
        "outputId": "2319db60-152f-44fc-c727-05466817eabf"
      },
      "source": [
        "\n",
        "for epoch in range(10):\n",
        "    train_loss, train_acc = train_model(model, train_iter, epoch)\n",
        "    val_loss, val_acc = eval_model(model, valid_iter)\n",
        "    \n",
        "    print(f'Epoch: {epoch+1:02}, Train Loss: {train_loss:.3f}, Train Acc: {train_acc:.2f}%, Val. Loss: {val_loss:3f}, Val. Acc: {val_acc:.2f}%')\n",
        "    \n",
        "test_loss, test_acc = eval_model(model, test_iter)\n",
        "print(f'Test Loss: {test_loss:.3f}, Test Acc: {test_acc:.2f}%')\n"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch: 1, Idx: 100, Training Loss: 0.5618, Training Accuracy:  78.12%\n",
            "Epoch: 1, Idx: 200, Training Loss: 0.5752, Training Accuracy:  68.75%\n",
            "Epoch: 1, Idx: 300, Training Loss: 0.3646, Training Accuracy:  84.38%\n",
            "Epoch: 1, Idx: 400, Training Loss: 0.4188, Training Accuracy:  81.25%\n",
            "Epoch: 1, Idx: 500, Training Loss: 0.8118, Training Accuracy:  68.75%\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torchtext/data/field.py:321: UserWarning: volatile was removed and now has no effect. Use `with torch.no_grad():` instead.\n",
            "  return Variable(arr, volatile=not train), lengths\n",
            "/usr/local/lib/python3.7/dist-packages/torchtext/data/field.py:322: UserWarning: volatile was removed and now has no effect. Use `with torch.no_grad():` instead.\n",
            "  return Variable(arr, volatile=not train)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 01, Train Loss: 0.500, Train Acc: 74.68%, Val. Loss: 0.409318, Val. Acc: 79.84%\n",
            "Epoch: 2, Idx: 100, Training Loss: 0.3630, Training Accuracy:  84.38%\n",
            "Epoch: 2, Idx: 200, Training Loss: 0.2427, Training Accuracy:  84.38%\n",
            "Epoch: 2, Idx: 300, Training Loss: 0.6100, Training Accuracy:  75.00%\n",
            "Epoch: 2, Idx: 400, Training Loss: 0.5128, Training Accuracy:  68.75%\n",
            "Epoch: 2, Idx: 500, Training Loss: 0.3353, Training Accuracy:  84.38%\n",
            "Epoch: 02, Train Loss: 0.404, Train Acc: 81.58%, Val. Loss: 0.363430, Val. Acc: 83.05%\n",
            "Epoch: 3, Idx: 100, Training Loss: 0.4209, Training Accuracy:  81.25%\n",
            "Epoch: 3, Idx: 200, Training Loss: 0.3722, Training Accuracy:  81.25%\n",
            "Epoch: 3, Idx: 300, Training Loss: 0.4726, Training Accuracy:  81.25%\n",
            "Epoch: 3, Idx: 400, Training Loss: 0.4197, Training Accuracy:  75.00%\n",
            "Epoch: 3, Idx: 500, Training Loss: 0.5725, Training Accuracy:  75.00%\n",
            "Epoch: 03, Train Loss: 0.352, Train Acc: 84.21%, Val. Loss: 0.364105, Val. Acc: 83.29%\n",
            "Epoch: 4, Idx: 100, Training Loss: 0.3276, Training Accuracy:  81.25%\n",
            "Epoch: 4, Idx: 200, Training Loss: 0.3190, Training Accuracy:  84.38%\n",
            "Epoch: 4, Idx: 300, Training Loss: 0.3192, Training Accuracy:  81.25%\n",
            "Epoch: 4, Idx: 400, Training Loss: 0.3468, Training Accuracy:  84.38%\n",
            "Epoch: 4, Idx: 500, Training Loss: 0.3960, Training Accuracy:  84.38%\n",
            "Epoch: 04, Train Loss: 0.312, Train Acc: 86.39%, Val. Loss: 0.371037, Val. Acc: 82.59%\n",
            "Epoch: 5, Idx: 100, Training Loss: 0.2751, Training Accuracy:  93.75%\n",
            "Epoch: 5, Idx: 200, Training Loss: 0.2091, Training Accuracy:  93.75%\n",
            "Epoch: 5, Idx: 300, Training Loss: 0.5730, Training Accuracy:  81.25%\n",
            "Epoch: 5, Idx: 400, Training Loss: 0.2439, Training Accuracy:  84.38%\n",
            "Epoch: 5, Idx: 500, Training Loss: 0.2319, Training Accuracy:  93.75%\n",
            "Epoch: 05, Train Loss: 0.269, Train Acc: 88.48%, Val. Loss: 0.369516, Val. Acc: 83.85%\n",
            "Epoch: 6, Idx: 100, Training Loss: 0.3177, Training Accuracy:  81.25%\n",
            "Epoch: 6, Idx: 200, Training Loss: 0.4831, Training Accuracy:  78.12%\n",
            "Epoch: 6, Idx: 300, Training Loss: 0.0979, Training Accuracy:  100.00%\n",
            "Epoch: 6, Idx: 400, Training Loss: 0.1686, Training Accuracy:  93.75%\n",
            "Epoch: 6, Idx: 500, Training Loss: 0.1559, Training Accuracy:  96.88%\n",
            "Epoch: 06, Train Loss: 0.239, Train Acc: 90.35%, Val. Loss: 0.379501, Val. Acc: 83.43%\n",
            "Epoch: 7, Idx: 100, Training Loss: 0.2812, Training Accuracy:  87.50%\n",
            "Epoch: 7, Idx: 200, Training Loss: 0.1574, Training Accuracy:  96.88%\n",
            "Epoch: 7, Idx: 300, Training Loss: 0.0601, Training Accuracy:  100.00%\n",
            "Epoch: 7, Idx: 400, Training Loss: 0.2383, Training Accuracy:  90.62%\n",
            "Epoch: 7, Idx: 500, Training Loss: 0.3109, Training Accuracy:  87.50%\n",
            "Epoch: 07, Train Loss: 0.211, Train Acc: 91.16%, Val. Loss: 0.398822, Val. Acc: 82.96%\n",
            "Epoch: 8, Idx: 100, Training Loss: 0.1824, Training Accuracy:  93.75%\n",
            "Epoch: 8, Idx: 200, Training Loss: 0.1994, Training Accuracy:  90.62%\n",
            "Epoch: 8, Idx: 300, Training Loss: 0.2334, Training Accuracy:  90.62%\n",
            "Epoch: 8, Idx: 400, Training Loss: 0.2106, Training Accuracy:  87.50%\n",
            "Epoch: 8, Idx: 500, Training Loss: 0.4483, Training Accuracy:  78.12%\n",
            "Epoch: 08, Train Loss: 0.184, Train Acc: 92.53%, Val. Loss: 0.407115, Val. Acc: 83.46%\n",
            "Epoch: 9, Idx: 100, Training Loss: 0.0677, Training Accuracy:  96.88%\n",
            "Epoch: 9, Idx: 200, Training Loss: 0.1273, Training Accuracy:  93.75%\n",
            "Epoch: 9, Idx: 300, Training Loss: 0.1377, Training Accuracy:  93.75%\n",
            "Epoch: 9, Idx: 400, Training Loss: 0.1893, Training Accuracy:  87.50%\n",
            "Epoch: 9, Idx: 500, Training Loss: 0.2026, Training Accuracy:  93.75%\n",
            "Epoch: 09, Train Loss: 0.173, Train Acc: 93.00%, Val. Loss: 0.410676, Val. Acc: 83.50%\n",
            "Epoch: 10, Idx: 100, Training Loss: 0.1268, Training Accuracy:  96.88%\n",
            "Epoch: 10, Idx: 200, Training Loss: 0.3121, Training Accuracy:  81.25%\n",
            "Epoch: 10, Idx: 300, Training Loss: 0.0726, Training Accuracy:  96.88%\n",
            "Epoch: 10, Idx: 400, Training Loss: 0.1684, Training Accuracy:  90.62%\n",
            "Epoch: 10, Idx: 500, Training Loss: 0.1953, Training Accuracy:  87.50%\n",
            "Epoch: 10, Train Loss: 0.160, Train Acc: 93.57%, Val. Loss: 0.436646, Val. Acc: 83.19%\n",
            "Test Loss: 0.445, Test Acc: 82.39%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wE1afAC4IE4H"
      },
      "source": [
        "Testing the model on custom input"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "N1kodbbG5tvz",
        "outputId": "695582fd-bd4d-4396-fdea-46c33d0a926e"
      },
      "source": [
        "test_sen1 = \"This is one of the best creation of Nolan. I can say, it's his magnum opus. Loved the soundtrack and especially those creative dialogues.\"\n",
        "test_sen2 = \"Ohh, such a ridiculous movie. Not gonna recommend it to anyone. Complete waste of time and money.\"\n",
        "\n",
        "test_sen1 = TEXT.preprocess(test_sen1)\n",
        "test_sen1 = [[TEXT.vocab.stoi[x] for x in test_sen1]]\n",
        "\n",
        "test_sen2 = TEXT.preprocess(test_sen2)\n",
        "test_sen2 = [[TEXT.vocab.stoi[x] for x in test_sen2]]\n",
        "\n",
        "with torch.no_grad():\n",
        "  test_sen = np.asarray(test_sen1)\n",
        "  test_sen = torch.LongTensor(test_sen)\n",
        "  test_tensor = Variable(test_sen)\n",
        "  test_tensor = test_tensor.cuda()\n",
        "  model.eval()\n",
        "  output = model(test_tensor, 1)\n",
        "  out = F.softmax(output, 1)\n",
        "  if (torch.argmax(out[0]) == 1):\n",
        "      print (\"Sentiment: Positive\")\n",
        "  else:\n",
        "      print (\"Sentiment: Negative\")\n",
        "\n",
        "with torch.no_grad():\n",
        "  test_sen = np.asarray(test_sen2)\n",
        "  test_sen = torch.LongTensor(test_sen)\n",
        "  test_tensor = Variable(test_sen)\n",
        "  test_tensor = test_tensor.cuda()\n",
        "  model.eval()\n",
        "  output = model(test_tensor, 1)\n",
        "  out = F.softmax(output, 1)\n",
        "  if (torch.argmax(out[0]) == 1):\n",
        "      print (\"Sentiment: Positive\")\n",
        "  else:\n",
        "      print (\"Sentiment: Negative\")\n",
        "\n"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Sentiment: Positive\n",
            "Sentiment: Negative\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}