F2B-08-CNN-for-Text-Classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# FLIP(03):  Natural Language Processing\n",
    "**(Module 02: Natural Language Processing)**\n",
    "\n",
    "---\n",
    "- Materials in this module include resources collected from various open-source online repositories.\n",
    "- You are free to use, but NOT allowed to change or distribute this package.\n",
    "\n",
    "Prepared by and for \n",
    "**Student Members** |\n",
    "2020 [TULIP Lab](http://www.tulip.org.au)\n",
    "\n",
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Session 08-Convolutional Neural Networks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.autograd import Variable\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "import nltk\n",
    "import random\n",
    "import numpy as np\n",
    "from collections import Counter, OrderedDict\n",
    "import nltk\n",
    "import re\n",
    "from copy import deepcopy\n",
    "flatten = lambda l: [item for sublist in l for item in sublist]\n",
    "random.seed(1024)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "USE_CUDA = torch.cuda.is_available()\n",
    "gpus = [0]\n",
    "torch.cuda.set_device(gpus[0])\n",
    "\n",
    "FloatTensor = torch.cuda.FloatTensor if USE_CUDA else torch.FloatTensor\n",
    "LongTensor = torch.cuda.LongTensor if USE_CUDA else torch.LongTensor\n",
    "ByteTensor = torch.cuda.ByteTensor if USE_CUDA else torch.ByteTensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def getBatch(batch_size, train_data):\n",
    "    random.shuffle(train_data)\n",
    "    sindex = 0\n",
    "    eindex = batch_size\n",
    "    while eindex < len(train_data):\n",
    "        batch = train_data[sindex: eindex]\n",
    "        temp = eindex\n",
    "        eindex = eindex + batch_size\n",
    "        sindex = temp\n",
    "        yield batch\n",
    "    \n",
    "    if eindex >= len(train_data):\n",
    "        batch = train_data[sindex:]\n",
    "        yield batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def pad_to_batch(batch):\n",
    "    x,y = zip(*batch)\n",
    "    max_x = max([s.size(1) for s in x])\n",
    "    x_p = []\n",
    "    for i in range(len(batch)):\n",
    "        if x[i].size(1) < max_x:\n",
    "            x_p.append(torch.cat([x[i], Variable(LongTensor([word2index['<PAD>']] * (max_x - x[i].size(1)))).view(1, -1)], 1))\n",
    "        else:\n",
    "            x_p.append(x[i])\n",
    "    return torch.cat(x_p), torch.cat(y).view(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def prepare_sequence(seq, to_index):\n",
    "    idxs = list(map(lambda w: to_index[w] if to_index.get(w) is not None else to_index[\"<UNK>\"], seq))\n",
    "    return Variable(LongTensor(idxs))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data load & Preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TREC question dataset(http://cogcomp.org/Data/QA/QC/)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Task involves\n",
    "classifying a question into 6 question\n",
    "types (whether the question is about person,\n",
    "location, numeric information, etc.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data = open('../dataset/train_5500.label.txt', 'r', encoding='latin-1').readlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data = [[d.split(':')[1][:-1], d.split(':')[0]] for d in data]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X, y = list(zip(*data))\n",
    "X = list(X)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Num masking "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It reduces the search space. ex. my birthday is 12.22 ==> my birthday is ##.##"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for i, x in enumerate(X):\n",
    "    X[i] = re.sub('\\d', '#', x).split()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Build Vocab "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vocab = list(set(flatten(X)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9117"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(set(y)) # num of class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "word2index={'<PAD>': 0, '<UNK>': 1}\n",
    "\n",
    "for vo in vocab:\n",
    "    if word2index.get(vo) is None:\n",
    "        word2index[vo] = len(word2index)\n",
    "        \n",
    "index2word = {v:k for k, v in word2index.items()}\n",
    "\n",
    "target2index = {}\n",
    "\n",
    "for cl in set(y):\n",
    "    if target2index.get(cl) is None:\n",
    "        target2index[cl] = len(target2index)\n",
    "\n",
    "index2target = {v:k for k, v in target2index.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_p, y_p = [], []\n",
    "for pair in zip(X,y):\n",
    "    X_p.append(prepare_sequence(pair[0], word2index).view(1, -1))\n",
    "    y_p.append(Variable(LongTensor([target2index[pair[1]]])).view(1, -1))\n",
    "    \n",
    "data_p = list(zip(X_p, y_p))\n",
    "random.shuffle(data_p)\n",
    "\n",
    "train_data = data_p[: int(len(data_p) * 0.9)]\n",
    "test_data = data_p[int(len(data_p) * 0.9):]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Pretrained word vector"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "you can download pretrained word vector from here https://github.com/mmihaltz/word2vec-GoogleNews-vectors "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import gensim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = gensim.models.KeyedVectors.load_word2vec_format('../dataset/GoogleNews-vectors-negative300.bin', binary=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3000000"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(model.index2word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pretrained = []\n",
    "\n",
    "for key in word2index.keys():\n",
    "    try:\n",
    "        pretrained.append(model[word2index[key]])\n",
    "    except:\n",
    "        pretrained.append(np.random.randn(300))\n",
    "        \n",
    "pretrained_vectors = np.vstack(pretrained)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Modeling "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![cnn-for-text-architecture.png](images/08.cnn-for-text-architecture.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class  CNNClassifier(nn.Module):\n",
    "    \n",
    "    def __init__(self, vocab_size, embedding_dim, output_size, kernel_dim=100, kernel_sizes=(3, 4, 5), dropout=0.5):\n",
    "        super(CNNClassifier,self).__init__()\n",
    "\n",
    "        self.embedding = nn.Embedding(vocab_size, embedding_dim)\n",
    "        self.convs = nn.ModuleList([nn.Conv2d(1, kernel_dim, (K, embedding_dim)) for K in kernel_sizes])\n",
    "\n",
    "        # kernal_size = (K,D) \n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "        self.fc = nn.Linear(len(kernel_sizes) * kernel_dim, output_size)\n",
    "    \n",
    "    \n",
    "    def init_weights(self, pretrained_word_vectors, is_static=False):\n",
    "        self.embedding.weight = nn.Parameter(torch.from_numpy(pretrained_word_vectors).float())\n",
    "        if is_static:\n",
    "            self.embedding.weight.requires_grad = False\n",
    "\n",
    "\n",
    "    def forward(self, inputs, is_training=False):\n",
    "        inputs = self.embedding(inputs).unsqueeze(1) # (B,1,T,D)\n",
    "        inputs = [F.relu(conv(inputs)).squeeze(3) for conv in self.convs] #[(N,Co,W), ...]*len(Ks)\n",
    "        inputs = [F.max_pool1d(i, i.size(2)).squeeze(2) for i in inputs] #[(N,Co), ...]*len(Ks)\n",
    "\n",
    "        concated = torch.cat(inputs, 1)\n",
    "\n",
    "        if is_training:\n",
    "            concated = self.dropout(concated) # (N,len(Ks)*Co)\n",
    "        out = self.fc(concated) \n",
    "        return F.log_softmax(out,1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It takes for a while if you use just cpu."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "EPOCH = 5\n",
    "BATCH_SIZE = 50\n",
    "KERNEL_SIZES = [3,4,5]\n",
    "KERNEL_DIM = 100\n",
    "LR = 0.001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = CNNClassifier(len(word2index), 300, len(target2index), KERNEL_DIM, KERNEL_SIZES)\n",
    "model.init_weights(pretrained_vectors) # initialize embedding matrix using pretrained vectors\n",
    "\n",
    "if USE_CUDA:\n",
    "    model = model.cuda()\n",
    "    \n",
    "loss_function = nn.CrossEntropyLoss()\n",
    "optimizer = optim.Adam(model.parameters(), lr=LR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0/5] mean_loss : 2.13\n",
      "[1/5] mean_loss : 0.12\n",
      "[2/5] mean_loss : 0.08\n",
      "[3/5] mean_loss : 0.02\n",
      "[4/5] mean_loss : 0.05\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(EPOCH):\n",
    "    losses = []\n",
    "    for i,batch in enumerate(getBatch(BATCH_SIZE, train_data)):\n",
    "        inputs,targets = pad_to_batch(batch)\n",
    "        \n",
    "        model.zero_grad()\n",
    "        preds = model(inputs, True)\n",
    "        \n",
    "        loss = loss_function(preds, targets)\n",
    "        losses.append(loss.data.tolist()[0])\n",
    "        loss.backward()\n",
    "        \n",
    "        #for param in model.parameters():\n",
    "        #    param.grad.data.clamp_(-3, 3)\n",
    "        \n",
    "        optimizer.step()\n",
    "        \n",
    "        if i % 100 == 0:\n",
    "            print(\"[%d/%d] mean_loss : %0.2f\" %(epoch, EPOCH, np.mean(losses)))\n",
    "            losses = []"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "accuracy = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "97.61904761904762\n"
     ]
    }
   ],
   "source": [
    "for test in test_data:\n",
    "    pred = model(test[0]).max(1)[1]\n",
    "    pred = pred.data.tolist()[0]\n",
    "    target = test[1].data.tolist()[0][0]\n",
    "    if pred == target:\n",
    "        accuracy += 1\n",
    "\n",
    "print(accuracy/len(test_data) * 100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Further topics "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* <a href=\"https://arxiv.org/pdf/1508.06615.pdf\">Character-Aware Neural Language Models</a>\n",
    "* <a href=\"https://papers.nips.cc/paper/5782-character-level-convolutional-networks-for-text-classification.pdf\">Character level CNN for text classification</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Suggested Reading"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* https://blog.statsbot.co/text-classifier-algorithms-in-machine-learning-acc115293278\n",
    "* <a href=\"https://arxiv.org/pdf/1607.01759\">Bag of Tricks for Efficient Text Classification</a>\n",
    "* <a href=\"https://arxiv.org/pdf/1708.02657\">Which Encoding is the Best for Text Classification in Chinese, English, Japanese and Korean?</a>"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}