word2vec.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Compare NLP Techniques: Build Model On word2vec Vectors"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Read In Cleaned Text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the cleaned training and test sets\n",
    "import gensim\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "X_train = pd.read_csv('../../../data/X_train.csv')\n",
    "X_test = pd.read_csv('../../../data/X_test.csv')\n",
    "y_train = pd.read_csv('../../../data/y_train.csv')\n",
    "y_test = pd.read_csv('../../../data/y_test.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create word2vec Vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train a basic word2vec model\n",
    "w2v_model = gensim.models.Word2Vec(X_train,\n",
    "                                   size=100,\n",
    "                                   window=5,\n",
    "                                   min_count=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>clean_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>['ne', 'thing', 'interesting', 'good', 'birthd...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>['alrite']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>['one', 'registered', 'subscribers', 'u', 'ent...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>['let', 'know', 'need', 'anything', 'else', 's...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>['thats', 'ok', 'popped', 'ask', 'bout', 'some...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>['oh', 'wow', 'thats', 'gay', 'firmware', 'upd...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>['accidentally', 'deleted', 'message', 'resend...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>['put', 'party', '7', 'days', 'week', 'study',...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>['reverse', 'cheating', 'mathematics']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>['todays', 'voda', 'numbers', 'ending', '5226'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>['turns', 'friends', 'staying', 'whole', 'show...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>['haha', 'figures', 'well', 'found', 'piece', ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>['come', 'alivebetter', 'correct', 'good', 'lo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>['canåõt', 'wait', 'cornwall', 'hope', 'tonigh...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>['cant', 'pick', 'phone', 'right', 'pls', 'sen...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>['private', '2003', 'account', 'statement', 'f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>['hey', 'sorry', 'didntgive', 'ya', 'bellearli...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>['yeshere', 'tv', 'always', 'available', 'work...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>['aight', 'ill', 'ask', 'roommates']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>['latest', 'news', 'police', 'station', 'toile...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>['lol', 'oh', 'babe', 'wont', 'sliding', 'plac...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>['ìï', 'takin', 'linear', 'algebra', 'today']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>['actual', 'exam', 'harder', 'nbme']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>['boo', 'time', 'u', 'get', 'u', 'supposed', '...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>['u', 'want', '2', 'meet', '2morro']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>['ok', 'r', 'meeting', 'later']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>['3', 'free', 'tarot', 'texts', 'find', 'love'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>['sleepingand', 'surfing']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>['asked', 'sen', 'come', 'chennai', 'search', ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>['tomorrow', 'onwards', 'eve', '6', '3', 'work']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1085</th>\n",
       "      <td>['wats', 'da', 'decision']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1086</th>\n",
       "      <td>['aathiwhere', 'dear']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1087</th>\n",
       "      <td>['sorry', 'never', 'hear', 'unless', 'book', '...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1088</th>\n",
       "      <td>['hi', 'hope', 'u', 'get', 'txtjourney', 'hasn...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1089</th>\n",
       "      <td>['hungry', 'gay', 'guys', 'feeling', 'hungry',...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1090</th>\n",
       "      <td>['yup', 'anything', 'lor', 'u', 'dun', 'wan', ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1091</th>\n",
       "      <td>['5', 'nightswe', 'nt', 'staying', 'port', 'st...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1092</th>\n",
       "      <td>['dear', 'voucher', 'holder', '2', 'claim', '1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1093</th>\n",
       "      <td>['sos', 'amount', 'get', 'pls']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1094</th>\n",
       "      <td>['taste', 'fish', 'curry', 'p']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1095</th>\n",
       "      <td>['youre', 'done', 'mean']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1096</th>\n",
       "      <td>['package', 'programs', 'well']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1097</th>\n",
       "      <td>['ìï', 'got', 'wat', 'buy', 'tell', 'us', 'ì',...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1098</th>\n",
       "      <td>['yup', 'ì', 'comin', '']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099</th>\n",
       "      <td>['u', '447801259231', 'secret', 'admirer', 'lo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1100</th>\n",
       "      <td>['thanks', 'vote', 'sing', 'along', 'stars', '...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1101</th>\n",
       "      <td>['gud', 'mrng', 'dear', 'hav', 'nice', 'day']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1102</th>\n",
       "      <td>['finished', 'work', 'yet', 'something']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1103</th>\n",
       "      <td>['im', 'already', 'back', 'home', 'probably']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1104</th>\n",
       "      <td>['plz', 'note', 'anyone', 'calling', 'mobile',...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1105</th>\n",
       "      <td>['congrats', '2', 'mobile', '3g', 'videophones...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1106</th>\n",
       "      <td>['lol', 'gonna', 'last', 'month', 'cashed', 'l...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1107</th>\n",
       "      <td>['dun', 'thk', 'ill', 'quit', 'yet', 'hmmm', '...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1108</th>\n",
       "      <td>['yes', 'pretty', 'lady', 'like', 'single']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1109</th>\n",
       "      <td>['yeah', 'probably', 'sure', 'ilol', 'let', 'u...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1110</th>\n",
       "      <td>['darren', 'saying', 'dat', 'u', 'meeting', 'd...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1111</th>\n",
       "      <td>['hey', 'next', 'sun', '1030', 'theres', 'basi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1112</th>\n",
       "      <td>['nothis', 'kallis', 'home', 'groundamla', 'ho...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1113</th>\n",
       "      <td>['wot', 'u', '2', 'u', 'weirdo']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1114</th>\n",
       "      <td>['okie', 'thanx']</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1115 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             clean_text\n",
       "0     ['ne', 'thing', 'interesting', 'good', 'birthd...\n",
       "1                                            ['alrite']\n",
       "2     ['one', 'registered', 'subscribers', 'u', 'ent...\n",
       "3     ['let', 'know', 'need', 'anything', 'else', 's...\n",
       "4     ['thats', 'ok', 'popped', 'ask', 'bout', 'some...\n",
       "5     ['oh', 'wow', 'thats', 'gay', 'firmware', 'upd...\n",
       "6     ['accidentally', 'deleted', 'message', 'resend...\n",
       "7     ['put', 'party', '7', 'days', 'week', 'study',...\n",
       "8                ['reverse', 'cheating', 'mathematics']\n",
       "9     ['todays', 'voda', 'numbers', 'ending', '5226'...\n",
       "10    ['turns', 'friends', 'staying', 'whole', 'show...\n",
       "11    ['haha', 'figures', 'well', 'found', 'piece', ...\n",
       "12    ['come', 'alivebetter', 'correct', 'good', 'lo...\n",
       "13    ['canåõt', 'wait', 'cornwall', 'hope', 'tonigh...\n",
       "14    ['cant', 'pick', 'phone', 'right', 'pls', 'sen...\n",
       "15    ['private', '2003', 'account', 'statement', 'f...\n",
       "16    ['hey', 'sorry', 'didntgive', 'ya', 'bellearli...\n",
       "17    ['yeshere', 'tv', 'always', 'available', 'work...\n",
       "18                 ['aight', 'ill', 'ask', 'roommates']\n",
       "19    ['latest', 'news', 'police', 'station', 'toile...\n",
       "20    ['lol', 'oh', 'babe', 'wont', 'sliding', 'plac...\n",
       "21        ['ìï', 'takin', 'linear', 'algebra', 'today']\n",
       "22                 ['actual', 'exam', 'harder', 'nbme']\n",
       "23    ['boo', 'time', 'u', 'get', 'u', 'supposed', '...\n",
       "24                 ['u', 'want', '2', 'meet', '2morro']\n",
       "25                      ['ok', 'r', 'meeting', 'later']\n",
       "26    ['3', 'free', 'tarot', 'texts', 'find', 'love'...\n",
       "27                           ['sleepingand', 'surfing']\n",
       "28    ['asked', 'sen', 'come', 'chennai', 'search', ...\n",
       "29     ['tomorrow', 'onwards', 'eve', '6', '3', 'work']\n",
       "...                                                 ...\n",
       "1085                         ['wats', 'da', 'decision']\n",
       "1086                             ['aathiwhere', 'dear']\n",
       "1087  ['sorry', 'never', 'hear', 'unless', 'book', '...\n",
       "1088  ['hi', 'hope', 'u', 'get', 'txtjourney', 'hasn...\n",
       "1089  ['hungry', 'gay', 'guys', 'feeling', 'hungry',...\n",
       "1090  ['yup', 'anything', 'lor', 'u', 'dun', 'wan', ...\n",
       "1091  ['5', 'nightswe', 'nt', 'staying', 'port', 'st...\n",
       "1092  ['dear', 'voucher', 'holder', '2', 'claim', '1...\n",
       "1093                    ['sos', 'amount', 'get', 'pls']\n",
       "1094                    ['taste', 'fish', 'curry', 'p']\n",
       "1095                          ['youre', 'done', 'mean']\n",
       "1096                    ['package', 'programs', 'well']\n",
       "1097  ['ìï', 'got', 'wat', 'buy', 'tell', 'us', 'ì',...\n",
       "1098                          ['yup', 'ì', 'comin', '']\n",
       "1099  ['u', '447801259231', 'secret', 'admirer', 'lo...\n",
       "1100  ['thanks', 'vote', 'sing', 'along', 'stars', '...\n",
       "1101      ['gud', 'mrng', 'dear', 'hav', 'nice', 'day']\n",
       "1102           ['finished', 'work', 'yet', 'something']\n",
       "1103      ['im', 'already', 'back', 'home', 'probably']\n",
       "1104  ['plz', 'note', 'anyone', 'calling', 'mobile',...\n",
       "1105  ['congrats', '2', 'mobile', '3g', 'videophones...\n",
       "1106  ['lol', 'gonna', 'last', 'month', 'cashed', 'l...\n",
       "1107  ['dun', 'thk', 'ill', 'quit', 'yet', 'hmmm', '...\n",
       "1108        ['yes', 'pretty', 'lady', 'like', 'single']\n",
       "1109  ['yeah', 'probably', 'sure', 'ilol', 'let', 'u...\n",
       "1110  ['darren', 'saying', 'dat', 'u', 'meeting', 'd...\n",
       "1111  ['hey', 'next', 'sun', '1030', 'theres', 'basi...\n",
       "1112  ['nothis', 'kallis', 'home', 'groundamla', 'ho...\n",
       "1113                   ['wot', 'u', '2', 'u', 'weirdo']\n",
       "1114                                  ['okie', 'thanx']\n",
       "\n",
       "[1115 rows x 1 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Replace the words in each text message with the learned word vector\n",
    "words = set(w2v_model.wv.index2word)\n",
    "\n",
    "X_train_vect = np.array([np.array([w2v_model.wv[i] for i in ls if i in words])\n",
    "                         for ls in X_train['clean_text']])\n",
    "\n",
    "X_test_vect = np.array([np.array([w2v_model.wv[i] for i in ls if i in words]) \n",
    "                        for ls in X_test['clean_text']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Average the word vectors for each sentence (and assign a vector of zeros if the model\n",
    "# did not learn any of the words in the text message during training\n",
    "X_train_vect_avg = []\n",
    "for v in X_train_vect:\n",
    "    if v.size:\n",
    "        X_train_vect_avg.append(v.mean(axis=0))\n",
    "    else:\n",
    "        X_train_vect_avg.append(np.zeros(100, dtype=float))\n",
    "        \n",
    "X_test_vect_avg = []\n",
    "for v in X_test_vect:\n",
    "    if v.size:\n",
    "        X_test_vect_avg.append(v.mean(axis=0))\n",
    "    else:\n",
    "        X_test_vect_avg.append(np.zeros(100, dtype=float))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(17, 100)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# What does the unaveraged version look like?\n",
    "X_train_vect[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100,)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# What does the averaged version look like?\n",
    "X_train_vect_avg[0].shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Fit RandomForestClassifier On Top Of Word Vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/acer/anaconda3/lib/python3.7/site-packages/sklearn/ensemble/forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "# Instantiate and fit a basic Random Forest model on top of the vectors\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "rf = RandomForestClassifier()\n",
    "rf_model = rf.fit(X_train_vect_avg, y_train.values.ravel())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use the trained model to make predictions on the test data\n",
    "y_pred = rf_model.predict(X_test_vect_avg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision: 0.461 / Recall: 0.243 / Accuracy: 0.865\n"
     ]
    }
   ],
   "source": [
    "# Evaluate the predictions of the model on the holdout test set\n",
    "from sklearn.metrics import precision_score, recall_score\n",
    "\n",
    "precision = precision_score(y_test, y_pred)\n",
    "recall = recall_score(y_test, y_pred)\n",
    "print('Precision: {} / Recall: {} / Accuracy: {}'.format(\n",
    "    round(precision, 3), round(recall, 3), round((y_pred==y_test['label']).sum()/len(y_pred), 3)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}