preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Compare NLP Techniques: Prep The Data For Modeling"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Read In & Clean Text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "      <th>clean_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Go until jurong point, crazy.. Available only ...</td>\n",
       "      <td>[go, jurong, point, crazy, available, bugis, n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>Ok lar... Joking wif u oni...</td>\n",
       "      <td>[ok, lar, joking, wif, u, oni]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>Free entry in 2 a wkly comp to win FA Cup fina...</td>\n",
       "      <td>[free, entry, 2, wkly, comp, win, fa, cup, fin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>U dun say so early hor... U c already then say...</td>\n",
       "      <td>[u, dun, say, early, hor, u, c, already, say]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>Nah I don't think he goes to usf, he lives aro...</td>\n",
       "      <td>[nah, dont, think, goes, usf, lives, around, t...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   label                                               text  \\\n",
       "0      0  Go until jurong point, crazy.. Available only ...   \n",
       "1      0                      Ok lar... Joking wif u oni...   \n",
       "2      1  Free entry in 2 a wkly comp to win FA Cup fina...   \n",
       "3      0  U dun say so early hor... U c already then say...   \n",
       "4      0  Nah I don't think he goes to usf, he lives aro...   \n",
       "\n",
       "                                          clean_text  \n",
       "0  [go, jurong, point, crazy, available, bugis, n...  \n",
       "1                     [ok, lar, joking, wif, u, oni]  \n",
       "2  [free, entry, 2, wkly, comp, win, fa, cup, fin...  \n",
       "3      [u, dun, say, early, hor, u, c, already, say]  \n",
       "4  [nah, dont, think, goes, usf, lives, around, t...  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Read in and clean data\n",
    "import nltk\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import re\n",
    "from sklearn.model_selection import train_test_split\n",
    "import string\n",
    "\n",
    "stopwords = nltk.corpus.stopwords.words('english')\n",
    "\n",
    "messages = pd.read_csv('../../../data/spam.csv', encoding='latin-1')\n",
    "messages = messages.drop(labels = [\"Unnamed: 2\", \"Unnamed: 3\", \"Unnamed: 4\"], axis = 1)\n",
    "messages.columns = [\"label\", \"text\"]\n",
    "messages['label'] = np.where(messages['label']=='spam', 1, 0)\n",
    "\n",
    "def clean_text(text):\n",
    "    text = \"\".join([word.lower() for word in text if word not in string.punctuation])\n",
    "    tokens = re.split('\\W+', text)\n",
    "    text = [word for word in tokens if word not in stopwords]\n",
    "    return text\n",
    "\n",
    "messages['clean_text'] = messages['text'].apply(lambda x: clean_text(x))\n",
    "messages.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split data into train and test set\n",
    "X_train, X_test, y_train, y_test = train_test_split(messages['clean_text'],\n",
    "                                                    messages['label'], test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "459     [going, thru, different, feelingwavering, deci...\n",
       "1023                              [may, call, later, pls]\n",
       "3471     [think, û, waiting, bus, inform, get, ever, get]\n",
       "4322                        [aight, well, keep, informed]\n",
       "74                                              [u, call]\n",
       "5487    [2p, per, min, call, germany, 08448350055, bt,...\n",
       "5067                                         [talk, g, x]\n",
       "4025                            [oh, ok, wats, ur, email]\n",
       "453     [ok, tell, stay, yeah, tough, optimistic, thin...\n",
       "5075    [well, im, glad, didnt, find, totally, disagre...\n",
       "774                                [sleepingand, surfing]\n",
       "1866                    [ì, go, buy, wif, meet, ì, later]\n",
       "1836                                         [wet, right]\n",
       "3374                                                 [, ]\n",
       "1627    [selected, stay, 1, 250, top, british, hotels,...\n",
       "773                      [wil, ltgt, minutes, got, space]\n",
       "5426                       [oh, yeah, diet, flew, window]\n",
       "4173             [pls, pls, drink, plenty, plenty, water]\n",
       "4534                        [im, late, tellmiss, im, way]\n",
       "1773                         [im, coming, whatever, want]\n",
       "1417    [wonder, cos, dun, rem, seeing, silver, car, t...\n",
       "2182                            [ps, u, ur, grown, right]\n",
       "3801                       [da, happy, sit, together, na]\n",
       "3197    [7, lor, change, 2, suntec, wat, time, u, coming]\n",
       "461     [great, never, better, day, gives, even, reaso...\n",
       "3287    [someone, u, know, asked, dating, service, 2, ...\n",
       "2520    [misplaced, number, sending, texts, old, numbe...\n",
       "5311    [kkim, going, tirunelvali, week, see, uncle, a...\n",
       "1490    [cant, believe, said, many, things, morning, r...\n",
       "3032                          [aight, lemme, know, whats]\n",
       "                              ...                        \n",
       "4315    [cant, speak, bcaz, mobile, problem, listen, c...\n",
       "1060    [im, working, technical, support, voice, proce...\n",
       "4031             [im, happy, babe, woo, hoo, party, dude]\n",
       "456     [look, amy, ure, beautiful, intelligent, woman...\n",
       "5178    [sms, auction, brand, new, nokia, 7250, 4, auc...\n",
       "112     [im, ok, wif, cos, like, 2, try, new, things, ...\n",
       "1999                   [lmao, wheres, fish, memory, need]\n",
       "4766    [unique, user, id, 1172, removal, send, stop, ...\n",
       "5496          [yes, trust, u, buy, new, stuff, asap, try]\n",
       "1229                     [jus, ans, lar, ull, noe, later]\n",
       "730     [email, alertfrom, jeri, stewartsize, 2kbsubje...\n",
       "2097                                [cutest, girl, world]\n",
       "1576            [nosy, guess, idk, reacting, im, freaked]\n",
       "3449                    [ya, telling, abt, tht, incident]\n",
       "3139    [sexy, sexy, cum, text, im, wet, warm, ready, ...\n",
       "544     [girl, stay, bed, girl, doesnt, need, recovery...\n",
       "784     [dont, think, turns, like, randomlly, within, ...\n",
       "3018                  [wat, time, u, wan, 2, meet, later]\n",
       "2403    [jesus, christ, bitch, im, trying, give, drugs...\n",
       "2509    [u, wake, already, wat, u, u, picking, us, lat...\n",
       "2069    [eerie, nokia, tones, 4u, rply, tone, title, 8...\n",
       "1127    [height, oh, shit, situation, guy, throws, luv...\n",
       "1563            [get, ready, put, excellent, sub, face, ]\n",
       "469                       [many, company, tell, language]\n",
       "2939         [1, new, message, please, call, 08712400200]\n",
       "2745                    [r, ì, going, 4, todays, meeting]\n",
       "1112                          [means, still, think, teju]\n",
       "3237                          [good, swimsuit, allowed, ]\n",
       "5120             [private, 2003, account, statement, 078]\n",
       "3229    [feel, like, dick, keep, sleeping, texts, face...\n",
       "Name: clean_text, Length: 4457, dtype: object"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# What do the first ten messages in the training set look like?\n",
    "X_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "459     0\n",
       "1023    0\n",
       "3471    0\n",
       "4322    0\n",
       "74      0\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# What do the labels look like?\n",
    "y_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Let's save the training and test sets to ensure we are using the same data for each model\n",
    "X_train.to_csv('../../../data/X_train.csv', header=True, index = False)\n",
    "X_test.to_csv('../../../data/X_test.csv', header=True, index = False)\n",
    "y_train.to_csv('../../../data/y_train.csv', header=True, index = False)\n",
    "y_test.to_csv('../../../data/y_test.csv', header=True, index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}