codetoxic new.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load in \n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "import re\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "# Input data files are available in the \"../input/\" directory.\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n",
    "\n",
    "# Any results you write to the current directory are saved as output."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "df = pd.read_csv(\"dataset_ML/train.csv\")\n",
    "df.head()\n",
    "\n",
    "X = df.iloc[:,1]\n",
    "y = df.iloc[:,2:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "_uuid": "d50b162e646e22db62d2ecca3351423fc9c59857",
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"Explanation\\nWhy the edits made under my username Hardcore Metallica Fan were reverted? They weren't vandalisms, just closure on some GAs after I voted at New York Dolls FAC. And please don't remove the template from the talk page since I'm retired now.89.205.38.27\",\n",
       " \"D'aww! He matches this background colour I'm seemingly stuck with. Thanks.  (talk) 21:51, January 11, 2016 (UTC)\",\n",
       " \"Hey man, I'm really not trying to edit war. It's just that this guy is constantly removing relevant information and talking to me through edits instead of my talk page. He seems to care more about the formatting than the actual info.\",\n",
       " '\"\\nMore\\nI can\\'t make any real suggestions on improvement - I wondered if the section statistics should be later on, or a subsection of \"\"types of accidents\"\"  -I think the references may need tidying so that they are all in the exact same format ie date format etc. I can do that later on, if no-one else does first - if you have any preferences for formatting style on references or want to do it yourself please let me know.\\n\\nThere appears to be a backlog on articles for review so I guess there may be a delay until a reviewer turns up. It\\'s listed in the relevant form eg Wikipedia:Good_article_nominations#Transport  \"']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comments = list(X)\n",
    "comments[:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "_uuid": "0da2429f207e0ab7a4cdbc34e04f8e632ad624eb"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['explanation\\nwhy the edits made under my username hardcore metallica fan were reverted? they werent vandalisms, just closure on some gas after i voted at new york dolls fac. and please dont remove the template from the talk page since im retired now.89.205.38.27',\n",
       " 'daww! he matches this background colour im seemingly stuck with. thanks.  (talk) 21:51, january 11, 2016 (utc)',\n",
       " 'hey man, im really not trying to edit war. its just that this guy is constantly removing relevant information and talking to me through edits instead of my talk page. he seems to care more about the formatting than the actual info.',\n",
       " '\"\\nmore\\ni cant make any real suggestions on improvement - i wondered if the section statistics should be later on, or a subsection of \"\"types of accidents\"\"  -i think the references may need tidying so that they are all in the exact same format ie date format etc. i can do that later on, if no-one else does first - if you have any preferences for formatting style on references or want to do it yourself please let me know.\\n\\nthere appears to be a backlog on articles for review so i guess there may be a delay until a reviewer turns up. its listed in the relevant form eg wikipedia:good_article_nominations#transport  \"']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comments = [re.sub(r'(\\')', \"\", w.lower()) for w in comments]\n",
    "comments[:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "_uuid": "663d488aba729506ac9104a6a605ab068ee1654c"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['explanation why the edits made under my username hardcore metallica fan were reverted  they werent vandalisms  just closure on some gas after i voted at new york dolls fac  and please dont remove the template from the talk page since im retired now ',\n",
       " 'daww  he matches this background colour im seemingly stuck with  thanks    talk    january      utc ',\n",
       " 'hey man  im really not trying to edit war  its just that this guy is constantly removing relevant information and talking to me through edits instead of my talk page  he seems to care more about the formatting than the actual info ',\n",
       " ' more i cant make any real suggestions on improvement   i wondered if the section statistics should be later on  or a subsection of  types of accidents    i think the references may need tidying so that they are all in the exact same format ie date format etc  i can do that later on  if no one else does first   if you have any preferences for formatting style on references or want to do it yourself please let me know there appears to be a backlog on articles for review so i guess there may be a delay until a reviewer turns up  its listed in the relevant form eg wikipedia good article nominations transport   ']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comments = [re.sub('[^A-Za-z ]+', ' ', w) for w in comments]\n",
    "comments[:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "_uuid": "7d95918d6215e71d2acf939d4927da21748516c8"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['explanation why the edits made under my username hardcore metallica fan were reverted they werent vandalisms just closure on some gas after i voted at new york dolls fac and please dont remove the template from the talk page since im retired now',\n",
       " 'daww he matches this background colour im seemingly stuck with thanks talk january utc',\n",
       " 'hey man im really not trying to edit war its just that this guy is constantly removing relevant information and talking to me through edits instead of my talk page he seems to care more about the formatting than the actual info',\n",
       " 'more i cant make any real suggestions on improvement i wondered if the section statistics should be later on or a subsection of types of accidents i think the references may need tidying so that they are all in the exact same format ie date format etc i can do that later on if no one else does first if you have any preferences for formatting style on references or want to do it yourself please let me know there appears to be a backlog on articles for review so i guess there may be a delay until a reviewer turns up its listed in the relevant form eg wikipedia good article nominations transport']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comments = [re.sub( '\\s+', ' ', w ).strip() for w in comments]\n",
    "comments[:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>toxic</th>\n",
       "      <th>severe_toxic</th>\n",
       "      <th>obscene</th>\n",
       "      <th>threat</th>\n",
       "      <th>insult</th>\n",
       "      <th>identity_hate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.904156</td>\n",
       "      <td>0.990004</td>\n",
       "      <td>0.947052</td>\n",
       "      <td>0.997004</td>\n",
       "      <td>0.950636</td>\n",
       "      <td>0.991195</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.095844</td>\n",
       "      <td>0.009996</td>\n",
       "      <td>0.052948</td>\n",
       "      <td>0.002996</td>\n",
       "      <td>0.049364</td>\n",
       "      <td>0.008805</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      toxic  severe_toxic   obscene    threat    insult  identity_hate\n",
       "0  0.904156      0.990004  0.947052  0.997004  0.950636       0.991195\n",
       "1  0.095844      0.009996  0.052948  0.002996  0.049364       0.008805"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>toxic</th>\n",
       "      <th>severe_toxic</th>\n",
       "      <th>obscene</th>\n",
       "      <th>threat</th>\n",
       "      <th>insult</th>\n",
       "      <th>identity_hate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.904028</td>\n",
       "      <td>0.990042</td>\n",
       "      <td>0.946947</td>\n",
       "      <td>0.996994</td>\n",
       "      <td>0.950589</td>\n",
       "      <td>0.991109</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.095972</td>\n",
       "      <td>0.009958</td>\n",
       "      <td>0.053053</td>\n",
       "      <td>0.003006</td>\n",
       "      <td>0.049411</td>\n",
       "      <td>0.008891</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      toxic  severe_toxic   obscene    threat    insult  identity_hate\n",
       "0  0.904028      0.990042  0.946947  0.996994  0.950589       0.991109\n",
       "1  0.095972      0.009958  0.053053  0.003006  0.049411       0.008891"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>toxic</th>\n",
       "      <th>severe_toxic</th>\n",
       "      <th>obscene</th>\n",
       "      <th>threat</th>\n",
       "      <th>insult</th>\n",
       "      <th>identity_hate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.906972</td>\n",
       "      <td>0.990161</td>\n",
       "      <td>0.948018</td>\n",
       "      <td>0.997274</td>\n",
       "      <td>0.951465</td>\n",
       "      <td>0.991321</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.093028</td>\n",
       "      <td>0.009839</td>\n",
       "      <td>0.051982</td>\n",
       "      <td>0.002726</td>\n",
       "      <td>0.048535</td>\n",
       "      <td>0.008679</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      toxic  severe_toxic   obscene    threat    insult  identity_hate\n",
       "0  0.906972      0.990161  0.948018  0.997274  0.951465       0.991321\n",
       "1  0.093028      0.009839  0.051982  0.002726  0.048535       0.008679"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>toxic</th>\n",
       "      <th>severe_toxic</th>\n",
       "      <th>obscene</th>\n",
       "      <th>threat</th>\n",
       "      <th>insult</th>\n",
       "      <th>identity_hate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.901144</td>\n",
       "      <td>0.98966</td>\n",
       "      <td>0.946264</td>\n",
       "      <td>0.99671</td>\n",
       "      <td>0.949789</td>\n",
       "      <td>0.991383</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.098856</td>\n",
       "      <td>0.01034</td>\n",
       "      <td>0.053736</td>\n",
       "      <td>0.00329</td>\n",
       "      <td>0.050211</td>\n",
       "      <td>0.008617</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      toxic  severe_toxic   obscene   threat    insult  identity_hate\n",
       "0  0.901144       0.98966  0.946264  0.99671  0.949789       0.991383\n",
       "1  0.098856       0.01034  0.053736  0.00329  0.050211       0.008617"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(comments, y, test_size=0.2, random_state=123)\n",
    "\n",
    "X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.2, random_state=123)\n",
    "\n",
    "display(y.apply(lambda x:  x.value_counts()/x.value_counts().sum()))\n",
    "display(y_train.apply(lambda x:  x.value_counts()/x.value_counts().sum()))\n",
    "display(y_test.apply(lambda x:  x.value_counts()/x.value_counts().sum()))\n",
    "display(y_val.apply(lambda x:  x.value_counts()/x.value_counts().sum()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "_uuid": "ec040ced4cb236d14ee18f82d1540ea63cf4e709"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<102124x134254 sparse matrix of type '<type 'numpy.int64'>'\n",
       "\twith 2554040 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "vect = CountVectorizer(decode_error='ignore',stop_words='english')\n",
    "train_dtm=vect.fit_transform(X_train)\n",
    "train_dtm\n",
    "#vect2 = TfidfVectorizer(decode_error='ignore',stop_words='english')\n",
    "#train_emails_tfid=vect2.fit_transform(comments)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "_uuid": "3c295cf72d6e18d2393cb6de48e227b2693b9b5c"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MultinomialNB(alpha=0.01, class_prior=None, fit_prior=True)"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.naive_bayes import MultinomialNB\n",
    "nb = MultinomialNB(alpha=0.01)\n",
    "nb.fit(train_dtm, y_train.toxic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "_uuid": "5348b72f9faf68db2392ce35d4446aa3c0318349"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.97160314911284318"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred_nb = nb.predict(train_dtm)\n",
    "from sklearn import metrics\n",
    "nbAcc=metrics.accuracy_score(y_train.toxic, y_pred_nb)\n",
    "nbAcc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_dtm=vect.transform(X_test)\n",
    "y_pred_nb_test = nb.predict(test_dtm)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "_uuid": "a0dd4bf32daef80e2d154c3e249a76f6b5a39e4c"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.94582484725050919"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nbAcc_test = metrics.accuracy_score(y_test.toxic, y_pred_nb_test)\n",
    "nbAcc_test"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}