streamline1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import re\n",
    "import random\n",
    "import json\n",
    "import scipy.stats as scs\n",
    "import feature_engineering as fe\n",
    "import feature_engineering.text_processing as fete\n",
    "import feature_engineering.frequency_selection as fefs\n",
    "import csv\n",
    "import xgboost as xgb\n",
    "\n",
    "from xgboost import plot_importance\n",
    "from pprint import pprint\n",
    "from matplotlib  import cm\n",
    "from collections import Counter\n",
    "from importlib import reload\n",
    "from gensim import corpora, matutils, models, similarities\n",
    "from nltk import PorterStemmer\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "from sklearn.preprocessing import OneHotEncoder, LabelEncoder\n",
    "from sklearn.preprocessing import scale, normalize, robust_scale\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import accuracy_score, log_loss\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from mpl_toolkits.mplot3d import Axes3D"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<input>:3: DeprecationWarning: invalid escape sequence \\|\n",
      "<input>:3: DeprecationWarning: invalid escape sequence \\|\n",
      "<input>:3: DeprecationWarning: invalid escape sequence \\|\n",
      "<input>:3: DeprecationWarning: invalid escape sequence \\|\n",
      "<input>:3: DeprecationWarning: invalid escape sequence \\|\n",
      "<input>:3: DeprecationWarning: invalid escape sequence \\|\n",
      "<input>:3: DeprecationWarning: invalid escape sequence \\|\n",
      "<input>:3: DeprecationWarning: invalid escape sequence \\|\n",
      "<input>:3: DeprecationWarning: invalid escape sequence \\|\n",
      "<input>:3: DeprecationWarning: invalid escape sequence \\|\n",
      "<input>:3: DeprecationWarning: invalid escape sequence \\|\n",
      "<input>:3: DeprecationWarning: invalid escape sequence \\|\n",
      "<input>:3: DeprecationWarning: invalid escape sequence \\|\n",
      "<input>:3: DeprecationWarning: invalid escape sequence \\|\n",
      "<input>:3: DeprecationWarning: invalid escape sequence \\|\n",
      "<input>:3: DeprecationWarning: invalid escape sequence \\|\n",
      "<input>:3: DeprecationWarning: invalid escape sequence \\|\n",
      "<input>:3: DeprecationWarning: invalid escape sequence \\|\n",
      "<ipython-input-3-a333d09d8244>:3: DeprecationWarning: invalid escape sequence \\|\n",
      "  text_train = pd.read_csv(\"./data/training_text\", sep=\"\\|\\|\", engine='python',\n"
     ]
    }
   ],
   "source": [
    "# Import data into pandas, merge class data and text data on ID\n",
    "class_train = pd.read_csv('./data/training_variants')\n",
    "text_train = pd.read_csv(\"./data/training_text\", sep=\"\\|\\|\", engine='python',\n",
    "                         header=None, skiprows=1, names=[\"ID\",\"Text\"])\n",
    "train = class_train.merge(text_train, on='ID')\n",
    "\n",
    "# create class label container\n",
    "class_labels = []\n",
    "for i in range(9):\n",
    "    class_labels.append('class' + str(i+1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make a stemmer object, define stop words\n",
    "stemmer = PorterStemmer()\n",
    "stop_words = set(stopwords.words('english'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>1-gram processing</b>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "class1 being processed...\n",
      "class2 being processed...\n",
      "class3 being processed...\n",
      "class4 being processed...\n",
      "class5 being processed...\n",
      "class6 being processed...\n",
      "class7 being processed...\n",
      "class8 being processed...\n",
      "class9 being processed...\n",
      "CPU times: user 7min 25s, sys: 390 ms, total: 7min 25s\n",
      "Wall time: 7min 26s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Create a dictionay. Key is the class label.\n",
    "# Value is \n",
    "classified_tokenized_docs = {}\n",
    "for i in range(9):\n",
    "    print('%s being processed...' % class_labels[i])\n",
    "    docs = [doc for j, doc in enumerate(train[train.Class == (i+1)]['Text'])]\n",
    "    \n",
    "    tokenized_docs = []\n",
    "    for k, doc in enumerate(docs):\n",
    "        # tokenize the doc (DO NOT MAKE IT A SET FOR LATER USE)\n",
    "        tokenized_doc = word_tokenize(fete.replace_with_whitespace(doc, hyphens='on'))\n",
    "\n",
    "        # Remove stop words and words with special characters\n",
    "        tokenized_doc = [word for word in tokenized_doc \\\n",
    "                         if re.search(r'^[A-Za-z]', word) \\\n",
    "                         if re.search(r'[A-Za-z0-9]$', word) \\\n",
    "                         if not re.search(r'[@#%&*()+=]', word) \\\n",
    "                         if len(word) > 1 \\\n",
    "                         if word.lower() not in stop_words]\n",
    "\n",
    "        # Apply stemmer to each word in the list\n",
    "        tokenized_doc = [stemmer.stem(word) for word in tokenized_doc]\n",
    "        \n",
    "        tokenized_docs.append(tokenized_doc)\n",
    "    \n",
    "    classified_tokenized_docs[class_labels[i]] = tokenized_docs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "class1 being processed...\n",
      "class2 being processed...\n",
      "class3 being processed...\n",
      "class4 being processed...\n",
      "class5 being processed...\n",
      "class6 being processed...\n",
      "class7 being processed...\n",
      "class8 being processed...\n",
      "class9 being processed...\n",
      "CPU times: user 25.3 s, sys: 509 ms, total: 25.8 s\n",
      "Wall time: 25.8 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Create adictionary. Keys are class labels\n",
    "# appearance_frequency looks at what fraction of\n",
    "# the doc within each class the token appears\n",
    "appearance_frequency = {}\n",
    "for i in range(9):\n",
    "    print('%s being processed...' % class_labels[i])\n",
    "    tokenized_docs = classified_tokenized_docs[class_labels[i]]\n",
    "    num_docs = len(tokenized_docs)\n",
    "    \n",
    "    app_freq_list = []\n",
    "    for doc in tokenized_docs:\n",
    "        c = Counter(doc)\n",
    "        freq = dict(c)\n",
    "        app_freq = {key:1 for key, value in freq.items() if value > 0}\n",
    "        app_freq_list.append(app_freq)\n",
    "    app_freq_table = pd.DataFrame(app_freq_list)\n",
    "    app_freq = dict(app_freq_table.sum(axis=0)/num_docs)\n",
    "    \n",
    "    appearance_frequency[class_labels[i]] = app_freq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# intersecting words among top%d appearing words in each class:  1391\n",
      "# intersecting words with >50% appearance:  287\n",
      "Table shape before removal:  (125448, 9)\n",
      "Table shape after removal:   (125161, 9)\n",
      "CPU times: user 519 ms, sys: 1 µs, total: 519 ms\n",
      "Wall time: 520 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Remove words that appear in high frequency that \n",
    "# are intersecting words among top 3000 words\n",
    "# in each class\n",
    "fracdocs = pd.DataFrame(appearance_frequency).fillna(value=0)\n",
    "n = 3000\n",
    "top_words = []\n",
    "for i in range(9):\n",
    "    tops = fracdocs[class_labels[i]].sort_values(ascending=False).head(n)\n",
    "    top_words.append(list(tops.index))\n",
    "\n",
    "overlap1 = set(top_words[0])\n",
    "for lis in top_words[1:]:\n",
    "    overlap1.intersection_update(lis)\n",
    "print('# intersecting words among top%d appearing words in each class: ', len(overlap1))\n",
    "    \n",
    "remove_list = []\n",
    "for i in range(9):\n",
    "    remove_words = [word for word in overlap1 \\\n",
    "                    if word in fracdocs[class_labels[i]] \\\n",
    "                    if fracdocs[class_labels[i]][word] > 0.5]\n",
    "    remove_list.append(list(remove_words))\n",
    "\n",
    "overlap2 = set(remove_list[0])\n",
    "for lis in remove_list[1:]:\n",
    "    overlap2.intersection_update(lis)\n",
    "print('# intersecting words with >50% appearance: ', len(overlap2))\n",
    "\n",
    "print('Table shape before removal: ', fracdocs.shape)\n",
    "fracdocs = fracdocs.drop(overlap2)\n",
    "print('Table shape after removal:  ', fracdocs.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====== n-class words extractions by relative differecne ======\n",
      "Input Type: fraction_of_docs\n",
      "Minimum Difference = 1.35\n",
      "Minimum Frequency = 0.45\n",
      "# of words in one_class_words: 185\n",
      "# of words in two_class_words: 56\n",
      "# of words in three_class_words: 5\n",
      "# of words in four_class_words: 7\n",
      "# of words in five_class_words: 2\n",
      "# of words in six_class_words: 3\n",
      "# of words in seven_class_words: 19\n",
      "# of words in eight_class_words: 36\n",
      "# of words in other_words: 124842\n",
      "Total # of words: 125155\n",
      "====== n-class words extractions by absolute differecne ======\n",
      "Input Type: fraction_of_docs\n",
      "Minimum Difference = 0.10\n",
      "Minimum Frequency = 0.60\n",
      "# of words in one_class_words: 176\n",
      "# of words in two_class_words: 50\n",
      "# of words in three_class_words: 7\n",
      "# of words in four_class_words: 5\n",
      "# of words in five_class_words: 6\n",
      "# of words in six_class_words: 3\n",
      "# of words in seven_class_words: 6\n",
      "# of words in eight_class_words: 6\n",
      "# of words in other_words: 124897\n",
      "Total # of words: 125156\n",
      "===== n-class words extractions by exclusive appearances =====\n",
      "Minimum Frequency = 0.175000\n",
      "# of words in one_class_words: 9\n",
      "# of words in two_class_words: 0\n",
      "# of words in three_class_words: 0\n",
      "# of words in four_class_words: 0\n",
      "# of words in five_class_words: 0\n",
      "# of words in six_class_words: 0\n",
      "# of words in seven_class_words: 0\n",
      "# of words in eight_class_words: 2\n",
      "# of words in other_words: 125150\n",
      "Total # of words: 125161\n",
      "'Relative' extraction: 313 words\n",
      "'Absolute' extraction: 259 words\n",
      "'Exclusive' extraction: 11 words\n",
      "# unique words extracted: 448 words\n",
      "CPU times: user 27.4 s, sys: 290 ms, total: 27.7 s\n",
      "Wall time: 27.2 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Use relative functions to select words by\n",
    "# differential appearance frequency in each class\n",
    "ncw_relative = fefs.getNClassWords(fracdocs, doc_type='fraction_of_docs',\n",
    "                            mode='relative', min_frequency=0.45,\n",
    "                            min_difference=1.35, print_result=True)\n",
    "ncw_absolute = fefs.getNClassWords(fracdocs, doc_type='fraction_of_docs',\n",
    "                            mode='absolute', min_frequency=0.6,\n",
    "                            min_difference=0.1, print_result=True)\n",
    "ncw_exclusive = fefs.getNClassExclusiveWords(fracdocs, min_frequency=0.175, print_result=True)\n",
    "words1 = fefs.selectNClassWords(ncw_relative, n=8)\n",
    "words2 = fefs.selectNClassWords(ncw_absolute, n=8)\n",
    "words3 = fefs.selectNClassWords(ncw_exclusive, n=8)\n",
    "select_words = set((words1 + words2 + words3))\n",
    "\n",
    "print(\"'Relative' extraction: %d words\" % len(words1))\n",
    "print(\"'Absolute' extraction: %d words\" % len(words2))\n",
    "print(\"'Exclusive' extraction: %d words\" % len(words3))\n",
    "print(\"# unique words extracted: %d words\" % len(select_words))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}