1_07_FeatureEngineering5_xgboost_try.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Go back to basics, figure out which class may have issue in predicting. Also try xgboost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import re\n",
    "import random\n",
    "import scipy.stats as scs\n",
    "import mskcc_functions as ski\n",
    "import feature_engineering as fe\n",
    "import xgboost as xgb\n",
    "\n",
    "from matplotlib  import cm\n",
    "from collections import Counter\n",
    "from importlib import reload\n",
    "from nltk import PorterStemmer\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "from sklearn.preprocessing import OneHotEncoder, LabelEncoder\n",
    "from sklearn.preprocessing import scale, normalize, robust_scale\n",
    "from sklearn.model_selection import train_test_split, StratifiedKFold\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import accuracy_score, log_loss\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "from sklearn.decomposition import PCA\n",
    "\n",
    "sns.set_context(\"paper\")\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Import the data files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<input>:2: DeprecationWarning: invalid escape sequence \\|\n",
      "<input>:2: DeprecationWarning: invalid escape sequence \\|\n",
      "<input>:2: DeprecationWarning: invalid escape sequence \\|\n",
      "<ipython-input-2-8825876aa8b2>:2: DeprecationWarning: invalid escape sequence \\|\n",
      "  text_train = pd.read_csv(\"./data/training_text\", sep=\"\\|\\|\", engine='python', header=None, skiprows=1, names=[\"ID\",\"Text\"])\n"
     ]
    }
   ],
   "source": [
    "class_train = pd.read_csv('./data/training_variants')\n",
    "text_train = pd.read_csv(\"./data/training_text\", sep=\"\\|\\|\", engine='python', header=None, skiprows=1, names=[\"ID\",\"Text\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>Function to create a table to look at true class vs predicted class</b>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Create mock predicted labels\n",
    "y = np.array(class_train['Class']).ravel()\n",
    "yhat = np.random.randint(low=1, high=10, size=len(class_train)).ravel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "9 in yhat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def print_classification_table(y, yhat):\n",
    "    '''\n",
    "    INPUT:\n",
    "    ======\n",
    "    y : array\n",
    "        array containing true class labels\n",
    "    yhat: array\n",
    "        array containing predicted class labels\n",
    "    \n",
    "    OUTPUT:\n",
    "    =======\n",
    "    table : DataFrame\n",
    "        DataFrame containing classification results\n",
    "    '''\n",
    "    y_yhat = pd.DataFrame([y, yhat], index=['true_class', 'pred_class']).T\n",
    "    table = y_yhat.groupby(['true_class','pred_class']).size().unstack('pred_class')\n",
    "    table = table.fillna(value=0)\n",
    "    num_classes = table.shape[0]\n",
    "    table['class_count'] = 0\n",
    "    table['truely_IDed'] = 0\n",
    "    table['true_ID_rate'] = 0\n",
    "    for i in range(num_classes):\n",
    "        table.iloc[i, -3] = np.sum(table.iloc[i, :])\n",
    "        table.iloc[i, -2]= table.iloc[i,i]\n",
    "        table.iloc[i, -1] = (table.iloc[i, -2] / table.iloc[i, -3])\n",
    "        \n",
    "    return table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>pred_class</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>class_count</th>\n",
       "      <th>truely_IDed</th>\n",
       "      <th>true_ID_rate</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>true_class</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>72</td>\n",
       "      <td>67</td>\n",
       "      <td>62</td>\n",
       "      <td>68</td>\n",
       "      <td>60</td>\n",
       "      <td>60</td>\n",
       "      <td>59</td>\n",
       "      <td>61</td>\n",
       "      <td>59</td>\n",
       "      <td>568.0</td>\n",
       "      <td>72</td>\n",
       "      <td>0.126761</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>39</td>\n",
       "      <td>60</td>\n",
       "      <td>55</td>\n",
       "      <td>47</td>\n",
       "      <td>48</td>\n",
       "      <td>49</td>\n",
       "      <td>54</td>\n",
       "      <td>53</td>\n",
       "      <td>47</td>\n",
       "      <td>452.0</td>\n",
       "      <td>60</td>\n",
       "      <td>0.132743</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>7</td>\n",
       "      <td>12</td>\n",
       "      <td>13</td>\n",
       "      <td>12</td>\n",
       "      <td>13</td>\n",
       "      <td>12</td>\n",
       "      <td>4</td>\n",
       "      <td>89.0</td>\n",
       "      <td>7</td>\n",
       "      <td>0.078652</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>70</td>\n",
       "      <td>76</td>\n",
       "      <td>77</td>\n",
       "      <td>70</td>\n",
       "      <td>75</td>\n",
       "      <td>82</td>\n",
       "      <td>91</td>\n",
       "      <td>83</td>\n",
       "      <td>62</td>\n",
       "      <td>686.0</td>\n",
       "      <td>70</td>\n",
       "      <td>0.102041</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>26</td>\n",
       "      <td>31</td>\n",
       "      <td>26</td>\n",
       "      <td>27</td>\n",
       "      <td>22</td>\n",
       "      <td>28</td>\n",
       "      <td>26</td>\n",
       "      <td>30</td>\n",
       "      <td>26</td>\n",
       "      <td>242.0</td>\n",
       "      <td>22</td>\n",
       "      <td>0.090909</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>28</td>\n",
       "      <td>33</td>\n",
       "      <td>23</td>\n",
       "      <td>38</td>\n",
       "      <td>34</td>\n",
       "      <td>30</td>\n",
       "      <td>23</td>\n",
       "      <td>275.0</td>\n",
       "      <td>38</td>\n",
       "      <td>0.138182</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>117</td>\n",
       "      <td>108</td>\n",
       "      <td>96</td>\n",
       "      <td>122</td>\n",
       "      <td>95</td>\n",
       "      <td>98</td>\n",
       "      <td>123</td>\n",
       "      <td>99</td>\n",
       "      <td>95</td>\n",
       "      <td>953.0</td>\n",
       "      <td>123</td>\n",
       "      <td>0.129066</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>19.0</td>\n",
       "      <td>3</td>\n",
       "      <td>0.157895</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>37.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.027027</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "pred_class    1    2   3    4   5   6    7   8   9  class_count  truely_IDed  \\\n",
       "true_class                                                                     \n",
       "1            72   67  62   68  60  60   59  61  59        568.0           72   \n",
       "2            39   60  55   47  48  49   54  53  47        452.0           60   \n",
       "3             8    8   7   12  13  12   13  12   4         89.0            7   \n",
       "4            70   76  77   70  75  82   91  83  62        686.0           70   \n",
       "5            26   31  26   27  22  28   26  30  26        242.0           22   \n",
       "6            33   33  28   33  23  38   34  30  23        275.0           38   \n",
       "7           117  108  96  122  95  98  123  99  95        953.0          123   \n",
       "8             1    3   2    2   3   3    1   3   1         19.0            3   \n",
       "9             9    4   2    4   3   4    6   4   1         37.0            1   \n",
       "\n",
       "pred_class  true_ID_rate  \n",
       "true_class                \n",
       "1               0.126761  \n",
       "2               0.132743  \n",
       "3               0.078652  \n",
       "4               0.102041  \n",
       "5               0.090909  \n",
       "6               0.138182  \n",
       "7               0.129066  \n",
       "8               0.157895  \n",
       "9               0.027027  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# test if the function workds\n",
    "table = print_classification_table(y, yhat)\n",
    "table"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>Prediction with Variants Info Only</b>\n",
    "- Keep it simple by using data that appear in the variants file\n",
    "- Predict with RandomForest, print the classification, figure out which part needs fiddling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Import Data and extract gene and mutation type info\n",
    "# Get Gene feature from 'train_variants' data\n",
    "X_gene = np.array(class_train.Gene)\n",
    "X_gene_int = LabelEncoder().fit_transform(X_gene.ravel()).reshape(-1, 1)\n",
    "X_gene_bin = OneHotEncoder().fit_transform(X_gene_int).toarray()\n",
    "gene_table = pd.DataFrame(X_gene_bin)\n",
    "\n",
    "# Get Mutation Type from 'train_variants' data\n",
    "mut_type = ski.convert_mutation_type(class_train)\n",
    "X_mtype = np.array(mut_type['mutation_type'])\n",
    "X_mtype_int = LabelEncoder().fit_transform(X_mtype.ravel()).reshape(-1, 1)\n",
    "X_mtype_bin = OneHotEncoder().fit_transform(X_mtype_int).toarray()\n",
    "mtype_table = pd.DataFrame(X_mtype_bin)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "features1 = pd.concat([gene_table, mtype_table], axis=1)\n",
    "X = np.array(features1).astype(float)\n",
    "y = np.array(class_train.Class).astype(int).ravel()\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.15, random_state=55)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.575150300601\n",
      "1.27738725603\n",
      "CPU times: user 667 ms, sys: 9.36 ms, total: 677 ms\n",
      "Wall time: 458 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Run RFC on the data\n",
    "rfc = RandomForestClassifier(n_estimators=100, max_depth=100, n_jobs=4)\n",
    "rfc.fit(X_train, y_train)\n",
    "y_pred = rfc.predict(X_test)\n",
    "y_proba = rfc.predict_proba(X_test)\n",
    "print(accuracy_score(y_test, y_pred))\n",
    "print(log_loss(y_test, y_proba))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>pred_class</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>class_count</th>\n",
       "      <th>truely_IDed</th>\n",
       "      <th>true_ID_rate</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>true_class</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>46.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>95.0</td>\n",
       "      <td>46.0</td>\n",
       "      <td>0.484211</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>61.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>0.295082</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.153846</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>18.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>61.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>96.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.062500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>38.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.105263</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>37.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.189189</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>130.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>154.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "pred_class     1     2     4     5     6      7    8    9  class_count  \\\n",
       "true_class                                                               \n",
       "1           46.0   2.0  24.0   8.0   8.0    7.0  0.0  0.0         95.0   \n",
       "2            2.0  18.0   1.0   0.0   0.0   40.0  0.0  0.0         61.0   \n",
       "3            0.0   0.0   2.0   2.0   0.0    9.0  0.0  0.0         13.0   \n",
       "4           18.0   0.0  61.0   6.0   2.0    9.0  0.0  0.0         96.0   \n",
       "5            1.0   0.0   3.0  14.0   4.0   16.0  0.0  0.0         38.0   \n",
       "6            7.0   0.0   2.0   7.0  14.0    7.0  0.0  0.0         37.0   \n",
       "7            1.0  20.0   0.0   0.0   3.0  130.0  0.0  0.0        154.0   \n",
       "8            0.0   0.0   0.0   0.0   0.0    0.0  1.0  0.0          1.0   \n",
       "9            0.0   0.0   0.0   1.0   0.0    0.0  0.0  3.0          4.0   \n",
       "\n",
       "pred_class  truely_IDed  true_ID_rate  \n",
       "true_class                             \n",
       "1                  46.0      0.484211  \n",
       "2                  18.0      0.295082  \n",
       "3                   2.0      0.153846  \n",
       "4                   6.0      0.062500  \n",
       "5                   4.0      0.105263  \n",
       "6                   7.0      0.189189  \n",
       "7                   0.0      0.000000  \n",
       "8                   0.0      0.000000  \n",
       "9                   4.0      1.000000  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Assess with the table function abovev\n",
    "print_classification_table(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average Accuracy : 0.589086\n",
      "Average log loss : 1.269971\n"
     ]
    }
   ],
   "source": [
    "accuracies = []\n",
    "log_loss_scores = []\n",
    "\n",
    "skf = StratifiedKFold(n_splits=10, shuffle=True)\n",
    "for train_index, test_index in skf.split(X, y):\n",
    "    X_train, X_test = X[train_index], X[test_index]\n",
    "    y_train, y_test = y[train_index], y[test_index]\n",
    "    \n",
    "    rfc = RandomForestClassifier(n_estimators=100, max_depth=100, n_jobs=4)\n",
    "    rfc.fit(X_train, y_train)\n",
    "    \n",
    "    y_pred = rfc.predict(X_test)\n",
    "    y_proba = rfc.predict_proba(X_test)\n",
    "    \n",
    "    accuracies.append(accuracy_score(y_test, y_pred))\n",
    "    log_loss_scores.append(log_loss(y_test, y_proba, labels=list(range(1,10))))\n",
    "\n",
    "print('Average Accuracy : %f' % np.mean(accuracies))\n",
    "print('Average log loss : %f' % np.mean(log_loss_scores))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "<b>Try XGBoost</b><br>\n",
    "This code was adopted from the1owl's public kernel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Adjust the class labels so it starts with 0\n",
    "y_adj = y-1\n",
    "\n",
    "# Split into test and train\n",
    "x1, x2, y1, y2 = train_test_split(X, y_adj, test_size=0.15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\ttrain-mlogloss:2.16359\tvalid-mlogloss:2.16588\n",
      "Multiple eval metrics have been passed: 'valid-mlogloss' will be used for early stopping.\n",
      "\n",
      "Will train until valid-mlogloss hasn't improved in 100 rounds.\n",
      "[500]\ttrain-mlogloss:1.14157\tvalid-mlogloss:1.22483\n",
      "[1000]\ttrain-mlogloss:1.05931\tvalid-mlogloss:1.16327\n",
      "[1500]\ttrain-mlogloss:1.01602\tvalid-mlogloss:1.13211\n",
      "[2000]\ttrain-mlogloss:0.991336\tvalid-mlogloss:1.11645\n",
      "[2500]\ttrain-mlogloss:0.976036\tvalid-mlogloss:1.10708\n",
      "[3000]\ttrain-mlogloss:0.96508\tvalid-mlogloss:1.10258\n",
      "[3500]\ttrain-mlogloss:0.957895\tvalid-mlogloss:1.09841\n",
      "[4000]\ttrain-mlogloss:0.952241\tvalid-mlogloss:1.09574\n",
      "[4500]\ttrain-mlogloss:0.947341\tvalid-mlogloss:1.09409\n",
      "[4999]\ttrain-mlogloss:0.942881\tvalid-mlogloss:1.09313\n",
      "CPU times: user 32min 11s, sys: 746 ms, total: 32min 12s\n",
      "Wall time: 4min 5s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Set up parameters for xgboost\n",
    "param = params = {\n",
    "        'eta': 0.03333,\n",
    "        'max_depth': 4,\n",
    "        'objective': 'multi:softprob',\n",
    "        'eval_metric': 'mlogloss',\n",
    "        'num_class': 9,\n",
    "        'seed': 1,\n",
    "        'silent': True\n",
    "        }\n",
    "\n",
    "watchlist = [(xgb.DMatrix(x1, y1), 'train'), (xgb.DMatrix(x2, y2), 'valid')]\n",
    "\n",
    "model = xgb.train(params, xgb.DMatrix(x1, y1), 5000,  watchlist, verbose_eval=500, early_stopping_rounds=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.0930910647\n"
     ]
    }
   ],
   "source": [
    "score = log_loss(y2, model.predict(xgb.DMatrix(x2), ntree_limit=model.best_ntree_limit), labels = list(range(9)))\n",
    "print(score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "xgboost appears to do slightly better than RandomForest. I need to create a better features (better vector space for the text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>Prediction with Variants info only - excluding point mutants and 'other'</b>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# get indices for rows that have 'Point Mutation' mutation type\n",
    "pm_ind = list(np.argwhere(X_mtype == 'Point Mutation').ravel())\n",
    "nonpm_ind = [ind for ind in range(len(X)) if ind not in pm_ind]\n",
    "\n",
    "# Slice X and y\n",
    "X_nonpm = X[nonpm_ind, :]\n",
    "y_nonpm = y[nonpm_ind]\n",
    "\n",
    "# Split dataset\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_nonpm, y_nonpm, test_size=0.15, random_state=55)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.62\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "y_true and y_pred contain different number of classes 5, 9. Please provide the true labels explicitly through the labels argument. Classes found in y_true: [1 2 4 5 7]",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<timed exec>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m/usr/lib64/python3.6/site-packages/sklearn/metrics/classification.py\u001b[0m in \u001b[0;36mlog_loss\u001b[0;34m(y_true, y_pred, eps, normalize, sample_weight, labels)\u001b[0m\n\u001b[1;32m   1652\u001b[0m                              \"y_true: {2}\".format(transformed_labels.shape[1],\n\u001b[1;32m   1653\u001b[0m                                                   \u001b[0my_pred\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1654\u001b[0;31m                                                   lb.classes_))\n\u001b[0m\u001b[1;32m   1655\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1656\u001b[0m             raise ValueError('The number of classes in labels is different '\n",
      "\u001b[0;31mValueError\u001b[0m: y_true and y_pred contain different number of classes 5, 9. Please provide the true labels explicitly through the labels argument. Classes found in y_true: [1 2 4 5 7]"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Run RFC on the sliced dataset\n",
    "rfc = RandomForestClassifier(n_estimators=100, max_depth=100, n_jobs=4)\n",
    "rfc.fit(X_train, y_train)\n",
    "y_pred = rfc.predict(X_test)\n",
    "y_proba = rfc.predict_proba(X_test)\n",
    "print(accuracy_score(y_test, y_pred))\n",
    "print(log_loss(y_test, y_proba))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "I'd need to rewrite the function to account for this. The class that doesn't appear in the true labels was predicted to exist..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b></b>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b></b>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b></b>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b></b>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b></b>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b></b>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b></b>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}