create_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "from sklearn.utils import resample\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import cross_validate, KFold\n",
    "\n",
    "from sklearn.metrics import recall_score, roc_auc_score, f1_score\n",
    "from sklearn.metrics import accuracy_score, roc_auc_score, \\\n",
    "                            classification_report, confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Read data\n",
    "In the last three tutorials, I have processed data and finally selected some relevant features for the project. So let's read the data with selected features."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_selected = pd.read_csv('./data/df_selected.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>loan_amnt</th>\n",
       "      <th>term</th>\n",
       "      <th>int_rate</th>\n",
       "      <th>installment</th>\n",
       "      <th>grade</th>\n",
       "      <th>sub_grade</th>\n",
       "      <th>emp_length</th>\n",
       "      <th>home_ownership</th>\n",
       "      <th>annual_inc</th>\n",
       "      <th>verification_status</th>\n",
       "      <th>...</th>\n",
       "      <th>total_acc</th>\n",
       "      <th>initial_list_status</th>\n",
       "      <th>application_type</th>\n",
       "      <th>acc_open_past_24mths</th>\n",
       "      <th>mort_acc</th>\n",
       "      <th>pub_rec_bankruptcies</th>\n",
       "      <th>tax_liens</th>\n",
       "      <th>disbursement_method</th>\n",
       "      <th>issue_month</th>\n",
       "      <th>credit_history</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>457484.000000</td>\n",
       "      <td>457484.000000</td>\n",
       "      <td>457484.000000</td>\n",
       "      <td>457484.00000</td>\n",
       "      <td>457484.000000</td>\n",
       "      <td>457484.000000</td>\n",
       "      <td>457484.000000</td>\n",
       "      <td>457484.000000</td>\n",
       "      <td>4.574840e+05</td>\n",
       "      <td>457484.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>457484.000000</td>\n",
       "      <td>457484.000000</td>\n",
       "      <td>457484.000000</td>\n",
       "      <td>457484.000000</td>\n",
       "      <td>457484.000000</td>\n",
       "      <td>457484.000000</td>\n",
       "      <td>457484.000000</td>\n",
       "      <td>457484.000000</td>\n",
       "      <td>457484.000000</td>\n",
       "      <td>457484.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>14560.685237</td>\n",
       "      <td>42.118666</td>\n",
       "      <td>13.297232</td>\n",
       "      <td>440.29019</td>\n",
       "      <td>1.844924</td>\n",
       "      <td>11.201323</td>\n",
       "      <td>5.766084</td>\n",
       "      <td>2.397426</td>\n",
       "      <td>7.584082e+04</td>\n",
       "      <td>0.296970</td>\n",
       "      <td>...</td>\n",
       "      <td>26.017174</td>\n",
       "      <td>0.566293</td>\n",
       "      <td>0.003296</td>\n",
       "      <td>4.811952</td>\n",
       "      <td>1.773319</td>\n",
       "      <td>0.148659</td>\n",
       "      <td>0.056844</td>\n",
       "      <td>0.000026</td>\n",
       "      <td>5.567150</td>\n",
       "      <td>16.454720</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>8536.329074</td>\n",
       "      <td>10.459931</td>\n",
       "      <td>4.526475</td>\n",
       "      <td>254.19396</td>\n",
       "      <td>1.331678</td>\n",
       "      <td>6.579267</td>\n",
       "      <td>3.724258</td>\n",
       "      <td>1.425130</td>\n",
       "      <td>6.589603e+04</td>\n",
       "      <td>0.456924</td>\n",
       "      <td>...</td>\n",
       "      <td>12.165157</td>\n",
       "      <td>0.495586</td>\n",
       "      <td>0.057319</td>\n",
       "      <td>3.156273</td>\n",
       "      <td>2.083654</td>\n",
       "      <td>0.398301</td>\n",
       "      <td>0.420798</td>\n",
       "      <td>0.005122</td>\n",
       "      <td>3.395717</td>\n",
       "      <td>7.430867</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1000.000000</td>\n",
       "      <td>36.000000</td>\n",
       "      <td>5.320000</td>\n",
       "      <td>14.01000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000e+02</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>8000.000000</td>\n",
       "      <td>36.000000</td>\n",
       "      <td>9.990000</td>\n",
       "      <td>256.23000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>4.600000e+04</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>17.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>11.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>12400.000000</td>\n",
       "      <td>36.000000</td>\n",
       "      <td>12.990000</td>\n",
       "      <td>378.20000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>6.500000e+04</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>24.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>15.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>20000.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>15.990000</td>\n",
       "      <td>580.73000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>9.000000e+04</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>33.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>20.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>40000.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>30.990000</td>\n",
       "      <td>1714.54000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>34.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>8.900060e+06</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>169.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>53.000000</td>\n",
       "      <td>47.000000</td>\n",
       "      <td>12.000000</td>\n",
       "      <td>85.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>70.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           loan_amnt           term       int_rate   installment  \\\n",
       "count  457484.000000  457484.000000  457484.000000  457484.00000   \n",
       "mean    14560.685237      42.118666      13.297232     440.29019   \n",
       "std      8536.329074      10.459931       4.526475     254.19396   \n",
       "min      1000.000000      36.000000       5.320000      14.01000   \n",
       "25%      8000.000000      36.000000       9.990000     256.23000   \n",
       "50%     12400.000000      36.000000      12.990000     378.20000   \n",
       "75%     20000.000000      60.000000      15.990000     580.73000   \n",
       "max     40000.000000      60.000000      30.990000    1714.54000   \n",
       "\n",
       "               grade      sub_grade     emp_length  home_ownership  \\\n",
       "count  457484.000000  457484.000000  457484.000000   457484.000000   \n",
       "mean        1.844924      11.201323       5.766084        2.397426   \n",
       "std         1.331678       6.579267       3.724258        1.425130   \n",
       "min         0.000000       0.000000       0.000000        0.000000   \n",
       "25%         1.000000       6.000000       2.000000        1.000000   \n",
       "50%         2.000000      11.000000       6.000000        3.000000   \n",
       "75%         3.000000      15.000000      10.000000        4.000000   \n",
       "max         6.000000      34.000000      10.000000        4.000000   \n",
       "\n",
       "         annual_inc  verification_status       ...            total_acc  \\\n",
       "count  4.574840e+05        457484.000000       ...        457484.000000   \n",
       "mean   7.584082e+04             0.296970       ...            26.017174   \n",
       "std    6.589603e+04             0.456924       ...            12.165157   \n",
       "min    1.000000e+02             0.000000       ...             2.000000   \n",
       "25%    4.600000e+04             0.000000       ...            17.000000   \n",
       "50%    6.500000e+04             0.000000       ...            24.000000   \n",
       "75%    9.000000e+04             1.000000       ...            33.000000   \n",
       "max    8.900060e+06             1.000000       ...           169.000000   \n",
       "\n",
       "       initial_list_status  application_type  acc_open_past_24mths  \\\n",
       "count        457484.000000     457484.000000         457484.000000   \n",
       "mean              0.566293          0.003296              4.811952   \n",
       "std               0.495586          0.057319              3.156273   \n",
       "min               0.000000          0.000000              0.000000   \n",
       "25%               0.000000          0.000000              3.000000   \n",
       "50%               1.000000          0.000000              4.000000   \n",
       "75%               1.000000          0.000000              6.000000   \n",
       "max               1.000000          1.000000             53.000000   \n",
       "\n",
       "            mort_acc  pub_rec_bankruptcies      tax_liens  \\\n",
       "count  457484.000000         457484.000000  457484.000000   \n",
       "mean        1.773319              0.148659       0.056844   \n",
       "std         2.083654              0.398301       0.420798   \n",
       "min         0.000000              0.000000       0.000000   \n",
       "25%         0.000000              0.000000       0.000000   \n",
       "50%         1.000000              0.000000       0.000000   \n",
       "75%         3.000000              0.000000       0.000000   \n",
       "max        47.000000             12.000000      85.000000   \n",
       "\n",
       "       disbursement_method    issue_month  credit_history  \n",
       "count        457484.000000  457484.000000   457484.000000  \n",
       "mean              0.000026       5.567150       16.454720  \n",
       "std               0.005122       3.395717        7.430867  \n",
       "min               0.000000       0.000000        3.000000  \n",
       "25%               0.000000       3.000000       11.000000  \n",
       "50%               0.000000       5.000000       15.000000  \n",
       "75%               0.000000       9.000000       20.000000  \n",
       "max               1.000000      11.000000       70.000000  \n",
       "\n",
       "[8 rows x 30 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_selected.describe(include = 'all')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Class balance\n",
    "Before we jump into anything, we must take care of the class unbalance problems. The following code shows the number of examples in each class."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    0.783494\n",
       "1    0.216506\n",
       "Name: loan_status, dtype: float64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_selected.loan_status.value_counts(normalize=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    358436\n",
       "1     99048\n",
       "Name: loan_status, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_selected.loan_status.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Loan-status class is imbalanced. To solve the problem of unbalance class issue there are many techniques that can be applied. For example: \n",
    "\n",
    "(1) Assign a class weight (2) Use ensemble algorithms with cross-validation (3) Upsample minority class or downsample the majority class\n",
    "\n",
    "I wrote a blog post describing the above three techniques. In this work, I have tried all the techniques and found upsampling minority class improves the model's generalization on unseen data. I the code below I upsample minority class with Scikit-learn ‘resample’ method."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Upsample the minority class\n",
    "One of the popular techniques for dealing with highly unbalanced data sets is called resampling. Although the technique has proven to be effective in many cases to solve the unbalanced class issue, however, these techniques also have their weaknesses. For example, over-sampling records from the minority class, which can lead to overfitting while removing random records from the majority class, which can cause loss of information. Alright, now let's see how upsampling works better in this project:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_major = df_selected[df_selected.loan_status == 0]\n",
    "df_minor = df_selected[df_selected.loan_status == 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_minor_upsmapled = resample(df_minor, replace = True, n_samples = 358436, random_state = 2018)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_minor_upsmapled = pd.concat([df_minor_upsmapled, df_major])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    358436\n",
       "0    358436\n",
       "Name: loan_status, dtype: int64"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_minor_upsmapled.loan_status.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In the above code, I first separate classes into two data frames: 1. df_major and 2. df_minor. Then I use df_minor to upsample it to the same number as the major class which is 358436. Notice that I keep the replace option to true. If I were downsampled then I would keep the replace option to false. Finally, I concatenate the upsampled minor class with major class. Loom at the loan status value counts. They are the same now. Now it's time to standardize the data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 0. Evaluate the model\n",
    "To see the performance of the unknown data, I wrote a function named as \"evaluate_model\" which prints different evaluation criteria: 1) accuracy, 2) ROC-AUC score, 3) confusion matrix and 4) detailed classification report."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_model(ytest, ypred, ypred_proba = None):\n",
    "    if ypred_proba is not None:\n",
    "        print('ROC-AUC score of the model: {}'.format(roc_auc_score(ytest, ypred_proba[:, 1])))\n",
    "    print('Accuracy of the model: {}\\n'.format(accuracy_score(ytest, ypred)))\n",
    "    print('Classification report: \\n{}\\n'.format(classification_report(ytest, ypred)))\n",
    "    print('Confusion matrix: \\n{}\\n'.format(confusion_matrix(ytest, ypred)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1. Standarize the data\n",
    "In this section, I summarize the data by removing the mean from each sample and then divide by the standard deviation. Zero mean and unit standard deviation helps the model’s optimization faster. I used the Scikit-learn StandardScaler method. Before that, I split the dataset into training and testing parts. The following code is self-explanatory:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df_minor_upsmapled.drop('loan_status', axis = 1)\n",
    "Y = df_minor_upsmapled.loan_status"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "xtrain, xtest, ytrain, ytest = train_test_split(X, Y, test_size=0.25, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "mms = StandardScaler()\n",
    "mms.fit(xtrain)\n",
    "xtrain_scaled = mms.transform(xtrain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(716872, 30)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.shape(df_minor_upsmapled)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now that our data is ready, we can move on building models. I always start with a simple algorithm like logistic regression to keep things simple and record the performance as a benchmark for complex models."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2. logistic regression model\n",
    "\n",
    "Logistic regression is a modeling technique borrowed from statistics. It is handier and go-to method for binary classification problems. As I said before the algorithm is relatively simple and easy to implement, I always first start with this technique and record the performance of the model for future complex model benchmarking purpose. It helps me move forward easily and intuitively. Alright, let's see how logistic regression can perform:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "logisticRegr = LogisticRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logisticRegr.fit(xtrain_scaled, ytrain)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In the above code, I used default parameters. Bellow, I standardize the test data using the same standardization parameters (mean and standard deviation) used for training data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "xtest_scaled = mms.transform(xtest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr_pred = logisticRegr.predict(xtest_scaled)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finally, let's see the performance of the logistic regression:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of the model: 0.66409066053633\n",
      "\n",
      "Classification report: \n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.66      0.68      0.67     89877\n",
      "          1       0.67      0.65      0.66     89341\n",
      "\n",
      "avg / total       0.66      0.66      0.66    179218\n",
      "\n",
      "\n",
      "Confusion matrix: \n",
      "[[60846 29031]\n",
      " [31170 58171]]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "evaluate_model(ytest, lr_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The result is not promising. The accuracy of the model is just a little above the random guessing. We see that the simplest model gives 66% accuracy. Therefore, we have to pick a better algorithm and tune its hyperparameters in such a way that, the model outperforms the logistic regression model. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Choose an appropriate model\n",
    "\n",
    "Choosing an appropriate model is another challenge for a data scientist. Sometimes even an experienced data scientist cannot tell which algorithm will perform the best before trying different algorithms. In our final dataset, almost 60% of our features are categorical. Therefore, a tree-based model may be a better choice. Still, it’s very unpredictable. If tree-based algorithms do not perform very well we might try another algorithm such as the neural network. In the project, I would try both bagging (Random forest)and boosted tree-based (LightGBM) algorithms. Alright, let's begin with random forest."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Random forest model\n",
    "Random Forest is a flexible, easy to use machine learning ensemble algorithm. The algorithom is so light and effective that even without hyper-parameter tuning, it can produce can great result. It is also one of the most used algorithms, because it’s simplicity and the fact that it can be used for both classification and regression tasks. Details of the method can be found on Scikit-sklearn webpage or in this blog post: https://towardsdatascience.com/the-random-forest-algorithm-d457d499ffcd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "def random_forest(xtrain, xtest, ytrain):\n",
    "    rf_params = {\n",
    "        'n_estimators': 126, \n",
    "        'max_depth': 14\n",
    "    }\n",
    "\n",
    "    rf = RandomForestClassifier(**rf_params)\n",
    "    rf.fit(xtrain, ytrain)\n",
    "    rfpred = rf.predict(xtest)\n",
    "    rfpred_proba = rf.predict_proba(xtest)\n",
    "    \n",
    "    return rfpred, rfpred_proba"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In the above function, I first define the hyperparameters. The most important hyperparameters of random forest are the number of estimators and the maximum depth of a tree. I try to find the optimal hyperparameter value in the iterative process. I manually start with a small number of estimator then increase slowly. I find this manual process efficient and intuitive rather than using GridSearchCV or RandomSearch. There is another technique for Bayesian hyperparameter optimization, which can be used to find a suitable set of hyperparameters. The technique seems to be more efficient and effective. In my next project, I would try it. More details on this technique can be found in William Koehrsen's blog post \"A Conceptual Explanation of Bayesian Hyperparameter Optimization for Machine Learning\". As I said before, I start with a low number with most important hyperparameter and once I find the optimum value I start with the next influential hyperparameter and so on. Alright, it's time to see the performance of random forest on test data sets:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "rfpred, rfpred_proba = random_forest(xtrain_scaled, xtest_scaled, ytrain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ROC-AUC score of the model: 0.8054282761077389\n",
      "Accuracy of the model: 0.7304177035788816\n",
      "\n",
      "Classification report: \n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.75      0.69      0.72     89877\n",
      "          1       0.71      0.77      0.74     89341\n",
      "\n",
      "avg / total       0.73      0.73      0.73    179218\n",
      "\n",
      "\n",
      "Confusion matrix: \n",
      "[[61972 27905]\n",
      " [20409 68932]]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "evaluate_model(ytest, rfpred, rfpred_proba)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Wow, Radom forest does better. Almost 11% accuracy jump from logistic regression. Which is a great achievement and proves that tree-based models perform well on categorical data. At this point, I stopped working on the random forest model as the model performance does not increase. I tried with other hyperparameters and increasing the n_estimators, that did not help. These difficulties helped me decide to use a gradient boosted tree. Since I plan not  to move further with Random forest, I must find out the robustness of the model. One way to find out the cross-validation. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cross validation\n",
    "Cross-validation is one of the effective ways to assess a model and its generalization power using an independent data set in practice. If the model’s performance on different folds is consistent, then we could say that the model is robust and performing well. In the following, we test the RF model’s robustness using Scikit-sklearn cross-validation method:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  ................................................................\n",
      "[CV]  ................................................................\n",
      "[CV]  ................................................................\n",
      "[CV]  ................................................................\n",
      "[CV]  , accuracy=0.7333258936874605, recall=0.7784095131921219, roc_auc=0.8090472984618574, f1=0.7450206288234457, total= 8.8min\n",
      "[CV]  ................................................................\n",
      "[CV]  , accuracy=0.7297920618978536, recall=0.767372723894463, roc_auc=0.8047750367596309, f1=0.7397721573404027, total= 8.8min\n",
      "[CV]  ................................................................\n",
      "[CV]  , accuracy=0.731019603466875, recall=0.7719435154217763, roc_auc=0.8065173873635427, f1=0.7417868875874875, total= 8.8min\n",
      "[CV]  ................................................................\n",
      "[CV]  , accuracy=0.7312427928430607, recall=0.7732441471571906, roc_auc=0.803348966208371, f1=0.7422680412371134, total= 8.8min\n",
      "[CV]  ................................................................\n",
      "[CV]  , accuracy=0.7337164750957854, recall=0.7742103307320699, roc_auc=0.8083778666451472, f1=0.7442707868178975, total= 8.4min\n",
      "[CV]  ................................................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   5 out of  10 | elapsed: 17.2min remaining: 17.2min\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  , accuracy=0.729991630242723, recall=0.7712289568545839, roc_auc=0.803770231154411, f1=0.740874283776306, total= 8.4min\n",
      "[CV]  ................................................................\n",
      "[CV]  , accuracy=0.7293778480424068, recall=0.770374224237244, roc_auc=0.8041309804369061, f1=0.740224959828602, total= 8.4min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   7 out of  10 | elapsed: 17.2min remaining:  7.4min\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  , accuracy=0.7343625034873988, recall=0.7772864097513843, roc_auc=0.8082918628438596, f1=0.7454824108065723, total= 8.4min\n",
      "[CV]  , accuracy=0.7320747698316749, recall=0.7717492288825301, roc_auc=0.8079808736099967, f1=0.7424873522944636, total= 5.1min\n",
      "[CV]  , accuracy=0.7320139870545347, recall=0.7727154483630012, roc_auc=0.8082303043905488, f1=0.7426867164339036, total= 5.1min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  10 out of  10 | elapsed: 22.3min finished\n"
     ]
    }
   ],
   "source": [
    "scoring = ['accuracy', 'recall', 'roc_auc', 'f1']\n",
    "scores = cross_validate(rf, X = xtrain_scaled, y = ytrain, scoring=scoring,\n",
    "                         cv = 10, return_train_score = False, verbose = 10, n_jobs= -1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In the above code, I used four different evaluation metrics to judge the model's generalization. Let's see the result:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'fit_time': array([519.01635194, 519.08185387, 518.91476393, 514.854949  ,\n",
       "        491.86662292, 491.60002613, 492.63204885, 491.53150296,\n",
       "        296.78954768, 297.04013801]),\n",
       " 'score_time': array([10.45862293, 10.56293011, 10.65213013, 10.40467215, 10.21855617,\n",
       "        10.32528472, 10.14684105, 10.33355498,  6.49080539,  6.19442701]),\n",
       " 'test_accuracy': array([0.72979206, 0.7310196 , 0.73124279, 0.73332589, 0.73371648,\n",
       "        0.72999163, 0.7343625 , 0.72937785, 0.73207477, 0.73201399]),\n",
       " 'test_recall': array([0.76737272, 0.77194352, 0.77324415, 0.77840951, 0.77421033,\n",
       "        0.77122896, 0.77728641, 0.77037422, 0.77174923, 0.77271545]),\n",
       " 'test_roc_auc': array([0.80477504, 0.80651739, 0.80334897, 0.8090473 , 0.80837787,\n",
       "        0.80377023, 0.80829186, 0.80413098, 0.80798087, 0.8082303 ]),\n",
       " 'test_f1': array([0.73977216, 0.74178689, 0.74226804, 0.74502063, 0.74427079,\n",
       "        0.74087428, 0.74548241, 0.74022496, 0.74248735, 0.74268672])}"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The above code snippet, print out the results from the cross-validation. If you see different evaluation metrics carefully, you would see the model indeed is a robust and performs consistently across the folds. Let's do some more specific observations of the metric scores: mean and variance:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F1 score# (1) mean: 0.7424874224946193 (2)variance: 3.4239691671447294e-06\n",
      "Recall score# (1) mean: 0.7728534498486367 (2)variance: 9.340496661280428e-06\n",
      "Accuracy score# (1) mean: 0.7316917565649772 (2)variance: 2.6660110240196636e-06\n"
     ]
    }
   ],
   "source": [
    "print('F1 score# (1) mean: {} (2)variance: {}'.format(np.mean(scores['test_f1']), np.var(scores['test_f1'])))\n",
    "print('Recall score# (1) mean: {} (2)variance: {}'.format(np.mean(scores['test_recall']), np.var(scores['test_recall'])))\n",
    "print('Accuracy score# (1) mean: {} (2)variance: {}'.format(np.mean(scores['test_accuracy']), np.var(scores['test_accuracy'])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It's good to see that every evaluation metric has a very low variance which again confirms the model robustness. Although the model is robust, I am not happy yet. We need to improve the model generalization on testing data. Next, I would try a gradient boosted tree-based algorithm.\n",
    "\n",
    "There are many gradients boosted tree-based algorithms available. For example XGBoost, LightGBM, CataBoost etc. I myself find LightGBM is faster and performs well on categorical data more than other algorithms I mentioned. So, let's get started with LightGBM."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. LightGBM model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {},
   "outputs": [],
   "source": [
    "import lightgbm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "metadata": {},
   "outputs": [],
   "source": [
    "lbg_params = {\n",
    "    'n_estimators': 8000,\n",
    "    'max_depth': 100,\n",
    "    'objective': 'binary',\n",
    "    'learning_rate' : 0.02,\n",
    "    'num_leaves' : 250,\n",
    "    'feature_fraction': 0.64, \n",
    "    'bagging_fraction': 0.8, \n",
    "    'bagging_freq': 1,\n",
    "    'boosting_type' : 'gbdt'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "metadata": {},
   "outputs": [],
   "source": [
    "lgb = lightgbm.LGBMClassifier(**lbg_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 230,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LGBMClassifier(bagging_fraction=0.8, bagging_freq=1, boosting_type='gbdt',\n",
       "        class_weight=None, colsample_bytree=1.0, feature_fraction=0.64,\n",
       "        learning_rate=0.02, max_depth=100, min_child_samples=20,\n",
       "        min_child_weight=0.001, min_split_gain=0.0, n_estimators=8000,\n",
       "        n_jobs=-1, num_leaves=250, objective='binary', random_state=None,\n",
       "        reg_alpha=0.0, reg_lambda=0.0, silent=True, subsample=1.0,\n",
       "        subsample_for_bin=200000, subsample_freq=0)"
      ]
     },
     "execution_count": 230,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lgb.fit(xtrain_scaled, ytrain)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In the above code, I first find the optimal hyperparameters manually similar way I did in the random forest model. I find the most important parameters are ‘n_estimators’ and ‘max_depth’ in the model. Let's see the model’s prediction and performance on the test data:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Test the model with test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/sabber/anaconda3/lib/python3.6/site-packages/sklearn/preprocessing/label.py:151: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if diff:\n"
     ]
    }
   ],
   "source": [
    "lgb_pred = lgb.predict(xtest_scaled)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "metadata": {},
   "outputs": [],
   "source": [
    "lgb_pred_proba = lgb.predict_proba(xtest_scaled)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 233,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ROC-AUC score of the model: 0.9586191656898193\n",
      "Accuracy of the model: 0.890546708477943\n",
      "\n",
      "Classification report: \n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.93      0.85      0.89     89877\n",
      "          1       0.86      0.94      0.90     89341\n",
      "\n",
      "avg / total       0.89      0.89      0.89    179218\n",
      "\n",
      "\n",
      "Confusion matrix: \n",
      "[[75963 13914]\n",
      " [ 5702 83639]]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "evaluate_model(ytest, lgb_pred, lgb_pred_proba)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The performance report seems promising. Accuracy jumped 35% from logistic regression and 23% from the random forest model. I stop here optimizing other hyperparameters. It took me about 4 hours to find the above two hyperparameters. Now I focus on the model's robustness using same technique cross-validation."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Cross validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "metadata": {},
   "outputs": [],
   "source": [
    "folds = list(KFold(5, shuffle=True, random_state=2016)\\\n",
    "             .split(xtrain_scaled, ytrain))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/sabber/anaconda3/lib/python3.6/site-packages/sklearn/preprocessing/label.py:151: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if diff:\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "cv: 0\n",
      "\n",
      "ROC-AUC score of the model: 0.9483670270426987\n",
      "Accuracy of the model: 0.8754963684890869\n",
      "\n",
      "Classification report: \n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.91      0.83      0.87     53472\n",
      "          1       0.84      0.92      0.88     54059\n",
      "\n",
      "avg / total       0.88      0.88      0.88    107531\n",
      "\n",
      "\n",
      "Confusion matrix: \n",
      "[[44284  9188]\n",
      " [ 4200 49859]]\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/sabber/anaconda3/lib/python3.6/site-packages/sklearn/preprocessing/label.py:151: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if diff:\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "cv: 1\n",
      "\n",
      "ROC-AUC score of the model: 0.9478880521895745\n",
      "Accuracy of the model: 0.8756730617217361\n",
      "\n",
      "Classification report: \n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.92      0.83      0.87     53861\n",
      "          1       0.84      0.92      0.88     53670\n",
      "\n",
      "avg / total       0.88      0.88      0.88    107531\n",
      "\n",
      "\n",
      "Confusion matrix: \n",
      "[[44633  9228]\n",
      " [ 4141 49529]]\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/sabber/anaconda3/lib/python3.6/site-packages/sklearn/preprocessing/label.py:151: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if diff:\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "cv: 2\n",
      "\n",
      "ROC-AUC score of the model: 0.9490374658331387\n",
      "Accuracy of the model: 0.8780909691158829\n",
      "\n",
      "Classification report: \n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.92      0.83      0.87     53504\n",
      "          1       0.85      0.92      0.88     54027\n",
      "\n",
      "avg / total       0.88      0.88      0.88    107531\n",
      "\n",
      "\n",
      "Confusion matrix: \n",
      "[[44457  9047]\n",
      " [ 4062 49965]]\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/sabber/anaconda3/lib/python3.6/site-packages/sklearn/preprocessing/label.py:151: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if diff:\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "cv: 3\n",
      "\n",
      "ROC-AUC score of the model: 0.9490402272662238\n",
      "Accuracy of the model: 0.8781188680473538\n",
      "\n",
      "Classification report: \n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.92      0.83      0.87     53958\n",
      "          1       0.85      0.92      0.88     53573\n",
      "\n",
      "avg / total       0.88      0.88      0.88    107531\n",
      "\n",
      "\n",
      "Confusion matrix: \n",
      "[[44916  9042]\n",
      " [ 4064 49509]]\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/sabber/anaconda3/lib/python3.6/site-packages/sklearn/preprocessing/label.py:151: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if diff:\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "cv: 4\n",
      "\n",
      "ROC-AUC score of the model: 0.9486846347287888\n",
      "Accuracy of the model: 0.8762112898725937\n",
      "\n",
      "Classification report: \n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.91      0.83      0.87     53764\n",
      "          1       0.84      0.92      0.88     53766\n",
      "\n",
      "avg / total       0.88      0.88      0.88    107530\n",
      "\n",
      "\n",
      "Confusion matrix: \n",
      "[[44671  9093]\n",
      " [ 4218 49548]]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for i, (train_idx, valid_idx) in enumerate(folds):\n",
    "    \n",
    "    ytrain = np.array(ytrain)\n",
    "    X_train = xtrain_scaled[train_idx]\n",
    "    y_train = ytrain[train_idx]\n",
    "    X_valid = xtrain_scaled[valid_idx]\n",
    "    y_valid = ytrain[valid_idx]\n",
    "    \n",
    "    lgb.fit(X_train, y_train)\n",
    "    pred = lgb.predict(X_valid)\n",
    "    pred_proba = lgb.predict_proba(X_valid)\n",
    "    \n",
    "    print('\\ncv: {}\\n'.format(i))\n",
    "    evaluate_model(y_valid, pred, pred_proba)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If you look at the print out of the above code, you would see LightGBM is also robust and performs consistently across different training folds. In this project, I did not discuss anything about overfitting. Hope I will write another blog on the topic."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}