HousingLoans_TrainData-Preparation-Cleaning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "### #A1d1_HousingLoans_TestData-Preparation-Cleaning to make a classification model on Challenge 2\n",
    "\n",
    "## Goal: Create, clean, & prepare data for accepted in USA dataset (Supervised ML)\n",
    "\n",
    "# Import Python pkgs pandas, numpy, matplotlib.pyplot, & seaborn\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import numpy.random as nr\n",
    "import math\n",
    "\n",
    "%matplotlib inline  \n",
    "# Start of magic command which configures execution environment, to display graphics w/in notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(500000, 22)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>row_id</th>\n",
       "      <th>loan_type</th>\n",
       "      <th>property_type</th>\n",
       "      <th>loan_purpose</th>\n",
       "      <th>occupancy</th>\n",
       "      <th>loan_amount</th>\n",
       "      <th>preapproval</th>\n",
       "      <th>msa_md</th>\n",
       "      <th>state_code</th>\n",
       "      <th>county_code</th>\n",
       "      <th>...</th>\n",
       "      <th>applicant_sex</th>\n",
       "      <th>applicant_income</th>\n",
       "      <th>population</th>\n",
       "      <th>minority_population_pct</th>\n",
       "      <th>ffiecmedian_family_income</th>\n",
       "      <th>tract_to_msa_md_income_pct</th>\n",
       "      <th>number_of_owner-occupied_units</th>\n",
       "      <th>number_of_1_to_4_family_units</th>\n",
       "      <th>lender</th>\n",
       "      <th>co_applicant</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>70.0</td>\n",
       "      <td>3</td>\n",
       "      <td>18</td>\n",
       "      <td>37</td>\n",
       "      <td>246</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>24.0</td>\n",
       "      <td>6203.0</td>\n",
       "      <td>44.230</td>\n",
       "      <td>60588.0</td>\n",
       "      <td>50.933</td>\n",
       "      <td>716.0</td>\n",
       "      <td>2642.0</td>\n",
       "      <td>4536</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>178.0</td>\n",
       "      <td>3</td>\n",
       "      <td>369</td>\n",
       "      <td>52</td>\n",
       "      <td>299</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>57.0</td>\n",
       "      <td>5774.0</td>\n",
       "      <td>15.905</td>\n",
       "      <td>54821.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>1622.0</td>\n",
       "      <td>2108.0</td>\n",
       "      <td>2458</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>163.0</td>\n",
       "      <td>3</td>\n",
       "      <td>16</td>\n",
       "      <td>10</td>\n",
       "      <td>306</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>67.0</td>\n",
       "      <td>6094.0</td>\n",
       "      <td>61.270</td>\n",
       "      <td>67719.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>760.0</td>\n",
       "      <td>1048.0</td>\n",
       "      <td>5710</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>155.0</td>\n",
       "      <td>1</td>\n",
       "      <td>305</td>\n",
       "      <td>47</td>\n",
       "      <td>180</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>105.0</td>\n",
       "      <td>6667.0</td>\n",
       "      <td>6.246</td>\n",
       "      <td>78439.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>2025.0</td>\n",
       "      <td>2299.0</td>\n",
       "      <td>5888</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>305.0</td>\n",
       "      <td>3</td>\n",
       "      <td>24</td>\n",
       "      <td>37</td>\n",
       "      <td>20</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>71.0</td>\n",
       "      <td>6732.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>63075.0</td>\n",
       "      <td>82.200</td>\n",
       "      <td>1464.0</td>\n",
       "      <td>1847.0</td>\n",
       "      <td>289</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>133.0</td>\n",
       "      <td>3</td>\n",
       "      <td>221</td>\n",
       "      <td>13</td>\n",
       "      <td>55</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>51.0</td>\n",
       "      <td>6078.0</td>\n",
       "      <td>4.821</td>\n",
       "      <td>82745.0</td>\n",
       "      <td>96.550</td>\n",
       "      <td>1827.0</td>\n",
       "      <td>2340.0</td>\n",
       "      <td>964</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>240.0</td>\n",
       "      <td>3</td>\n",
       "      <td>374</td>\n",
       "      <td>28</td>\n",
       "      <td>131</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>104.0</td>\n",
       "      <td>6068.0</td>\n",
       "      <td>26.427</td>\n",
       "      <td>65282.0</td>\n",
       "      <td>81.068</td>\n",
       "      <td>1863.0</td>\n",
       "      <td>2560.0</td>\n",
       "      <td>5488</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>210.0</td>\n",
       "      <td>3</td>\n",
       "      <td>322</td>\n",
       "      <td>37</td>\n",
       "      <td>35</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>55.0</td>\n",
       "      <td>6030.0</td>\n",
       "      <td>78.153</td>\n",
       "      <td>108353.0</td>\n",
       "      <td>59.439</td>\n",
       "      <td>969.0</td>\n",
       "      <td>1601.0</td>\n",
       "      <td>2442</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>209.0</td>\n",
       "      <td>3</td>\n",
       "      <td>24</td>\n",
       "      <td>37</td>\n",
       "      <td>20</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>244.0</td>\n",
       "      <td>5151.0</td>\n",
       "      <td>88.156</td>\n",
       "      <td>63414.0</td>\n",
       "      <td>62.813</td>\n",
       "      <td>411.0</td>\n",
       "      <td>481.0</td>\n",
       "      <td>2118</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>197.0</td>\n",
       "      <td>3</td>\n",
       "      <td>194</td>\n",
       "      <td>9</td>\n",
       "      <td>20</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>86.0</td>\n",
       "      <td>7916.0</td>\n",
       "      <td>24.893</td>\n",
       "      <td>65927.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>1861.0</td>\n",
       "      <td>2123.0</td>\n",
       "      <td>3507</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   row_id  loan_type  property_type  loan_purpose  occupancy  loan_amount  \\\n",
       "0       0          3              1             1          1         70.0   \n",
       "1       1          1              1             3          1        178.0   \n",
       "2       2          2              1             3          1        163.0   \n",
       "3       3          1              1             1          1        155.0   \n",
       "4       4          1              1             1          1        305.0   \n",
       "5       5          1              1             3          1        133.0   \n",
       "6       6          3              1             1          1        240.0   \n",
       "7       7          2              1             1          1        210.0   \n",
       "8       8          1              1             3          2        209.0   \n",
       "9       9          1              1             3          1        197.0   \n",
       "\n",
       "   preapproval  msa_md  state_code  county_code  ...  applicant_sex  \\\n",
       "0            3      18          37          246  ...              1   \n",
       "1            3     369          52          299  ...              1   \n",
       "2            3      16          10          306  ...              1   \n",
       "3            1     305          47          180  ...              1   \n",
       "4            3      24          37           20  ...              2   \n",
       "5            3     221          13           55  ...              2   \n",
       "6            3     374          28          131  ...              2   \n",
       "7            3     322          37           35  ...              1   \n",
       "8            3      24          37           20  ...              1   \n",
       "9            3     194           9           20  ...              1   \n",
       "\n",
       "   applicant_income  population  minority_population_pct  \\\n",
       "0              24.0      6203.0                   44.230   \n",
       "1              57.0      5774.0                   15.905   \n",
       "2              67.0      6094.0                   61.270   \n",
       "3             105.0      6667.0                    6.246   \n",
       "4              71.0      6732.0                  100.000   \n",
       "5              51.0      6078.0                    4.821   \n",
       "6             104.0      6068.0                   26.427   \n",
       "7              55.0      6030.0                   78.153   \n",
       "8             244.0      5151.0                   88.156   \n",
       "9              86.0      7916.0                   24.893   \n",
       "\n",
       "   ffiecmedian_family_income  tract_to_msa_md_income_pct  \\\n",
       "0                    60588.0                      50.933   \n",
       "1                    54821.0                     100.000   \n",
       "2                    67719.0                     100.000   \n",
       "3                    78439.0                     100.000   \n",
       "4                    63075.0                      82.200   \n",
       "5                    82745.0                      96.550   \n",
       "6                    65282.0                      81.068   \n",
       "7                   108353.0                      59.439   \n",
       "8                    63414.0                      62.813   \n",
       "9                    65927.0                     100.000   \n",
       "\n",
       "   number_of_owner-occupied_units  number_of_1_to_4_family_units  lender  \\\n",
       "0                           716.0                         2642.0    4536   \n",
       "1                          1622.0                         2108.0    2458   \n",
       "2                           760.0                         1048.0    5710   \n",
       "3                          2025.0                         2299.0    5888   \n",
       "4                          1464.0                         1847.0     289   \n",
       "5                          1827.0                         2340.0     964   \n",
       "6                          1863.0                         2560.0    5488   \n",
       "7                           969.0                         1601.0    2442   \n",
       "8                           411.0                          481.0    2118   \n",
       "9                          1861.0                         2123.0    3507   \n",
       "\n",
       "   co_applicant  \n",
       "0         False  \n",
       "1         False  \n",
       "2         False  \n",
       "3          True  \n",
       "4         False  \n",
       "5         False  \n",
       "6         False  \n",
       "7          True  \n",
       "8          True  \n",
       "9         False  \n",
       "\n",
       "[10 rows x 22 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load training feature dataset, display shape, & explore first 10 rows of Pandas data frame\n",
    "\n",
    "LoanTrainV = pd.read_csv('April_DAT102x_TrainingValues_2019-04-01.csv', header=0)\n",
    "print(LoanTrainV.shape)\n",
    "LoanTrainV.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "row_id                              int64\n",
       "loan_type                           int64\n",
       "property_type                       int64\n",
       "loan_purpose                        int64\n",
       "occupancy                           int64\n",
       "loan_amount                       float64\n",
       "preapproval                         int64\n",
       "msa_md                              int64\n",
       "state_code                          int64\n",
       "county_code                         int64\n",
       "applicant_ethnicity                 int64\n",
       "applicant_race                      int64\n",
       "applicant_sex                       int64\n",
       "applicant_income                  float64\n",
       "population                        float64\n",
       "minority_population_pct           float64\n",
       "ffiecmedian_family_income         float64\n",
       "tract_to_msa_md_income_pct        float64\n",
       "number_of_owner-occupied_units    float64\n",
       "number_of_1_to_4_family_units     float64\n",
       "lender                              int64\n",
       "co_applicant                         bool\n",
       "dtype: object"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Above- We have a total of 22 columns and 500,000 rows in the training feature dataset.\n",
    "\n",
    "#Below- Review data types for each column\n",
    "LoanTrainV.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(500000, 2)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>row_id</th>\n",
       "      <th>accepted</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   row_id  accepted\n",
       "0       0         1\n",
       "1       1         0\n",
       "2       2         1\n",
       "3       3         1\n",
       "4       4         1\n",
       "5       5         1\n",
       "6       6         1\n",
       "7       7         1\n",
       "8       8         1\n",
       "9       9         0"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load label dataset, display shape, & explore first 10 rows of Pandas data frame\n",
    "\n",
    "LoanTrainL = pd.read_csv('April_DAT102x_TrainingLabels_2019-04-01.csv', header=0)\n",
    "print(LoanTrainL.shape)\n",
    "LoanTrainL.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "row_id      int64\n",
       "accepted    int64\n",
       "dtype: object"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Above- We have a total of 2 columns and 500,000 rows in the Label dataset.\n",
    "\n",
    "\n",
    "#Below- Review data types for each column\n",
    "\n",
    "LoanTrainL.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "row_id                            False\n",
       "loan_type                         False\n",
       "property_type                     False\n",
       "loan_purpose                      False\n",
       "occupancy                         False\n",
       "loan_amount                       False\n",
       "preapproval                       False\n",
       "msa_md                            False\n",
       "state_code                        False\n",
       "county_code                       False\n",
       "applicant_ethnicity               False\n",
       "applicant_race                    False\n",
       "applicant_sex                     False\n",
       "applicant_income                   True\n",
       "population                         True\n",
       "minority_population_pct            True\n",
       "ffiecmedian_family_income          True\n",
       "tract_to_msa_md_income_pct         True\n",
       "number_of_owner-occupied_units     True\n",
       "number_of_1_to_4_family_units      True\n",
       "lender                            False\n",
       "co_applicant                      False\n",
       "dtype: bool"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Find missing values w/in dataset\n",
    " # Can be coded as character string, numeric value like -999 or missing value NaN, or a NULL value.\n",
    "# Then determine how to treat missing values.\n",
    " # Remove rows w/ missing values\n",
    " # Remove features w/ missing values\n",
    " # Impute by replacing values w/ mean or median using simple algorithms, or more complex ones like SMOTE or expectation maximization (EM).\n",
    " # Use nearest neighbor, averaging, forward filling or backward filling.\n",
    "\n",
    "# Shows which features are missing values\n",
    "\n",
    "#(LoanTrainV.astype(np.object) == '?').any()  # This will only work w/ strings\n",
    "pd.isna(LoanTrainV).any()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "co_applicant 0\n"
     ]
    }
   ],
   "source": [
    "#Above- Training feature dataset does seem to contain missing values in 7 columns.\n",
    "\n",
    "#Below- Find out how many values are missing w/in each object/categorical feature.\n",
    "\n",
    "for col in LoanTrainV.columns:\n",
    "    if LoanTrainV[col].dtype == bool:\n",
    "        count = 0\n",
    "        count = [count + 1 for x in LoanTrainV[col] if pd.isna(x)]\n",
    "        print(col + ' ' + str(sum(count)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loan_amount 0\n",
      "applicant_income 39948\n",
      "population 22465\n",
      "minority_population_pct 22466\n",
      "ffiecmedian_family_income 22440\n",
      "tract_to_msa_md_income_pct 22514\n",
      "number_of_owner-occupied_units 22565\n",
      "number_of_1_to_4_family_units 22530\n"
     ]
    }
   ],
   "source": [
    "#Above- Training feature dataset does not seem to contain missing values in boolean columns.\n",
    "\n",
    "#Below- Find out how many values are missing w/in each float64 feature.\n",
    "\n",
    "for col in LoanTrainV.columns:\n",
    "    if LoanTrainV[col].dtype == float :\n",
    "        count = 0\n",
    "        count = [count + 1 for x in LoanTrainV[col] if pd.isna(x)]\n",
    "        print(col + ' ' + str(sum(count)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "row_id 0\n",
      "loan_type 0\n",
      "property_type 0\n",
      "loan_purpose 0\n",
      "occupancy 0\n",
      "preapproval 0\n",
      "msa_md 0\n",
      "state_code 0\n",
      "county_code 0\n",
      "applicant_ethnicity 0\n",
      "applicant_race 0\n",
      "applicant_sex 0\n",
      "lender 0\n"
     ]
    }
   ],
   "source": [
    "#Above- Training feature dataset does seem to contain missing values in 7 float columns.\n",
    "\n",
    "\n",
    "#Below- Find out how many values are missing w/in each int64 feature.\n",
    "\n",
    "for col in LoanTrainV.columns:\n",
    "    if LoanTrainV[col].dtype == int :\n",
    "        count = 0\n",
    "        count = [count + 1 for x in LoanTrainV[col] if pd.isna(x)]\n",
    "        print(col + ' ' + str(sum(count)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n",
      "76982\n"
     ]
    }
   ],
   "source": [
    "#Above- Training feature dataset does not seem to contain missing values in integer columns.\n",
    "\n",
    "#Below- Find out how many values are missing as '-1' w/in this integer column feature.\n",
    "\n",
    "print(any(LoanTrainV.msa_md == -1))\n",
    "print(LoanTrainV.msa_md[LoanTrainV.msa_md == -1].count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n",
      "19132\n"
     ]
    }
   ],
   "source": [
    "#Above- Training feature column msa_md does seem to contain 76,982 missing values.\n",
    "\n",
    "#Below- Find out how many values are missing as '-1' w/in this integer column feature.\n",
    "\n",
    "print(any(LoanTrainV.state_code == -1))\n",
    "print(LoanTrainV.state_code[LoanTrainV.state_code == -1].count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n",
      "20466\n"
     ]
    }
   ],
   "source": [
    "#Above- Training feature column state_code does seem to contain 19,132 missing values.\n",
    "\n",
    "#Below- Find out how many values are missing as '-1' w/in this integer column feature.\n",
    "\n",
    "print(any(LoanTrainV.county_code == -1))\n",
    "print(LoanTrainV.county_code[LoanTrainV.county_code == -1].count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "row_id      False\n",
       "accepted    False\n",
       "dtype: bool"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Above- Training feature column county_code does seem to contain 20,466 missing values.\n",
    "\n",
    "# Find missing values w/in label dataset\n",
    " # Can be coded as character string, numeric value like -999 or missing value NaN, or a NULL value.\n",
    "# Then determine how to treat missing values.\n",
    " # Remove rows w/ missing values\n",
    " # Remove features w/ missing values\n",
    " # Impute by replacing values w/ mean or median using simple algorithms, or more complex ones like SMOTE or expectation maximization (EM).\n",
    " # Use nearest neighbor, averaging, forward filling or backward filling.\n",
    "\n",
    "# Shows which features are missing values\n",
    "\n",
    "#(LoanTrainV.astype(np.object) == '?').any()  # This will only work w/ strings\n",
    "pd.isna(LoanTrainL).any()\n",
    "#pd.isnull(LoanTrainV)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(500000, 22)\n",
      "(500000,)\n"
     ]
    }
   ],
   "source": [
    "#Above- Label dataset does seem to contain missing values in float columns.\n",
    "\n",
    "#Below- Examine feature training data row_id for duplicates by checking for unique IDs.\n",
    "\n",
    "print(LoanTrainV.shape)\n",
    "print(LoanTrainV.row_id.unique().shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>row_id</th>\n",
       "      <th>loan_type</th>\n",
       "      <th>property_type</th>\n",
       "      <th>loan_purpose</th>\n",
       "      <th>occupancy</th>\n",
       "      <th>loan_amount</th>\n",
       "      <th>preapproval</th>\n",
       "      <th>msa_md</th>\n",
       "      <th>state_code</th>\n",
       "      <th>county_code</th>\n",
       "      <th>...</th>\n",
       "      <th>applicant_race</th>\n",
       "      <th>applicant_sex</th>\n",
       "      <th>applicant_income</th>\n",
       "      <th>population</th>\n",
       "      <th>minority_population_pct</th>\n",
       "      <th>ffiecmedian_family_income</th>\n",
       "      <th>tract_to_msa_md_income_pct</th>\n",
       "      <th>number_of_owner-occupied_units</th>\n",
       "      <th>number_of_1_to_4_family_units</th>\n",
       "      <th>lender</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>500000.000000</td>\n",
       "      <td>500000.000000</td>\n",
       "      <td>500000.000000</td>\n",
       "      <td>500000.000000</td>\n",
       "      <td>500000.000000</td>\n",
       "      <td>500000.000000</td>\n",
       "      <td>500000.000000</td>\n",
       "      <td>500000.000000</td>\n",
       "      <td>500000.000000</td>\n",
       "      <td>500000.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>500000.000000</td>\n",
       "      <td>500000.000000</td>\n",
       "      <td>460052.000000</td>\n",
       "      <td>477535.000000</td>\n",
       "      <td>477534.000000</td>\n",
       "      <td>477560.000000</td>\n",
       "      <td>477486.000000</td>\n",
       "      <td>477435.000000</td>\n",
       "      <td>477470.000000</td>\n",
       "      <td>500000.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>249999.500000</td>\n",
       "      <td>1.366276</td>\n",
       "      <td>1.047650</td>\n",
       "      <td>2.066810</td>\n",
       "      <td>1.109590</td>\n",
       "      <td>221.753158</td>\n",
       "      <td>2.764722</td>\n",
       "      <td>181.606972</td>\n",
       "      <td>23.726924</td>\n",
       "      <td>144.542062</td>\n",
       "      <td>...</td>\n",
       "      <td>4.786586</td>\n",
       "      <td>1.462374</td>\n",
       "      <td>102.389521</td>\n",
       "      <td>5416.833956</td>\n",
       "      <td>31.617310</td>\n",
       "      <td>69235.603298</td>\n",
       "      <td>91.832624</td>\n",
       "      <td>1427.718282</td>\n",
       "      <td>1886.147065</td>\n",
       "      <td>3720.121344</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>144337.711635</td>\n",
       "      <td>0.690555</td>\n",
       "      <td>0.231404</td>\n",
       "      <td>0.948371</td>\n",
       "      <td>0.326092</td>\n",
       "      <td>590.641648</td>\n",
       "      <td>0.543061</td>\n",
       "      <td>138.464169</td>\n",
       "      <td>15.982768</td>\n",
       "      <td>100.243612</td>\n",
       "      <td>...</td>\n",
       "      <td>1.024927</td>\n",
       "      <td>0.677685</td>\n",
       "      <td>153.534496</td>\n",
       "      <td>2728.144999</td>\n",
       "      <td>26.333938</td>\n",
       "      <td>14810.058791</td>\n",
       "      <td>14.210924</td>\n",
       "      <td>737.559511</td>\n",
       "      <td>914.123744</td>\n",
       "      <td>1838.313175</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>14.000000</td>\n",
       "      <td>0.534000</td>\n",
       "      <td>17858.000000</td>\n",
       "      <td>3.981000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>124999.750000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>93.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>25.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>57.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>47.000000</td>\n",
       "      <td>3744.000000</td>\n",
       "      <td>10.700000</td>\n",
       "      <td>59731.000000</td>\n",
       "      <td>88.067250</td>\n",
       "      <td>944.000000</td>\n",
       "      <td>1301.000000</td>\n",
       "      <td>2442.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>249999.500000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>162.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>192.000000</td>\n",
       "      <td>26.000000</td>\n",
       "      <td>131.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>74.000000</td>\n",
       "      <td>4975.000000</td>\n",
       "      <td>22.901000</td>\n",
       "      <td>67526.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>1327.000000</td>\n",
       "      <td>1753.000000</td>\n",
       "      <td>3731.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>374999.250000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>266.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>314.000000</td>\n",
       "      <td>37.000000</td>\n",
       "      <td>246.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>117.000000</td>\n",
       "      <td>6467.000000</td>\n",
       "      <td>46.020000</td>\n",
       "      <td>75351.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>1780.000000</td>\n",
       "      <td>2309.000000</td>\n",
       "      <td>5436.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>499999.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>100878.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>408.000000</td>\n",
       "      <td>52.000000</td>\n",
       "      <td>324.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>10139.000000</td>\n",
       "      <td>37097.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>125248.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>8771.000000</td>\n",
       "      <td>13623.000000</td>\n",
       "      <td>6508.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              row_id      loan_type  property_type   loan_purpose  \\\n",
       "count  500000.000000  500000.000000  500000.000000  500000.000000   \n",
       "mean   249999.500000       1.366276       1.047650       2.066810   \n",
       "std    144337.711635       0.690555       0.231404       0.948371   \n",
       "min         0.000000       1.000000       1.000000       1.000000   \n",
       "25%    124999.750000       1.000000       1.000000       1.000000   \n",
       "50%    249999.500000       1.000000       1.000000       2.000000   \n",
       "75%    374999.250000       2.000000       1.000000       3.000000   \n",
       "max    499999.000000       4.000000       3.000000       3.000000   \n",
       "\n",
       "           occupancy    loan_amount    preapproval         msa_md  \\\n",
       "count  500000.000000  500000.000000  500000.000000  500000.000000   \n",
       "mean        1.109590     221.753158       2.764722     181.606972   \n",
       "std         0.326092     590.641648       0.543061     138.464169   \n",
       "min         1.000000       1.000000       1.000000      -1.000000   \n",
       "25%         1.000000      93.000000       3.000000      25.000000   \n",
       "50%         1.000000     162.000000       3.000000     192.000000   \n",
       "75%         1.000000     266.000000       3.000000     314.000000   \n",
       "max         3.000000  100878.000000       3.000000     408.000000   \n",
       "\n",
       "          state_code    county_code  ...  applicant_race  applicant_sex  \\\n",
       "count  500000.000000  500000.000000  ...   500000.000000  500000.000000   \n",
       "mean       23.726924     144.542062  ...        4.786586       1.462374   \n",
       "std        15.982768     100.243612  ...        1.024927       0.677685   \n",
       "min        -1.000000      -1.000000  ...        1.000000       1.000000   \n",
       "25%         6.000000      57.000000  ...        5.000000       1.000000   \n",
       "50%        26.000000     131.000000  ...        5.000000       1.000000   \n",
       "75%        37.000000     246.000000  ...        5.000000       2.000000   \n",
       "max        52.000000     324.000000  ...        7.000000       4.000000   \n",
       "\n",
       "       applicant_income     population  minority_population_pct  \\\n",
       "count     460052.000000  477535.000000            477534.000000   \n",
       "mean         102.389521    5416.833956                31.617310   \n",
       "std          153.534496    2728.144999                26.333938   \n",
       "min            1.000000      14.000000                 0.534000   \n",
       "25%           47.000000    3744.000000                10.700000   \n",
       "50%           74.000000    4975.000000                22.901000   \n",
       "75%          117.000000    6467.000000                46.020000   \n",
       "max        10139.000000   37097.000000               100.000000   \n",
       "\n",
       "       ffiecmedian_family_income  tract_to_msa_md_income_pct  \\\n",
       "count              477560.000000               477486.000000   \n",
       "mean                69235.603298                   91.832624   \n",
       "std                 14810.058791                   14.210924   \n",
       "min                 17858.000000                    3.981000   \n",
       "25%                 59731.000000                   88.067250   \n",
       "50%                 67526.000000                  100.000000   \n",
       "75%                 75351.000000                  100.000000   \n",
       "max                125248.000000                  100.000000   \n",
       "\n",
       "       number_of_owner-occupied_units  number_of_1_to_4_family_units  \\\n",
       "count                   477435.000000                  477470.000000   \n",
       "mean                      1427.718282                    1886.147065   \n",
       "std                        737.559511                     914.123744   \n",
       "min                          4.000000                       1.000000   \n",
       "25%                        944.000000                    1301.000000   \n",
       "50%                       1327.000000                    1753.000000   \n",
       "75%                       1780.000000                    2309.000000   \n",
       "max                       8771.000000                   13623.000000   \n",
       "\n",
       "              lender  \n",
       "count  500000.000000  \n",
       "mean     3720.121344  \n",
       "std      1838.313175  \n",
       "min         0.000000  \n",
       "25%      2442.000000  \n",
       "50%      3731.000000  \n",
       "75%      5436.000000  \n",
       "max      6508.000000  \n",
       "\n",
       "[8 rows x 21 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Above- Training feature dataset does not seem to contain duplicate values.\n",
    "\n",
    "# Compute & display summary statistics for numeric columns of training data\n",
    "\n",
    "LoanTrainV.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(500000, 2)\n",
      "(500000,)\n"
     ]
    }
   ],
   "source": [
    "#Below- Examine label data for duplicates by checking for unique IDs (row_id).\n",
    "\n",
    "print(LoanTrainL.shape)\n",
    "print(LoanTrainL.row_id.unique().shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>row_id</th>\n",
       "      <th>accepted</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>500000.000000</td>\n",
       "      <td>500000.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>249999.500000</td>\n",
       "      <td>0.500228</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>144337.711635</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>124999.750000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>249999.500000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>374999.250000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>499999.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              row_id       accepted\n",
       "count  500000.000000  500000.000000\n",
       "mean   249999.500000       0.500228\n",
       "std    144337.711635       0.500000\n",
       "min         0.000000       0.000000\n",
       "25%    124999.750000       0.000000\n",
       "50%    249999.500000       1.000000\n",
       "75%    374999.250000       1.000000\n",
       "max    499999.000000       1.000000"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Above- Label dataset does not seem to contain duplicate values.\n",
    "\n",
    "#Below- Compute & display summary statistics for numeric columns of label data\n",
    "\n",
    "LoanTrainL.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(500000, 22)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>row_id</th>\n",
       "      <th>loan_type</th>\n",
       "      <th>property_type</th>\n",
       "      <th>loan_purpose</th>\n",
       "      <th>occupancy</th>\n",
       "      <th>loan_amount</th>\n",
       "      <th>preapproval</th>\n",
       "      <th>msa_md</th>\n",
       "      <th>state_code</th>\n",
       "      <th>county_code</th>\n",
       "      <th>...</th>\n",
       "      <th>applicant_sex</th>\n",
       "      <th>applicant_income</th>\n",
       "      <th>population</th>\n",
       "      <th>minority_population_pct</th>\n",
       "      <th>ffiecmedian_family_income</th>\n",
       "      <th>tract_to_msa_md_income_pct</th>\n",
       "      <th>number_of_owner-occupied_units</th>\n",
       "      <th>number_of_1_to_4_family_units</th>\n",
       "      <th>lender</th>\n",
       "      <th>co_applicant</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>70.0</td>\n",
       "      <td>3</td>\n",
       "      <td>18</td>\n",
       "      <td>37</td>\n",
       "      <td>246</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>24.0</td>\n",
       "      <td>6203.0</td>\n",
       "      <td>44.230</td>\n",
       "      <td>60588.0</td>\n",
       "      <td>50.933</td>\n",
       "      <td>716.0</td>\n",
       "      <td>2642.0</td>\n",
       "      <td>4536</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>178.0</td>\n",
       "      <td>3</td>\n",
       "      <td>369</td>\n",
       "      <td>52</td>\n",
       "      <td>299</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>57.0</td>\n",
       "      <td>5774.0</td>\n",
       "      <td>15.905</td>\n",
       "      <td>54821.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>1622.0</td>\n",
       "      <td>2108.0</td>\n",
       "      <td>2458</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>163.0</td>\n",
       "      <td>3</td>\n",
       "      <td>16</td>\n",
       "      <td>10</td>\n",
       "      <td>306</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>67.0</td>\n",
       "      <td>6094.0</td>\n",
       "      <td>61.270</td>\n",
       "      <td>67719.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>760.0</td>\n",
       "      <td>1048.0</td>\n",
       "      <td>5710</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>155.0</td>\n",
       "      <td>1</td>\n",
       "      <td>305</td>\n",
       "      <td>47</td>\n",
       "      <td>180</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>105.0</td>\n",
       "      <td>6667.0</td>\n",
       "      <td>6.246</td>\n",
       "      <td>78439.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>2025.0</td>\n",
       "      <td>2299.0</td>\n",
       "      <td>5888</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>305.0</td>\n",
       "      <td>3</td>\n",
       "      <td>24</td>\n",
       "      <td>37</td>\n",
       "      <td>20</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>71.0</td>\n",
       "      <td>6732.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>63075.0</td>\n",
       "      <td>82.200</td>\n",
       "      <td>1464.0</td>\n",
       "      <td>1847.0</td>\n",
       "      <td>289</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>133.0</td>\n",
       "      <td>3</td>\n",
       "      <td>221</td>\n",
       "      <td>13</td>\n",
       "      <td>55</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>51.0</td>\n",
       "      <td>6078.0</td>\n",
       "      <td>4.821</td>\n",
       "      <td>82745.0</td>\n",
       "      <td>96.550</td>\n",
       "      <td>1827.0</td>\n",
       "      <td>2340.0</td>\n",
       "      <td>964</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>240.0</td>\n",
       "      <td>3</td>\n",
       "      <td>374</td>\n",
       "      <td>28</td>\n",
       "      <td>131</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>104.0</td>\n",
       "      <td>6068.0</td>\n",
       "      <td>26.427</td>\n",
       "      <td>65282.0</td>\n",
       "      <td>81.068</td>\n",
       "      <td>1863.0</td>\n",
       "      <td>2560.0</td>\n",
       "      <td>5488</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>210.0</td>\n",
       "      <td>3</td>\n",
       "      <td>322</td>\n",
       "      <td>37</td>\n",
       "      <td>35</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>55.0</td>\n",
       "      <td>6030.0</td>\n",
       "      <td>78.153</td>\n",
       "      <td>108353.0</td>\n",
       "      <td>59.439</td>\n",
       "      <td>969.0</td>\n",
       "      <td>1601.0</td>\n",
       "      <td>2442</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>209.0</td>\n",
       "      <td>3</td>\n",
       "      <td>24</td>\n",
       "      <td>37</td>\n",
       "      <td>20</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>244.0</td>\n",
       "      <td>5151.0</td>\n",
       "      <td>88.156</td>\n",
       "      <td>63414.0</td>\n",
       "      <td>62.813</td>\n",
       "      <td>411.0</td>\n",
       "      <td>481.0</td>\n",
       "      <td>2118</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>197.0</td>\n",
       "      <td>3</td>\n",
       "      <td>194</td>\n",
       "      <td>9</td>\n",
       "      <td>20</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>86.0</td>\n",
       "      <td>7916.0</td>\n",
       "      <td>24.893</td>\n",
       "      <td>65927.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>1861.0</td>\n",
       "      <td>2123.0</td>\n",
       "      <td>3507</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   row_id  loan_type  property_type  loan_purpose  occupancy  loan_amount  \\\n",
       "0       0          3              1             1          1         70.0   \n",
       "1       1          1              1             3          1        178.0   \n",
       "2       2          2              1             3          1        163.0   \n",
       "3       3          1              1             1          1        155.0   \n",
       "4       4          1              1             1          1        305.0   \n",
       "5       5          1              1             3          1        133.0   \n",
       "6       6          3              1             1          1        240.0   \n",
       "7       7          2              1             1          1        210.0   \n",
       "8       8          1              1             3          2        209.0   \n",
       "9       9          1              1             3          1        197.0   \n",
       "\n",
       "   preapproval  msa_md  state_code  county_code  ...  applicant_sex  \\\n",
       "0            3      18          37          246  ...              1   \n",
       "1            3     369          52          299  ...              1   \n",
       "2            3      16          10          306  ...              1   \n",
       "3            1     305          47          180  ...              1   \n",
       "4            3      24          37           20  ...              2   \n",
       "5            3     221          13           55  ...              2   \n",
       "6            3     374          28          131  ...              2   \n",
       "7            3     322          37           35  ...              1   \n",
       "8            3      24          37           20  ...              1   \n",
       "9            3     194           9           20  ...              1   \n",
       "\n",
       "   applicant_income  population  minority_population_pct  \\\n",
       "0              24.0      6203.0                   44.230   \n",
       "1              57.0      5774.0                   15.905   \n",
       "2              67.0      6094.0                   61.270   \n",
       "3             105.0      6667.0                    6.246   \n",
       "4              71.0      6732.0                  100.000   \n",
       "5              51.0      6078.0                    4.821   \n",
       "6             104.0      6068.0                   26.427   \n",
       "7              55.0      6030.0                   78.153   \n",
       "8             244.0      5151.0                   88.156   \n",
       "9              86.0      7916.0                   24.893   \n",
       "\n",
       "   ffiecmedian_family_income  tract_to_msa_md_income_pct  \\\n",
       "0                    60588.0                      50.933   \n",
       "1                    54821.0                     100.000   \n",
       "2                    67719.0                     100.000   \n",
       "3                    78439.0                     100.000   \n",
       "4                    63075.0                      82.200   \n",
       "5                    82745.0                      96.550   \n",
       "6                    65282.0                      81.068   \n",
       "7                   108353.0                      59.439   \n",
       "8                    63414.0                      62.813   \n",
       "9                    65927.0                     100.000   \n",
       "\n",
       "   number_of_owner-occupied_units  number_of_1_to_4_family_units  lender  \\\n",
       "0                           716.0                         2642.0    4536   \n",
       "1                          1622.0                         2108.0    2458   \n",
       "2                           760.0                         1048.0    5710   \n",
       "3                          2025.0                         2299.0    5888   \n",
       "4                          1464.0                         1847.0     289   \n",
       "5                          1827.0                         2340.0     964   \n",
       "6                          1863.0                         2560.0    5488   \n",
       "7                           969.0                         1601.0    2442   \n",
       "8                           411.0                          481.0    2118   \n",
       "9                          1861.0                         2123.0    3507   \n",
       "\n",
       "   co_applicant  \n",
       "0         False  \n",
       "1         False  \n",
       "2         False  \n",
       "3          True  \n",
       "4         False  \n",
       "5         False  \n",
       "6         False  \n",
       "7          True  \n",
       "8          True  \n",
       "9         False  \n",
       "\n",
       "[10 rows x 22 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# rename training data\n",
    "\n",
    "LoanTrain = LoanTrainV\n",
    "\n",
    "print(LoanTrain.shape)\n",
    "LoanTrain.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(500000, 22)\n",
      "(318,)\n"
     ]
    }
   ],
   "source": [
    "#Below- Examine training data for county_code duplicates by checking for unique IDs.\n",
    "\n",
    "print(LoanTrain.shape)\n",
    "print(LoanTrain.county_code.unique().shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loan_amount 0.0\n",
      "applicant_income 7.989599999999999\n",
      "population 4.492999999999999\n",
      "minority_population_pct 4.4932\n",
      "ffiecmedian_family_income 4.488\n",
      "tract_to_msa_md_income_pct 4.5028\n",
      "number_of_owner-occupied_units 4.513\n",
      "number_of_1_to_4_family_units 4.506\n"
     ]
    }
   ],
   "source": [
    "# Below - Handle missing data\n",
    " # Set limit that features which are missing 20% or more of their data are removed.\n",
    "    \n",
    "for col in LoanTrain.columns:\n",
    "    if LoanTrain[col].dtype == float :\n",
    "        count = 0\n",
    "        count = [count + 1 for x in LoanTrain[col] if pd.isna(x)]\n",
    "        prct = ((sum(count))/500000)*100\n",
    "        print(col + ' ' + str(prct))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "15.3964\n",
      "3.8264\n",
      "4.0932\n"
     ]
    }
   ],
   "source": [
    "#Below- Find out how many values are missing as '-1' w/in this integer column feature.\n",
    " # Set limit that features which are missing 20% or more of their data are removed.\n",
    "    \n",
    "missing_msa_md = LoanTrain.msa_md[LoanTrain.msa_md == -1].count()\n",
    "missing_state_code = LoanTrain.state_code[LoanTrain.state_code == -1].count()\n",
    "missing_county_code = LoanTrain.county_code[LoanTrain.county_code == -1].count()\n",
    "\n",
    "pctmiss_msa_ma = ((missing_msa_md)/500000)*100\n",
    "pctmiss_state_code = ((missing_state_code)/500000)*100\n",
    "pctmiss_county_code = ((missing_county_code)/500000)*100\n",
    "\n",
    "print(pctmiss_msa_ma)\n",
    "print(pctmiss_state_code)\n",
    "print(pctmiss_county_code)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Above- None of the columns are above the 20% mark in the dataset.\n",
    "\n",
    "#Below- Save this data to a csv (backup)\n",
    "\n",
    "LoanTrain.to_csv('LoanTrain_2019-04-21.csv', index = False, header = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(500000, 23)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>row_id</th>\n",
       "      <th>loan_type</th>\n",
       "      <th>property_type</th>\n",
       "      <th>loan_purpose</th>\n",
       "      <th>occupancy</th>\n",
       "      <th>loan_amount</th>\n",
       "      <th>preapproval</th>\n",
       "      <th>msa_md</th>\n",
       "      <th>state_code</th>\n",
       "      <th>county_code</th>\n",
       "      <th>...</th>\n",
       "      <th>applicant_income</th>\n",
       "      <th>population</th>\n",
       "      <th>minority_population_pct</th>\n",
       "      <th>ffiecmedian_family_income</th>\n",
       "      <th>tract_to_msa_md_income_pct</th>\n",
       "      <th>number_of_owner-occupied_units</th>\n",
       "      <th>number_of_1_to_4_family_units</th>\n",
       "      <th>lender</th>\n",
       "      <th>co_applicant</th>\n",
       "      <th>accepted</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>70</td>\n",
       "      <td>3</td>\n",
       "      <td>18</td>\n",
       "      <td>37</td>\n",
       "      <td>246</td>\n",
       "      <td>...</td>\n",
       "      <td>24.0</td>\n",
       "      <td>6203.0</td>\n",
       "      <td>44.230</td>\n",
       "      <td>60588.0</td>\n",
       "      <td>50.933</td>\n",
       "      <td>716.0</td>\n",
       "      <td>2642.0</td>\n",
       "      <td>4536</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>178</td>\n",
       "      <td>3</td>\n",
       "      <td>369</td>\n",
       "      <td>52</td>\n",
       "      <td>299</td>\n",
       "      <td>...</td>\n",
       "      <td>57.0</td>\n",
       "      <td>5774.0</td>\n",
       "      <td>15.905</td>\n",
       "      <td>54821.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>1622.0</td>\n",
       "      <td>2108.0</td>\n",
       "      <td>2458</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>163</td>\n",
       "      <td>3</td>\n",
       "      <td>16</td>\n",
       "      <td>10</td>\n",
       "      <td>306</td>\n",
       "      <td>...</td>\n",
       "      <td>67.0</td>\n",
       "      <td>6094.0</td>\n",
       "      <td>61.270</td>\n",
       "      <td>67719.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>760.0</td>\n",
       "      <td>1048.0</td>\n",
       "      <td>5710</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>155</td>\n",
       "      <td>1</td>\n",
       "      <td>305</td>\n",
       "      <td>47</td>\n",
       "      <td>180</td>\n",
       "      <td>...</td>\n",
       "      <td>105.0</td>\n",
       "      <td>6667.0</td>\n",
       "      <td>6.246</td>\n",
       "      <td>78439.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>2025.0</td>\n",
       "      <td>2299.0</td>\n",
       "      <td>5888</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>305</td>\n",
       "      <td>3</td>\n",
       "      <td>24</td>\n",
       "      <td>37</td>\n",
       "      <td>20</td>\n",
       "      <td>...</td>\n",
       "      <td>71.0</td>\n",
       "      <td>6732.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>63075.0</td>\n",
       "      <td>82.200</td>\n",
       "      <td>1464.0</td>\n",
       "      <td>1847.0</td>\n",
       "      <td>289</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>133</td>\n",
       "      <td>3</td>\n",
       "      <td>221</td>\n",
       "      <td>13</td>\n",
       "      <td>55</td>\n",
       "      <td>...</td>\n",
       "      <td>51.0</td>\n",
       "      <td>6078.0</td>\n",
       "      <td>4.821</td>\n",
       "      <td>82745.0</td>\n",
       "      <td>96.550</td>\n",
       "      <td>1827.0</td>\n",
       "      <td>2340.0</td>\n",
       "      <td>964</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>240</td>\n",
       "      <td>3</td>\n",
       "      <td>374</td>\n",
       "      <td>28</td>\n",
       "      <td>131</td>\n",
       "      <td>...</td>\n",
       "      <td>104.0</td>\n",
       "      <td>6068.0</td>\n",
       "      <td>26.427</td>\n",
       "      <td>65282.0</td>\n",
       "      <td>81.068</td>\n",
       "      <td>1863.0</td>\n",
       "      <td>2560.0</td>\n",
       "      <td>5488</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>210</td>\n",
       "      <td>3</td>\n",
       "      <td>322</td>\n",
       "      <td>37</td>\n",
       "      <td>35</td>\n",
       "      <td>...</td>\n",
       "      <td>55.0</td>\n",
       "      <td>6030.0</td>\n",
       "      <td>78.153</td>\n",
       "      <td>108353.0</td>\n",
       "      <td>59.439</td>\n",
       "      <td>969.0</td>\n",
       "      <td>1601.0</td>\n",
       "      <td>2442</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>209</td>\n",
       "      <td>3</td>\n",
       "      <td>24</td>\n",
       "      <td>37</td>\n",
       "      <td>20</td>\n",
       "      <td>...</td>\n",
       "      <td>244.0</td>\n",
       "      <td>5151.0</td>\n",
       "      <td>88.156</td>\n",
       "      <td>63414.0</td>\n",
       "      <td>62.813</td>\n",
       "      <td>411.0</td>\n",
       "      <td>481.0</td>\n",
       "      <td>2118</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>197</td>\n",
       "      <td>3</td>\n",
       "      <td>194</td>\n",
       "      <td>9</td>\n",
       "      <td>20</td>\n",
       "      <td>...</td>\n",
       "      <td>86.0</td>\n",
       "      <td>7916.0</td>\n",
       "      <td>24.893</td>\n",
       "      <td>65927.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>1861.0</td>\n",
       "      <td>2123.0</td>\n",
       "      <td>3507</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10 rows × 23 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   row_id  loan_type  property_type  loan_purpose  occupancy  loan_amount  \\\n",
       "0       0          3              1             1          1           70   \n",
       "1       1          1              1             3          1          178   \n",
       "2       2          2              1             3          1          163   \n",
       "3       3          1              1             1          1          155   \n",
       "4       4          1              1             1          1          305   \n",
       "5       5          1              1             3          1          133   \n",
       "6       6          3              1             1          1          240   \n",
       "7       7          2              1             1          1          210   \n",
       "8       8          1              1             3          2          209   \n",
       "9       9          1              1             3          1          197   \n",
       "\n",
       "   preapproval  msa_md  state_code  county_code  ...  applicant_income  \\\n",
       "0            3      18          37          246  ...              24.0   \n",
       "1            3     369          52          299  ...              57.0   \n",
       "2            3      16          10          306  ...              67.0   \n",
       "3            1     305          47          180  ...             105.0   \n",
       "4            3      24          37           20  ...              71.0   \n",
       "5            3     221          13           55  ...              51.0   \n",
       "6            3     374          28          131  ...             104.0   \n",
       "7            3     322          37           35  ...              55.0   \n",
       "8            3      24          37           20  ...             244.0   \n",
       "9            3     194           9           20  ...              86.0   \n",
       "\n",
       "   population  minority_population_pct  ffiecmedian_family_income  \\\n",
       "0      6203.0                   44.230                    60588.0   \n",
       "1      5774.0                   15.905                    54821.0   \n",
       "2      6094.0                   61.270                    67719.0   \n",
       "3      6667.0                    6.246                    78439.0   \n",
       "4      6732.0                  100.000                    63075.0   \n",
       "5      6078.0                    4.821                    82745.0   \n",
       "6      6068.0                   26.427                    65282.0   \n",
       "7      6030.0                   78.153                   108353.0   \n",
       "8      5151.0                   88.156                    63414.0   \n",
       "9      7916.0                   24.893                    65927.0   \n",
       "\n",
       "   tract_to_msa_md_income_pct  number_of_owner-occupied_units  \\\n",
       "0                      50.933                           716.0   \n",
       "1                     100.000                          1622.0   \n",
       "2                     100.000                           760.0   \n",
       "3                     100.000                          2025.0   \n",
       "4                      82.200                          1464.0   \n",
       "5                      96.550                          1827.0   \n",
       "6                      81.068                          1863.0   \n",
       "7                      59.439                           969.0   \n",
       "8                      62.813                           411.0   \n",
       "9                     100.000                          1861.0   \n",
       "\n",
       "   number_of_1_to_4_family_units  lender  co_applicant  accepted  \n",
       "0                         2642.0    4536         False         1  \n",
       "1                         2108.0    2458         False         0  \n",
       "2                         1048.0    5710         False         1  \n",
       "3                         2299.0    5888          True         1  \n",
       "4                         1847.0     289         False         1  \n",
       "5                         2340.0     964         False         1  \n",
       "6                         2560.0    5488         False         1  \n",
       "7                         1601.0    2442          True         1  \n",
       "8                          481.0    2118          True         1  \n",
       "9                         2123.0    3507         False         0  \n",
       "\n",
       "[10 rows x 23 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Sorted the backup csv by smallest to largest for state_code then county_code. No data deleted.\n",
    "# Load the grouped feature dataset, display shape, & explore first 10 rows of Pandas data frame.\n",
    "# Labels (accepted) were concatenated to end of dataset.\n",
    "\n",
    "LoanTrainG = pd.read_csv('LoanTrain_Groups_2019-04-21.csv', header=0)\n",
    "print(LoanTrainG.shape)\n",
    "LoanTrainG.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n",
      "75900\n"
     ]
    }
   ],
   "source": [
    "#Below- RECHECK-Find out how many values are missing as '-1' w/in this integer column feature.\n",
    "\n",
    "print(any(LoanTrainG.msa_md == -1))\n",
    "print(LoanTrainG.msa_md[LoanTrainG.msa_md == -1].count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n",
      "19132\n"
     ]
    }
   ],
   "source": [
    "#Above- Still have 75,900 missing data for msa_md.\n",
    "\n",
    "#Below- RECHECK-Find out how many values are missing as '-1' w/in this integer column feature.\n",
    "\n",
    "print(any(LoanTrainG.state_code == -1))\n",
    "print(LoanTrainG.state_code[LoanTrainG.state_code == -1].count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n",
      "20466\n"
     ]
    }
   ],
   "source": [
    "#Above- Still have 19,132 missing data for state_code.\n",
    "\n",
    "#Below- RECHECK-Find out how many values are missing as '-1' w/in this integer column feature.\n",
    "\n",
    "print(any(LoanTrainG.county_code == -1))\n",
    "print(LoanTrainG.county_code[LoanTrainG.county_code == -1].count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "row_id                            249999.500\n",
      "loan_type                              1.000\n",
      "property_type                          1.000\n",
      "loan_purpose                           2.000\n",
      "occupancy                              1.000\n",
      "loan_amount                          162.000\n",
      "preapproval                            3.000\n",
      "msa_md                               194.000\n",
      "state_code                            26.000\n",
      "county_code                          131.000\n",
      "applicant_ethnicity                    2.000\n",
      "applicant_race                         5.000\n",
      "applicant_sex                          1.000\n",
      "applicant_income                      74.000\n",
      "population                          4975.000\n",
      "minority_population_pct               22.901\n",
      "ffiecmedian_family_income          67526.000\n",
      "tract_to_msa_md_income_pct           100.000\n",
      "number_of_owner-occupied_units      1327.000\n",
      "number_of_1_to_4_family_units       1753.000\n",
      "lender                              3731.000\n",
      "co_applicant                           0.000\n",
      "accepted                               1.000\n",
      "dtype: float64\n"
     ]
    }
   ],
   "source": [
    "#Above- Still have 20,466 missing data for county_code.\n",
    "\n",
    "#Below- Since the mean and medians were so different from each other, indicating skewness, we will replace missing values in rows\n",
    " # with medians instead of means. \n",
    "    \n",
    "# Calculate median of each column in dataset.\n",
    "\n",
    "median= LoanTrainG.median()\n",
    "print(median)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>row_id</th>\n",
       "      <th>loan_type</th>\n",
       "      <th>property_type</th>\n",
       "      <th>loan_purpose</th>\n",
       "      <th>occupancy</th>\n",
       "      <th>loan_amount</th>\n",
       "      <th>preapproval</th>\n",
       "      <th>msa_md</th>\n",
       "      <th>state_code</th>\n",
       "      <th>county_code</th>\n",
       "      <th>...</th>\n",
       "      <th>applicant_income</th>\n",
       "      <th>population</th>\n",
       "      <th>minority_population_pct</th>\n",
       "      <th>ffiecmedian_family_income</th>\n",
       "      <th>tract_to_msa_md_income_pct</th>\n",
       "      <th>number_of_owner-occupied_units</th>\n",
       "      <th>number_of_1_to_4_family_units</th>\n",
       "      <th>lender</th>\n",
       "      <th>co_applicant</th>\n",
       "      <th>accepted</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>70</td>\n",
       "      <td>3</td>\n",
       "      <td>18</td>\n",
       "      <td>37</td>\n",
       "      <td>246</td>\n",
       "      <td>...</td>\n",
       "      <td>24.0</td>\n",
       "      <td>6203.0</td>\n",
       "      <td>44.230</td>\n",
       "      <td>60588.0</td>\n",
       "      <td>50.933</td>\n",
       "      <td>716.0</td>\n",
       "      <td>2642.0</td>\n",
       "      <td>4536</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>178</td>\n",
       "      <td>3</td>\n",
       "      <td>369</td>\n",
       "      <td>52</td>\n",
       "      <td>299</td>\n",
       "      <td>...</td>\n",
       "      <td>57.0</td>\n",
       "      <td>5774.0</td>\n",
       "      <td>15.905</td>\n",
       "      <td>54821.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>1622.0</td>\n",
       "      <td>2108.0</td>\n",
       "      <td>2458</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>163</td>\n",
       "      <td>3</td>\n",
       "      <td>16</td>\n",
       "      <td>10</td>\n",
       "      <td>306</td>\n",
       "      <td>...</td>\n",
       "      <td>67.0</td>\n",
       "      <td>6094.0</td>\n",
       "      <td>61.270</td>\n",
       "      <td>67719.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>760.0</td>\n",
       "      <td>1048.0</td>\n",
       "      <td>5710</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>155</td>\n",
       "      <td>1</td>\n",
       "      <td>305</td>\n",
       "      <td>47</td>\n",
       "      <td>180</td>\n",
       "      <td>...</td>\n",
       "      <td>105.0</td>\n",
       "      <td>6667.0</td>\n",
       "      <td>6.246</td>\n",
       "      <td>78439.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>2025.0</td>\n",
       "      <td>2299.0</td>\n",
       "      <td>5888</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>305</td>\n",
       "      <td>3</td>\n",
       "      <td>24</td>\n",
       "      <td>37</td>\n",
       "      <td>20</td>\n",
       "      <td>...</td>\n",
       "      <td>71.0</td>\n",
       "      <td>6732.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>63075.0</td>\n",
       "      <td>82.200</td>\n",
       "      <td>1464.0</td>\n",
       "      <td>1847.0</td>\n",
       "      <td>289</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>133</td>\n",
       "      <td>3</td>\n",
       "      <td>221</td>\n",
       "      <td>13</td>\n",
       "      <td>55</td>\n",
       "      <td>...</td>\n",
       "      <td>51.0</td>\n",
       "      <td>6078.0</td>\n",
       "      <td>4.821</td>\n",
       "      <td>82745.0</td>\n",
       "      <td>96.550</td>\n",
       "      <td>1827.0</td>\n",
       "      <td>2340.0</td>\n",
       "      <td>964</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>240</td>\n",
       "      <td>3</td>\n",
       "      <td>374</td>\n",
       "      <td>28</td>\n",
       "      <td>131</td>\n",
       "      <td>...</td>\n",
       "      <td>104.0</td>\n",
       "      <td>6068.0</td>\n",
       "      <td>26.427</td>\n",
       "      <td>65282.0</td>\n",
       "      <td>81.068</td>\n",
       "      <td>1863.0</td>\n",
       "      <td>2560.0</td>\n",
       "      <td>5488</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>210</td>\n",
       "      <td>3</td>\n",
       "      <td>322</td>\n",
       "      <td>37</td>\n",
       "      <td>35</td>\n",
       "      <td>...</td>\n",
       "      <td>55.0</td>\n",
       "      <td>6030.0</td>\n",
       "      <td>78.153</td>\n",
       "      <td>108353.0</td>\n",
       "      <td>59.439</td>\n",
       "      <td>969.0</td>\n",
       "      <td>1601.0</td>\n",
       "      <td>2442</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>209</td>\n",
       "      <td>3</td>\n",
       "      <td>24</td>\n",
       "      <td>37</td>\n",
       "      <td>20</td>\n",
       "      <td>...</td>\n",
       "      <td>244.0</td>\n",
       "      <td>5151.0</td>\n",
       "      <td>88.156</td>\n",
       "      <td>63414.0</td>\n",
       "      <td>62.813</td>\n",
       "      <td>411.0</td>\n",
       "      <td>481.0</td>\n",
       "      <td>2118</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>197</td>\n",
       "      <td>3</td>\n",
       "      <td>194</td>\n",
       "      <td>9</td>\n",
       "      <td>20</td>\n",
       "      <td>...</td>\n",
       "      <td>86.0</td>\n",
       "      <td>7916.0</td>\n",
       "      <td>24.893</td>\n",
       "      <td>65927.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>1861.0</td>\n",
       "      <td>2123.0</td>\n",
       "      <td>3507</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>319</td>\n",
       "      <td>3</td>\n",
       "      <td>358</td>\n",
       "      <td>32</td>\n",
       "      <td>244</td>\n",
       "      <td>...</td>\n",
       "      <td>78.0</td>\n",
       "      <td>5931.0</td>\n",
       "      <td>39.082</td>\n",
       "      <td>109743.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>1315.0</td>\n",
       "      <td>1382.0</td>\n",
       "      <td>838</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>187</td>\n",
       "      <td>3</td>\n",
       "      <td>154</td>\n",
       "      <td>44</td>\n",
       "      <td>55</td>\n",
       "      <td>...</td>\n",
       "      <td>63.0</td>\n",
       "      <td>6917.0</td>\n",
       "      <td>6.922</td>\n",
       "      <td>73503.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>1858.0</td>\n",
       "      <td>2050.0</td>\n",
       "      <td>5154</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>12</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>138</td>\n",
       "      <td>3</td>\n",
       "      <td>61</td>\n",
       "      <td>28</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9958.0</td>\n",
       "      <td>41.404</td>\n",
       "      <td>60081.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>2220.0</td>\n",
       "      <td>2727.0</td>\n",
       "      <td>4664</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>240</td>\n",
       "      <td>3</td>\n",
       "      <td>141</td>\n",
       "      <td>6</td>\n",
       "      <td>250</td>\n",
       "      <td>...</td>\n",
       "      <td>90.0</td>\n",
       "      <td>4076.0</td>\n",
       "      <td>37.949</td>\n",
       "      <td>76369.0</td>\n",
       "      <td>87.937</td>\n",
       "      <td>835.0</td>\n",
       "      <td>1696.0</td>\n",
       "      <td>5154</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>151</td>\n",
       "      <td>3</td>\n",
       "      <td>281</td>\n",
       "      <td>2</td>\n",
       "      <td>210</td>\n",
       "      <td>...</td>\n",
       "      <td>172.0</td>\n",
       "      <td>2165.0</td>\n",
       "      <td>3.916</td>\n",
       "      <td>63560.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>839.0</td>\n",
       "      <td>1263.0</td>\n",
       "      <td>5473</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>208</td>\n",
       "      <td>3</td>\n",
       "      <td>101</td>\n",
       "      <td>16</td>\n",
       "      <td>313</td>\n",
       "      <td>...</td>\n",
       "      <td>53.0</td>\n",
       "      <td>5841.0</td>\n",
       "      <td>63.554</td>\n",
       "      <td>69265.0</td>\n",
       "      <td>82.998</td>\n",
       "      <td>1196.0</td>\n",
       "      <td>1632.0</td>\n",
       "      <td>5316</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>16</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>177</td>\n",
       "      <td>3</td>\n",
       "      <td>333</td>\n",
       "      <td>6</td>\n",
       "      <td>80</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6420.0</td>\n",
       "      <td>29.818</td>\n",
       "      <td>68065.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>1553.0</td>\n",
       "      <td>2001.0</td>\n",
       "      <td>3354</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>17</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>138</td>\n",
       "      <td>3</td>\n",
       "      <td>-1</td>\n",
       "      <td>37</td>\n",
       "      <td>59</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4193.0</td>\n",
       "      <td>14.996</td>\n",
       "      <td>57774.0</td>\n",
       "      <td>74.411</td>\n",
       "      <td>1247.0</td>\n",
       "      <td>1998.0</td>\n",
       "      <td>2566</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>18</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>195</td>\n",
       "      <td>1</td>\n",
       "      <td>214</td>\n",
       "      <td>15</td>\n",
       "      <td>180</td>\n",
       "      <td>...</td>\n",
       "      <td>111.0</td>\n",
       "      <td>3792.0</td>\n",
       "      <td>8.760</td>\n",
       "      <td>66533.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>1309.0</td>\n",
       "      <td>1354.0</td>\n",
       "      <td>4701</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>19</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>90</td>\n",
       "      <td>3</td>\n",
       "      <td>101</td>\n",
       "      <td>16</td>\n",
       "      <td>49</td>\n",
       "      <td>...</td>\n",
       "      <td>63.0</td>\n",
       "      <td>7316.0</td>\n",
       "      <td>36.347</td>\n",
       "      <td>69795.0</td>\n",
       "      <td>92.747</td>\n",
       "      <td>1498.0</td>\n",
       "      <td>1903.0</td>\n",
       "      <td>1462</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>111</td>\n",
       "      <td>3</td>\n",
       "      <td>278</td>\n",
       "      <td>2</td>\n",
       "      <td>208</td>\n",
       "      <td>...</td>\n",
       "      <td>41.0</td>\n",
       "      <td>6059.0</td>\n",
       "      <td>96.448</td>\n",
       "      <td>50655.0</td>\n",
       "      <td>72.441</td>\n",
       "      <td>1325.0</td>\n",
       "      <td>1551.0</td>\n",
       "      <td>5726</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>21</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>192</td>\n",
       "      <td>2</td>\n",
       "      <td>197</td>\n",
       "      <td>37</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>69.0</td>\n",
       "      <td>4625.0</td>\n",
       "      <td>68.823</td>\n",
       "      <td>48604.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>865.0</td>\n",
       "      <td>1083.0</td>\n",
       "      <td>4145</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>22</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>20</td>\n",
       "      <td>3</td>\n",
       "      <td>109</td>\n",
       "      <td>14</td>\n",
       "      <td>131</td>\n",
       "      <td>...</td>\n",
       "      <td>11.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4347</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>23</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>764</td>\n",
       "      <td>3</td>\n",
       "      <td>206</td>\n",
       "      <td>30</td>\n",
       "      <td>203</td>\n",
       "      <td>...</td>\n",
       "      <td>217.0</td>\n",
       "      <td>9184.0</td>\n",
       "      <td>12.486</td>\n",
       "      <td>104550.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>2545.0</td>\n",
       "      <td>2425.0</td>\n",
       "      <td>2054</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>24</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>602</td>\n",
       "      <td>2</td>\n",
       "      <td>389</td>\n",
       "      <td>37</td>\n",
       "      <td>180</td>\n",
       "      <td>...</td>\n",
       "      <td>85.0</td>\n",
       "      <td>8585.0</td>\n",
       "      <td>31.068</td>\n",
       "      <td>85202.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>2159.0</td>\n",
       "      <td>2550.0</td>\n",
       "      <td>3817</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>292</td>\n",
       "      <td>3</td>\n",
       "      <td>333</td>\n",
       "      <td>6</td>\n",
       "      <td>80</td>\n",
       "      <td>...</td>\n",
       "      <td>99.0</td>\n",
       "      <td>4346.0</td>\n",
       "      <td>16.489</td>\n",
       "      <td>70745.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>1198.0</td>\n",
       "      <td>1349.0</td>\n",
       "      <td>2458</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>26</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>113</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>...</td>\n",
       "      <td>54.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2839</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>27</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>15</td>\n",
       "      <td>3</td>\n",
       "      <td>254</td>\n",
       "      <td>48</td>\n",
       "      <td>56</td>\n",
       "      <td>...</td>\n",
       "      <td>53.0</td>\n",
       "      <td>4885.0</td>\n",
       "      <td>2.693</td>\n",
       "      <td>62795.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>1462.0</td>\n",
       "      <td>1955.0</td>\n",
       "      <td>4419</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>28</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>75</td>\n",
       "      <td>3</td>\n",
       "      <td>175</td>\n",
       "      <td>22</td>\n",
       "      <td>35</td>\n",
       "      <td>...</td>\n",
       "      <td>52.0</td>\n",
       "      <td>6184.0</td>\n",
       "      <td>7.898</td>\n",
       "      <td>57325.0</td>\n",
       "      <td>96.479</td>\n",
       "      <td>1970.0</td>\n",
       "      <td>2834.0</td>\n",
       "      <td>6240</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>29</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>368</td>\n",
       "      <td>2</td>\n",
       "      <td>24</td>\n",
       "      <td>37</td>\n",
       "      <td>20</td>\n",
       "      <td>...</td>\n",
       "      <td>75.0</td>\n",
       "      <td>3327.0</td>\n",
       "      <td>93.204</td>\n",
       "      <td>62340.0</td>\n",
       "      <td>62.557</td>\n",
       "      <td>382.0</td>\n",
       "      <td>760.0</td>\n",
       "      <td>2240</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499970</th>\n",
       "      <td>499970</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>295</td>\n",
       "      <td>3</td>\n",
       "      <td>371</td>\n",
       "      <td>37</td>\n",
       "      <td>260</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5033.0</td>\n",
       "      <td>58.914</td>\n",
       "      <td>74980.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>1564.0</td>\n",
       "      <td>1746.0</td>\n",
       "      <td>2108</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499971</th>\n",
       "      <td>499971</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>34</td>\n",
       "      <td>3</td>\n",
       "      <td>287</td>\n",
       "      <td>5</td>\n",
       "      <td>313</td>\n",
       "      <td>...</td>\n",
       "      <td>78.0</td>\n",
       "      <td>10114.0</td>\n",
       "      <td>18.839</td>\n",
       "      <td>73998.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>3030.0</td>\n",
       "      <td>3624.0</td>\n",
       "      <td>2816</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499972</th>\n",
       "      <td>499972</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>42</td>\n",
       "      <td>3</td>\n",
       "      <td>246</td>\n",
       "      <td>5</td>\n",
       "      <td>260</td>\n",
       "      <td>...</td>\n",
       "      <td>20.0</td>\n",
       "      <td>3688.0</td>\n",
       "      <td>94.824</td>\n",
       "      <td>64139.0</td>\n",
       "      <td>57.455</td>\n",
       "      <td>659.0</td>\n",
       "      <td>1567.0</td>\n",
       "      <td>3581</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499973</th>\n",
       "      <td>499973</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>323</td>\n",
       "      <td>3</td>\n",
       "      <td>356</td>\n",
       "      <td>5</td>\n",
       "      <td>98</td>\n",
       "      <td>...</td>\n",
       "      <td>48.0</td>\n",
       "      <td>3478.0</td>\n",
       "      <td>35.680</td>\n",
       "      <td>53969.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>460.0</td>\n",
       "      <td>600.0</td>\n",
       "      <td>1919</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499974</th>\n",
       "      <td>499974</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>366</td>\n",
       "      <td>2</td>\n",
       "      <td>141</td>\n",
       "      <td>6</td>\n",
       "      <td>250</td>\n",
       "      <td>...</td>\n",
       "      <td>50.0</td>\n",
       "      <td>5085.0</td>\n",
       "      <td>35.779</td>\n",
       "      <td>78743.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>1313.0</td>\n",
       "      <td>1612.0</td>\n",
       "      <td>1928</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499975</th>\n",
       "      <td>499975</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>181</td>\n",
       "      <td>3</td>\n",
       "      <td>382</td>\n",
       "      <td>14</td>\n",
       "      <td>115</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4499.0</td>\n",
       "      <td>12.106</td>\n",
       "      <td>67680.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>1407.0</td>\n",
       "      <td>1504.0</td>\n",
       "      <td>169</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499976</th>\n",
       "      <td>499976</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>242</td>\n",
       "      <td>3</td>\n",
       "      <td>281</td>\n",
       "      <td>2</td>\n",
       "      <td>163</td>\n",
       "      <td>...</td>\n",
       "      <td>69.0</td>\n",
       "      <td>9737.0</td>\n",
       "      <td>8.471</td>\n",
       "      <td>62190.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>3044.0</td>\n",
       "      <td>3612.0</td>\n",
       "      <td>2458</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499977</th>\n",
       "      <td>499977</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>52</td>\n",
       "      <td>3</td>\n",
       "      <td>120</td>\n",
       "      <td>0</td>\n",
       "      <td>307</td>\n",
       "      <td>...</td>\n",
       "      <td>86.0</td>\n",
       "      <td>5317.0</td>\n",
       "      <td>24.059</td>\n",
       "      <td>60028.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>1231.0</td>\n",
       "      <td>1871.0</td>\n",
       "      <td>2775</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499978</th>\n",
       "      <td>499978</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>172</td>\n",
       "      <td>3</td>\n",
       "      <td>-1</td>\n",
       "      <td>40</td>\n",
       "      <td>134</td>\n",
       "      <td>...</td>\n",
       "      <td>54.0</td>\n",
       "      <td>2735.0</td>\n",
       "      <td>2.910</td>\n",
       "      <td>61429.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>1095.0</td>\n",
       "      <td>2267.0</td>\n",
       "      <td>3354</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499979</th>\n",
       "      <td>499979</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>148</td>\n",
       "      <td>3</td>\n",
       "      <td>278</td>\n",
       "      <td>2</td>\n",
       "      <td>208</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7088.0</td>\n",
       "      <td>70.859</td>\n",
       "      <td>50269.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>2103.0</td>\n",
       "      <td>2377.0</td>\n",
       "      <td>2458</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499980</th>\n",
       "      <td>499980</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>211</td>\n",
       "      <td>3</td>\n",
       "      <td>374</td>\n",
       "      <td>28</td>\n",
       "      <td>131</td>\n",
       "      <td>...</td>\n",
       "      <td>76.0</td>\n",
       "      <td>3326.0</td>\n",
       "      <td>38.107</td>\n",
       "      <td>65762.0</td>\n",
       "      <td>94.810</td>\n",
       "      <td>512.0</td>\n",
       "      <td>599.0</td>\n",
       "      <td>4449</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499981</th>\n",
       "      <td>499981</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>134</td>\n",
       "      <td>3</td>\n",
       "      <td>284</td>\n",
       "      <td>39</td>\n",
       "      <td>129</td>\n",
       "      <td>...</td>\n",
       "      <td>72.0</td>\n",
       "      <td>4867.0</td>\n",
       "      <td>7.183</td>\n",
       "      <td>57188.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>1576.0</td>\n",
       "      <td>1978.0</td>\n",
       "      <td>6300</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499982</th>\n",
       "      <td>499982</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>90</td>\n",
       "      <td>3</td>\n",
       "      <td>157</td>\n",
       "      <td>31</td>\n",
       "      <td>129</td>\n",
       "      <td>...</td>\n",
       "      <td>39.0</td>\n",
       "      <td>5727.0</td>\n",
       "      <td>30.918</td>\n",
       "      <td>54546.0</td>\n",
       "      <td>85.171</td>\n",
       "      <td>1543.0</td>\n",
       "      <td>2526.0</td>\n",
       "      <td>4129</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499983</th>\n",
       "      <td>499983</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>205</td>\n",
       "      <td>2</td>\n",
       "      <td>221</td>\n",
       "      <td>13</td>\n",
       "      <td>129</td>\n",
       "      <td>...</td>\n",
       "      <td>164.0</td>\n",
       "      <td>3331.0</td>\n",
       "      <td>20.892</td>\n",
       "      <td>80134.0</td>\n",
       "      <td>69.377</td>\n",
       "      <td>657.0</td>\n",
       "      <td>1474.0</td>\n",
       "      <td>6093</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499984</th>\n",
       "      <td>499984</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>194</td>\n",
       "      <td>3</td>\n",
       "      <td>370</td>\n",
       "      <td>5</td>\n",
       "      <td>124</td>\n",
       "      <td>...</td>\n",
       "      <td>33.0</td>\n",
       "      <td>7859.0</td>\n",
       "      <td>44.802</td>\n",
       "      <td>60826.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>2754.0</td>\n",
       "      <td>2969.0</td>\n",
       "      <td>4683</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499985</th>\n",
       "      <td>499985</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>201</td>\n",
       "      <td>3</td>\n",
       "      <td>10</td>\n",
       "      <td>34</td>\n",
       "      <td>79</td>\n",
       "      <td>...</td>\n",
       "      <td>60.0</td>\n",
       "      <td>1834.0</td>\n",
       "      <td>14.957</td>\n",
       "      <td>90502.0</td>\n",
       "      <td>88.586</td>\n",
       "      <td>738.0</td>\n",
       "      <td>787.0</td>\n",
       "      <td>986</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499986</th>\n",
       "      <td>499986</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>295</td>\n",
       "      <td>3</td>\n",
       "      <td>114</td>\n",
       "      <td>4</td>\n",
       "      <td>233</td>\n",
       "      <td>...</td>\n",
       "      <td>101.0</td>\n",
       "      <td>3426.0</td>\n",
       "      <td>16.397</td>\n",
       "      <td>68947.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>1023.0</td>\n",
       "      <td>1251.0</td>\n",
       "      <td>6044</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499987</th>\n",
       "      <td>499987</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>171</td>\n",
       "      <td>3</td>\n",
       "      <td>61</td>\n",
       "      <td>28</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>73.0</td>\n",
       "      <td>7295.0</td>\n",
       "      <td>19.892</td>\n",
       "      <td>60113.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>2308.0</td>\n",
       "      <td>3175.0</td>\n",
       "      <td>3968</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499988</th>\n",
       "      <td>499988</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>144</td>\n",
       "      <td>3</td>\n",
       "      <td>16</td>\n",
       "      <td>31</td>\n",
       "      <td>61</td>\n",
       "      <td>...</td>\n",
       "      <td>77.0</td>\n",
       "      <td>9221.0</td>\n",
       "      <td>11.097</td>\n",
       "      <td>64458.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>2777.0</td>\n",
       "      <td>3172.0</td>\n",
       "      <td>5710</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499989</th>\n",
       "      <td>499989</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>630</td>\n",
       "      <td>3</td>\n",
       "      <td>395</td>\n",
       "      <td>37</td>\n",
       "      <td>165</td>\n",
       "      <td>...</td>\n",
       "      <td>202.0</td>\n",
       "      <td>4868.0</td>\n",
       "      <td>40.355</td>\n",
       "      <td>90302.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>1251.0</td>\n",
       "      <td>1410.0</td>\n",
       "      <td>5710</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499990</th>\n",
       "      <td>499990</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>190</td>\n",
       "      <td>3</td>\n",
       "      <td>18</td>\n",
       "      <td>37</td>\n",
       "      <td>246</td>\n",
       "      <td>...</td>\n",
       "      <td>79.0</td>\n",
       "      <td>4671.0</td>\n",
       "      <td>85.037</td>\n",
       "      <td>61141.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>720.0</td>\n",
       "      <td>915.0</td>\n",
       "      <td>2118</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499991</th>\n",
       "      <td>499991</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>151</td>\n",
       "      <td>1</td>\n",
       "      <td>72</td>\n",
       "      <td>41</td>\n",
       "      <td>296</td>\n",
       "      <td>...</td>\n",
       "      <td>27.0</td>\n",
       "      <td>8161.0</td>\n",
       "      <td>23.668</td>\n",
       "      <td>59031.0</td>\n",
       "      <td>86.953</td>\n",
       "      <td>2224.0</td>\n",
       "      <td>2919.0</td>\n",
       "      <td>6451</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499992</th>\n",
       "      <td>499992</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>310</td>\n",
       "      <td>2</td>\n",
       "      <td>156</td>\n",
       "      <td>47</td>\n",
       "      <td>313</td>\n",
       "      <td>...</td>\n",
       "      <td>138.0</td>\n",
       "      <td>4858.0</td>\n",
       "      <td>10.196</td>\n",
       "      <td>73821.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>1511.0</td>\n",
       "      <td>1803.0</td>\n",
       "      <td>5907</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499993</th>\n",
       "      <td>499993</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>200</td>\n",
       "      <td>3</td>\n",
       "      <td>199</td>\n",
       "      <td>26</td>\n",
       "      <td>45</td>\n",
       "      <td>...</td>\n",
       "      <td>127.0</td>\n",
       "      <td>9535.0</td>\n",
       "      <td>5.584</td>\n",
       "      <td>65139.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>2913.0</td>\n",
       "      <td>3289.0</td>\n",
       "      <td>3354</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499994</th>\n",
       "      <td>499994</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>31</td>\n",
       "      <td>3</td>\n",
       "      <td>-1</td>\n",
       "      <td>37</td>\n",
       "      <td>156</td>\n",
       "      <td>...</td>\n",
       "      <td>147.0</td>\n",
       "      <td>3765.0</td>\n",
       "      <td>13.789</td>\n",
       "      <td>58049.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>1154.0</td>\n",
       "      <td>1955.0</td>\n",
       "      <td>5391</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499995</th>\n",
       "      <td>499995</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>150</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>...</td>\n",
       "      <td>87.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>969</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499996</th>\n",
       "      <td>499996</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>140</td>\n",
       "      <td>3</td>\n",
       "      <td>74</td>\n",
       "      <td>47</td>\n",
       "      <td>98</td>\n",
       "      <td>...</td>\n",
       "      <td>216.0</td>\n",
       "      <td>3452.0</td>\n",
       "      <td>6.887</td>\n",
       "      <td>86307.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>1423.0</td>\n",
       "      <td>1944.0</td>\n",
       "      <td>5359</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499997</th>\n",
       "      <td>499997</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>22</td>\n",
       "      <td>3</td>\n",
       "      <td>-1</td>\n",
       "      <td>46</td>\n",
       "      <td>131</td>\n",
       "      <td>...</td>\n",
       "      <td>35.0</td>\n",
       "      <td>2925.0</td>\n",
       "      <td>36.046</td>\n",
       "      <td>47826.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>1062.0</td>\n",
       "      <td>1762.0</td>\n",
       "      <td>2318</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499998</th>\n",
       "      <td>499998</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>35</td>\n",
       "      <td>3</td>\n",
       "      <td>367</td>\n",
       "      <td>6</td>\n",
       "      <td>149</td>\n",
       "      <td>...</td>\n",
       "      <td>41.0</td>\n",
       "      <td>3442.0</td>\n",
       "      <td>98.879</td>\n",
       "      <td>60327.0</td>\n",
       "      <td>62.803</td>\n",
       "      <td>618.0</td>\n",
       "      <td>1297.0</td>\n",
       "      <td>5339</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499999</th>\n",
       "      <td>499999</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>71</td>\n",
       "      <td>3</td>\n",
       "      <td>408</td>\n",
       "      <td>28</td>\n",
       "      <td>133</td>\n",
       "      <td>...</td>\n",
       "      <td>85.0</td>\n",
       "      <td>3856.0</td>\n",
       "      <td>12.095</td>\n",
       "      <td>45033.0</td>\n",
       "      <td>100.000</td>\n",
       "      <td>1525.0</td>\n",
       "      <td>2750.0</td>\n",
       "      <td>5710</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>500000 rows × 23 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        row_id  loan_type  property_type  loan_purpose  occupancy  \\\n",
       "0            0          3              1             1          1   \n",
       "1            1          1              1             3          1   \n",
       "2            2          2              1             3          1   \n",
       "3            3          1              1             1          1   \n",
       "4            4          1              1             1          1   \n",
       "5            5          1              1             3          1   \n",
       "6            6          3              1             1          1   \n",
       "7            7          2              1             1          1   \n",
       "8            8          1              1             3          2   \n",
       "9            9          1              1             3          1   \n",
       "10          10          1              1             1          1   \n",
       "11          11          1              1             1          1   \n",
       "12          12          2              1             3          1   \n",
       "13          13          1              1             1          2   \n",
       "14          14          1              1             1          2   \n",
       "15          15          1              1             3          1   \n",
       "16          16          1              1             3          2   \n",
       "17          17          2              2             3          1   \n",
       "18          18          3              1             1          1   \n",
       "19          19          1              1             3          1   \n",
       "20          20          1              1             3          1   \n",
       "21          21          2              1             1          1   \n",
       "22          22          1              1             3          2   \n",
       "23          23          1              1             1          1   \n",
       "24          24          1              1             1          1   \n",
       "25          25          1              1             3          1   \n",
       "26          26          1              1             1          1   \n",
       "27          27          1              1             2          1   \n",
       "28          28          1              1             2          1   \n",
       "29          29          1              1             1          1   \n",
       "...        ...        ...            ...           ...        ...   \n",
       "499970  499970          3              1             3          1   \n",
       "499971  499971          1              1             1          1   \n",
       "499972  499972          1              1             2          1   \n",
       "499973  499973          1              1             1          1   \n",
       "499974  499974          1              1             1          1   \n",
       "499975  499975          1              1             3          1   \n",
       "499976  499976          1              1             3          1   \n",
       "499977  499977          1              1             3          1   \n",
       "499978  499978          1              1             3          1   \n",
       "499979  499979          1              1             3          1   \n",
       "499980  499980          1              1             1          1   \n",
       "499981  499981          2              1             1          1   \n",
       "499982  499982          1              1             3          1   \n",
       "499983  499983          1              1             1          2   \n",
       "499984  499984          2              1             1          1   \n",
       "499985  499985          1              1             1          1   \n",
       "499986  499986          2              1             3          1   \n",
       "499987  499987          2              1             1          1   \n",
       "499988  499988          1              1             3          2   \n",
       "499989  499989          1              1             3          1   \n",
       "499990  499990          1              1             3          1   \n",
       "499991  499991          1              1             1          1   \n",
       "499992  499992          1              1             1          1   \n",
       "499993  499993          1              1             3          1   \n",
       "499994  499994          1              1             2          1   \n",
       "499995  499995          1              1             1          2   \n",
       "499996  499996          1              1             1          1   \n",
       "499997  499997          1              2             1          1   \n",
       "499998  499998          1              1             2          1   \n",
       "499999  499999          1              1             3          1   \n",
       "\n",
       "        loan_amount  preapproval  msa_md  state_code  county_code  ...  \\\n",
       "0                70            3      18          37          246  ...   \n",
       "1               178            3     369          52          299  ...   \n",
       "2               163            3      16          10          306  ...   \n",
       "3               155            1     305          47          180  ...   \n",
       "4               305            3      24          37           20  ...   \n",
       "5               133            3     221          13           55  ...   \n",
       "6               240            3     374          28          131  ...   \n",
       "7               210            3     322          37           35  ...   \n",
       "8               209            3      24          37           20  ...   \n",
       "9               197            3     194           9           20  ...   \n",
       "10              319            3     358          32          244  ...   \n",
       "11              187            3     154          44           55  ...   \n",
       "12              138            3      61          28            3  ...   \n",
       "13              240            3     141           6          250  ...   \n",
       "14              151            3     281           2          210  ...   \n",
       "15              208            3     101          16          313  ...   \n",
       "16              177            3     333           6           80  ...   \n",
       "17              138            3      -1          37           59  ...   \n",
       "18              195            1     214          15          180  ...   \n",
       "19               90            3     101          16           49  ...   \n",
       "20              111            3     278           2          208  ...   \n",
       "21              192            2     197          37            3  ...   \n",
       "22               20            3     109          14          131  ...   \n",
       "23              764            3     206          30          203  ...   \n",
       "24              602            2     389          37          180  ...   \n",
       "25              292            3     333           6           80  ...   \n",
       "26              113            1      -1          -1           -1  ...   \n",
       "27               15            3     254          48           56  ...   \n",
       "28               75            3     175          22           35  ...   \n",
       "29              368            2      24          37           20  ...   \n",
       "...             ...          ...     ...         ...          ...  ...   \n",
       "499970          295            3     371          37          260  ...   \n",
       "499971           34            3     287           5          313  ...   \n",
       "499972           42            3     246           5          260  ...   \n",
       "499973          323            3     356           5           98  ...   \n",
       "499974          366            2     141           6          250  ...   \n",
       "499975          181            3     382          14          115  ...   \n",
       "499976          242            3     281           2          163  ...   \n",
       "499977           52            3     120           0          307  ...   \n",
       "499978          172            3      -1          40          134  ...   \n",
       "499979          148            3     278           2          208  ...   \n",
       "499980          211            3     374          28          131  ...   \n",
       "499981          134            3     284          39          129  ...   \n",
       "499982           90            3     157          31          129  ...   \n",
       "499983          205            2     221          13          129  ...   \n",
       "499984          194            3     370           5          124  ...   \n",
       "499985          201            3      10          34           79  ...   \n",
       "499986          295            3     114           4          233  ...   \n",
       "499987          171            3      61          28            3  ...   \n",
       "499988          144            3      16          31           61  ...   \n",
       "499989          630            3     395          37          165  ...   \n",
       "499990          190            3      18          37          246  ...   \n",
       "499991          151            1      72          41          296  ...   \n",
       "499992          310            2     156          47          313  ...   \n",
       "499993          200            3     199          26           45  ...   \n",
       "499994           31            3      -1          37          156  ...   \n",
       "499995          150            1      -1          -1           -1  ...   \n",
       "499996          140            3      74          47           98  ...   \n",
       "499997           22            3      -1          46          131  ...   \n",
       "499998           35            3     367           6          149  ...   \n",
       "499999           71            3     408          28          133  ...   \n",
       "\n",
       "        applicant_income  population  minority_population_pct  \\\n",
       "0                   24.0      6203.0                   44.230   \n",
       "1                   57.0      5774.0                   15.905   \n",
       "2                   67.0      6094.0                   61.270   \n",
       "3                  105.0      6667.0                    6.246   \n",
       "4                   71.0      6732.0                  100.000   \n",
       "5                   51.0      6078.0                    4.821   \n",
       "6                  104.0      6068.0                   26.427   \n",
       "7                   55.0      6030.0                   78.153   \n",
       "8                  244.0      5151.0                   88.156   \n",
       "9                   86.0      7916.0                   24.893   \n",
       "10                  78.0      5931.0                   39.082   \n",
       "11                  63.0      6917.0                    6.922   \n",
       "12                   NaN      9958.0                   41.404   \n",
       "13                  90.0      4076.0                   37.949   \n",
       "14                 172.0      2165.0                    3.916   \n",
       "15                  53.0      5841.0                   63.554   \n",
       "16                   NaN      6420.0                   29.818   \n",
       "17                   NaN      4193.0                   14.996   \n",
       "18                 111.0      3792.0                    8.760   \n",
       "19                  63.0      7316.0                   36.347   \n",
       "20                  41.0      6059.0                   96.448   \n",
       "21                  69.0      4625.0                   68.823   \n",
       "22                  11.0         NaN                      NaN   \n",
       "23                 217.0      9184.0                   12.486   \n",
       "24                  85.0      8585.0                   31.068   \n",
       "25                  99.0      4346.0                   16.489   \n",
       "26                  54.0         NaN                      NaN   \n",
       "27                  53.0      4885.0                    2.693   \n",
       "28                  52.0      6184.0                    7.898   \n",
       "29                  75.0      3327.0                   93.204   \n",
       "...                  ...         ...                      ...   \n",
       "499970               NaN      5033.0                   58.914   \n",
       "499971              78.0     10114.0                   18.839   \n",
       "499972              20.0      3688.0                   94.824   \n",
       "499973              48.0      3478.0                   35.680   \n",
       "499974              50.0      5085.0                   35.779   \n",
       "499975               NaN      4499.0                   12.106   \n",
       "499976              69.0      9737.0                    8.471   \n",
       "499977              86.0      5317.0                   24.059   \n",
       "499978              54.0      2735.0                    2.910   \n",
       "499979               NaN      7088.0                   70.859   \n",
       "499980              76.0      3326.0                   38.107   \n",
       "499981              72.0      4867.0                    7.183   \n",
       "499982              39.0      5727.0                   30.918   \n",
       "499983             164.0      3331.0                   20.892   \n",
       "499984              33.0      7859.0                   44.802   \n",
       "499985              60.0      1834.0                   14.957   \n",
       "499986             101.0      3426.0                   16.397   \n",
       "499987              73.0      7295.0                   19.892   \n",
       "499988              77.0      9221.0                   11.097   \n",
       "499989             202.0      4868.0                   40.355   \n",
       "499990              79.0      4671.0                   85.037   \n",
       "499991              27.0      8161.0                   23.668   \n",
       "499992             138.0      4858.0                   10.196   \n",
       "499993             127.0      9535.0                    5.584   \n",
       "499994             147.0      3765.0                   13.789   \n",
       "499995              87.0         NaN                      NaN   \n",
       "499996             216.0      3452.0                    6.887   \n",
       "499997              35.0      2925.0                   36.046   \n",
       "499998              41.0      3442.0                   98.879   \n",
       "499999              85.0      3856.0                   12.095   \n",
       "\n",
       "        ffiecmedian_family_income  tract_to_msa_md_income_pct  \\\n",
       "0                         60588.0                      50.933   \n",
       "1                         54821.0                     100.000   \n",
       "2                         67719.0                     100.000   \n",
       "3                         78439.0                     100.000   \n",
       "4                         63075.0                      82.200   \n",
       "5                         82745.0                      96.550   \n",
       "6                         65282.0                      81.068   \n",
       "7                        108353.0                      59.439   \n",
       "8                         63414.0                      62.813   \n",
       "9                         65927.0                     100.000   \n",
       "10                       109743.0                     100.000   \n",
       "11                        73503.0                     100.000   \n",
       "12                        60081.0                     100.000   \n",
       "13                        76369.0                      87.937   \n",
       "14                        63560.0                     100.000   \n",
       "15                        69265.0                      82.998   \n",
       "16                        68065.0                     100.000   \n",
       "17                        57774.0                      74.411   \n",
       "18                        66533.0                     100.000   \n",
       "19                        69795.0                      92.747   \n",
       "20                        50655.0                      72.441   \n",
       "21                        48604.0                     100.000   \n",
       "22                            NaN                         NaN   \n",
       "23                       104550.0                     100.000   \n",
       "24                        85202.0                     100.000   \n",
       "25                        70745.0                     100.000   \n",
       "26                            NaN                         NaN   \n",
       "27                        62795.0                     100.000   \n",
       "28                        57325.0                      96.479   \n",
       "29                        62340.0                      62.557   \n",
       "...                           ...                         ...   \n",
       "499970                    74980.0                     100.000   \n",
       "499971                    73998.0                     100.000   \n",
       "499972                    64139.0                      57.455   \n",
       "499973                    53969.0                     100.000   \n",
       "499974                    78743.0                     100.000   \n",
       "499975                    67680.0                     100.000   \n",
       "499976                    62190.0                     100.000   \n",
       "499977                    60028.0                     100.000   \n",
       "499978                    61429.0                     100.000   \n",
       "499979                    50269.0                     100.000   \n",
       "499980                    65762.0                      94.810   \n",
       "499981                    57188.0                     100.000   \n",
       "499982                    54546.0                      85.171   \n",
       "499983                    80134.0                      69.377   \n",
       "499984                    60826.0                     100.000   \n",
       "499985                    90502.0                      88.586   \n",
       "499986                    68947.0                     100.000   \n",
       "499987                    60113.0                     100.000   \n",
       "499988                    64458.0                     100.000   \n",
       "499989                    90302.0                     100.000   \n",
       "499990                    61141.0                     100.000   \n",
       "499991                    59031.0                      86.953   \n",
       "499992                    73821.0                     100.000   \n",
       "499993                    65139.0                     100.000   \n",
       "499994                    58049.0                     100.000   \n",
       "499995                        NaN                         NaN   \n",
       "499996                    86307.0                     100.000   \n",
       "499997                    47826.0                     100.000   \n",
       "499998                    60327.0                      62.803   \n",
       "499999                    45033.0                     100.000   \n",
       "\n",
       "        number_of_owner-occupied_units  number_of_1_to_4_family_units  lender  \\\n",
       "0                                716.0                         2642.0    4536   \n",
       "1                               1622.0                         2108.0    2458   \n",
       "2                                760.0                         1048.0    5710   \n",
       "3                               2025.0                         2299.0    5888   \n",
       "4                               1464.0                         1847.0     289   \n",
       "5                               1827.0                         2340.0     964   \n",
       "6                               1863.0                         2560.0    5488   \n",
       "7                                969.0                         1601.0    2442   \n",
       "8                                411.0                          481.0    2118   \n",
       "9                               1861.0                         2123.0    3507   \n",
       "10                              1315.0                         1382.0     838   \n",
       "11                              1858.0                         2050.0    5154   \n",
       "12                              2220.0                         2727.0    4664   \n",
       "13                               835.0                         1696.0    5154   \n",
       "14                               839.0                         1263.0    5473   \n",
       "15                              1196.0                         1632.0    5316   \n",
       "16                              1553.0                         2001.0    3354   \n",
       "17                              1247.0                         1998.0    2566   \n",
       "18                              1309.0                         1354.0    4701   \n",
       "19                              1498.0                         1903.0    1462   \n",
       "20                              1325.0                         1551.0    5726   \n",
       "21                               865.0                         1083.0    4145   \n",
       "22                                 NaN                            NaN    4347   \n",
       "23                              2545.0                         2425.0    2054   \n",
       "24                              2159.0                         2550.0    3817   \n",
       "25                              1198.0                         1349.0    2458   \n",
       "26                                 NaN                            NaN    2839   \n",
       "27                              1462.0                         1955.0    4419   \n",
       "28                              1970.0                         2834.0    6240   \n",
       "29                               382.0                          760.0    2240   \n",
       "...                                ...                            ...     ...   \n",
       "499970                          1564.0                         1746.0    2108   \n",
       "499971                          3030.0                         3624.0    2816   \n",
       "499972                           659.0                         1567.0    3581   \n",
       "499973                           460.0                          600.0    1919   \n",
       "499974                          1313.0                         1612.0    1928   \n",
       "499975                          1407.0                         1504.0     169   \n",
       "499976                          3044.0                         3612.0    2458   \n",
       "499977                          1231.0                         1871.0    2775   \n",
       "499978                          1095.0                         2267.0    3354   \n",
       "499979                          2103.0                         2377.0    2458   \n",
       "499980                           512.0                          599.0    4449   \n",
       "499981                          1576.0                         1978.0    6300   \n",
       "499982                          1543.0                         2526.0    4129   \n",
       "499983                           657.0                         1474.0    6093   \n",
       "499984                          2754.0                         2969.0    4683   \n",
       "499985                           738.0                          787.0     986   \n",
       "499986                          1023.0                         1251.0    6044   \n",
       "499987                          2308.0                         3175.0    3968   \n",
       "499988                          2777.0                         3172.0    5710   \n",
       "499989                          1251.0                         1410.0    5710   \n",
       "499990                           720.0                          915.0    2118   \n",
       "499991                          2224.0                         2919.0    6451   \n",
       "499992                          1511.0                         1803.0    5907   \n",
       "499993                          2913.0                         3289.0    3354   \n",
       "499994                          1154.0                         1955.0    5391   \n",
       "499995                             NaN                            NaN     969   \n",
       "499996                          1423.0                         1944.0    5359   \n",
       "499997                          1062.0                         1762.0    2318   \n",
       "499998                           618.0                         1297.0    5339   \n",
       "499999                          1525.0                         2750.0    5710   \n",
       "\n",
       "        co_applicant  accepted  \n",
       "0              False         1  \n",
       "1              False         0  \n",
       "2              False         1  \n",
       "3               True         1  \n",
       "4              False         1  \n",
       "5              False         1  \n",
       "6              False         1  \n",
       "7               True         1  \n",
       "8               True         1  \n",
       "9              False         0  \n",
       "10             False         1  \n",
       "11             False         1  \n",
       "12             False         1  \n",
       "13             False         1  \n",
       "14              True         0  \n",
       "15             False         1  \n",
       "16             False         0  \n",
       "17              True         1  \n",
       "18              True         1  \n",
       "19              True         0  \n",
       "20              True         0  \n",
       "21              True         1  \n",
       "22             False         0  \n",
       "23              True         1  \n",
       "24              True         0  \n",
       "25             False         0  \n",
       "26             False         0  \n",
       "27              True         1  \n",
       "28             False         1  \n",
       "29             False         1  \n",
       "...              ...       ...  \n",
       "499970          True         1  \n",
       "499971          True         1  \n",
       "499972         False         0  \n",
       "499973         False         0  \n",
       "499974         False         0  \n",
       "499975          True         1  \n",
       "499976          True         0  \n",
       "499977         False         0  \n",
       "499978         False         0  \n",
       "499979         False         0  \n",
       "499980          True         1  \n",
       "499981          True         1  \n",
       "499982         False         1  \n",
       "499983         False         1  \n",
       "499984         False         1  \n",
       "499985         False         1  \n",
       "499986         False         0  \n",
       "499987         False         1  \n",
       "499988         False         0  \n",
       "499989          True         1  \n",
       "499990         False         0  \n",
       "499991         False         0  \n",
       "499992         False         1  \n",
       "499993          True         0  \n",
       "499994         False         0  \n",
       "499995          True         0  \n",
       "499996          True         0  \n",
       "499997         False         0  \n",
       "499998         False         0  \n",
       "499999          True         0  \n",
       "\n",
       "[500000 rows x 23 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Below- RECHECK missing values for county_code & msa_md in dataset.\n",
    "\n",
    "LoanTrainG"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(500000, 23)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Fill the remaining numerical missing values with median values for each column.\n",
    "\n",
    "LoanTrainG = LoanTrainG.fillna(median)\n",
    "LoanTrainG.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "applicant_income 0\n",
      "population 0\n",
      "minority_population_pct 0\n",
      "ffiecmedian_family_income 0\n",
      "tract_to_msa_md_income_pct 0\n",
      "number_of_owner-occupied_units 0\n",
      "number_of_1_to_4_family_units 0\n"
     ]
    }
   ],
   "source": [
    "# RECHECK- Now double check and look for missing values again in training dataset.\n",
    "\n",
    "for col in LoanTrainG.columns:\n",
    "    if LoanTrainG[col].dtype == float:\n",
    "        count = 0\n",
    "        count = [count + 1 for x in LoanTrainG[col] if pd.isna(x)]\n",
    "        print(col + ' ' + str(sum(count)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "15.18\n",
      "3.8264\n",
      "4.0932\n"
     ]
    }
   ],
   "source": [
    "# RECHECK - Find out how many values are missing as '-1' w/in this integer column feature.\n",
    "    \n",
    "missing_msa_md = LoanTrainG.msa_md[LoanTrainG.msa_md == -1].count()\n",
    "missing_state_code = LoanTrainG.state_code[LoanTrainG.state_code == -1].count()\n",
    "missing_county_code = LoanTrainG.county_code[LoanTrainG.county_code == -1].count()\n",
    "\n",
    "pctmiss_msa_ma = ((missing_msa_md)/500000)*100\n",
    "pctmiss_state_code = ((missing_state_code)/500000)*100\n",
    "pctmiss_county_code = ((missing_county_code)/500000)*100\n",
    "\n",
    "print(pctmiss_msa_ma)\n",
    "print(pctmiss_state_code)\n",
    "print(pctmiss_county_code)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>row_id</th>\n",
       "      <th>loan_type</th>\n",
       "      <th>property_type</th>\n",
       "      <th>loan_purpose</th>\n",
       "      <th>occupancy</th>\n",
       "      <th>loan_amount</th>\n",
       "      <th>preapproval</th>\n",
       "      <th>msa_md</th>\n",
       "      <th>state_code</th>\n",
       "      <th>county_code</th>\n",
       "      <th>...</th>\n",
       "      <th>applicant_sex</th>\n",
       "      <th>applicant_income</th>\n",
       "      <th>population</th>\n",
       "      <th>minority_population_pct</th>\n",
       "      <th>ffiecmedian_family_income</th>\n",
       "      <th>tract_to_msa_md_income_pct</th>\n",
       "      <th>number_of_owner-occupied_units</th>\n",
       "      <th>number_of_1_to_4_family_units</th>\n",
       "      <th>lender</th>\n",
       "      <th>accepted</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>500000.000000</td>\n",
       "      <td>500000.000000</td>\n",
       "      <td>500000.000000</td>\n",
       "      <td>500000.000000</td>\n",
       "      <td>500000.000000</td>\n",
       "      <td>500000.000000</td>\n",
       "      <td>500000.000000</td>\n",
       "      <td>500000.000000</td>\n",
       "      <td>500000.000000</td>\n",
       "      <td>500000.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>500000.000000</td>\n",
       "      <td>500000.000000</td>\n",
       "      <td>500000.000000</td>\n",
       "      <td>500000.000000</td>\n",
       "      <td>500000.000000</td>\n",
       "      <td>500000.000000</td>\n",
       "      <td>500000.000000</td>\n",
       "      <td>500000.000000</td>\n",
       "      <td>500000.000000</td>\n",
       "      <td>500000.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>249999.500000</td>\n",
       "      <td>1.366276</td>\n",
       "      <td>1.047650</td>\n",
       "      <td>2.066810</td>\n",
       "      <td>1.109590</td>\n",
       "      <td>221.753158</td>\n",
       "      <td>2.764722</td>\n",
       "      <td>182.085868</td>\n",
       "      <td>23.726924</td>\n",
       "      <td>144.542062</td>\n",
       "      <td>...</td>\n",
       "      <td>1.462374</td>\n",
       "      <td>100.121312</td>\n",
       "      <td>5396.982356</td>\n",
       "      <td>31.225669</td>\n",
       "      <td>69158.876302</td>\n",
       "      <td>92.200385</td>\n",
       "      <td>1423.172866</td>\n",
       "      <td>1880.147458</td>\n",
       "      <td>3720.121344</td>\n",
       "      <td>0.500228</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>144337.711635</td>\n",
       "      <td>0.690555</td>\n",
       "      <td>0.231404</td>\n",
       "      <td>0.948371</td>\n",
       "      <td>0.326092</td>\n",
       "      <td>590.641648</td>\n",
       "      <td>0.543061</td>\n",
       "      <td>138.325897</td>\n",
       "      <td>15.982768</td>\n",
       "      <td>100.243612</td>\n",
       "      <td>...</td>\n",
       "      <td>0.677685</td>\n",
       "      <td>147.474440</td>\n",
       "      <td>2667.723303</td>\n",
       "      <td>25.798784</td>\n",
       "      <td>14478.232811</td>\n",
       "      <td>13.990187</td>\n",
       "      <td>721.027517</td>\n",
       "      <td>893.717989</td>\n",
       "      <td>1838.313175</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>14.000000</td>\n",
       "      <td>0.534000</td>\n",
       "      <td>17858.000000</td>\n",
       "      <td>3.981000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>124999.750000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>93.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>25.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>57.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>49.000000</td>\n",
       "      <td>3805.000000</td>\n",
       "      <td>11.191000</td>\n",
       "      <td>60071.000000</td>\n",
       "      <td>89.145000</td>\n",
       "      <td>963.000000</td>\n",
       "      <td>1323.000000</td>\n",
       "      <td>2442.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>249999.500000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>162.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>194.000000</td>\n",
       "      <td>26.000000</td>\n",
       "      <td>131.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>74.000000</td>\n",
       "      <td>4975.000000</td>\n",
       "      <td>22.901000</td>\n",
       "      <td>67526.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>1327.000000</td>\n",
       "      <td>1753.000000</td>\n",
       "      <td>3731.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>374999.250000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>266.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>314.000000</td>\n",
       "      <td>37.000000</td>\n",
       "      <td>246.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>112.000000</td>\n",
       "      <td>6379.000000</td>\n",
       "      <td>44.486000</td>\n",
       "      <td>74714.250000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>1754.000000</td>\n",
       "      <td>2275.000000</td>\n",
       "      <td>5436.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>499999.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>100878.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>408.000000</td>\n",
       "      <td>52.000000</td>\n",
       "      <td>324.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>10139.000000</td>\n",
       "      <td>37097.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>125248.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>8771.000000</td>\n",
       "      <td>13623.000000</td>\n",
       "      <td>6508.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              row_id      loan_type  property_type   loan_purpose  \\\n",
       "count  500000.000000  500000.000000  500000.000000  500000.000000   \n",
       "mean   249999.500000       1.366276       1.047650       2.066810   \n",
       "std    144337.711635       0.690555       0.231404       0.948371   \n",
       "min         0.000000       1.000000       1.000000       1.000000   \n",
       "25%    124999.750000       1.000000       1.000000       1.000000   \n",
       "50%    249999.500000       1.000000       1.000000       2.000000   \n",
       "75%    374999.250000       2.000000       1.000000       3.000000   \n",
       "max    499999.000000       4.000000       3.000000       3.000000   \n",
       "\n",
       "           occupancy    loan_amount    preapproval         msa_md  \\\n",
       "count  500000.000000  500000.000000  500000.000000  500000.000000   \n",
       "mean        1.109590     221.753158       2.764722     182.085868   \n",
       "std         0.326092     590.641648       0.543061     138.325897   \n",
       "min         1.000000       1.000000       1.000000      -1.000000   \n",
       "25%         1.000000      93.000000       3.000000      25.000000   \n",
       "50%         1.000000     162.000000       3.000000     194.000000   \n",
       "75%         1.000000     266.000000       3.000000     314.000000   \n",
       "max         3.000000  100878.000000       3.000000     408.000000   \n",
       "\n",
       "          state_code    county_code  ...  applicant_sex  applicant_income  \\\n",
       "count  500000.000000  500000.000000  ...  500000.000000     500000.000000   \n",
       "mean       23.726924     144.542062  ...       1.462374        100.121312   \n",
       "std        15.982768     100.243612  ...       0.677685        147.474440   \n",
       "min        -1.000000      -1.000000  ...       1.000000          1.000000   \n",
       "25%         6.000000      57.000000  ...       1.000000         49.000000   \n",
       "50%        26.000000     131.000000  ...       1.000000         74.000000   \n",
       "75%        37.000000     246.000000  ...       2.000000        112.000000   \n",
       "max        52.000000     324.000000  ...       4.000000      10139.000000   \n",
       "\n",
       "          population  minority_population_pct  ffiecmedian_family_income  \\\n",
       "count  500000.000000            500000.000000              500000.000000   \n",
       "mean     5396.982356                31.225669               69158.876302   \n",
       "std      2667.723303                25.798784               14478.232811   \n",
       "min        14.000000                 0.534000               17858.000000   \n",
       "25%      3805.000000                11.191000               60071.000000   \n",
       "50%      4975.000000                22.901000               67526.000000   \n",
       "75%      6379.000000                44.486000               74714.250000   \n",
       "max     37097.000000               100.000000              125248.000000   \n",
       "\n",
       "       tract_to_msa_md_income_pct  number_of_owner-occupied_units  \\\n",
       "count               500000.000000                   500000.000000   \n",
       "mean                    92.200385                     1423.172866   \n",
       "std                     13.990187                      721.027517   \n",
       "min                      3.981000                        4.000000   \n",
       "25%                     89.145000                      963.000000   \n",
       "50%                    100.000000                     1327.000000   \n",
       "75%                    100.000000                     1754.000000   \n",
       "max                    100.000000                     8771.000000   \n",
       "\n",
       "       number_of_1_to_4_family_units         lender       accepted  \n",
       "count                  500000.000000  500000.000000  500000.000000  \n",
       "mean                     1880.147458    3720.121344       0.500228  \n",
       "std                       893.717989    1838.313175       0.500000  \n",
       "min                         1.000000       0.000000       0.000000  \n",
       "25%                      1323.000000    2442.000000       0.000000  \n",
       "50%                      1753.000000    3731.000000       1.000000  \n",
       "75%                      2275.000000    5436.000000       1.000000  \n",
       "max                     13623.000000    6508.000000       1.000000  \n",
       "\n",
       "[8 rows x 22 columns]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Above - Will keep the missing interger values as '-1' within the training dataset.\n",
    "\n",
    "#Below - Recalculate the summary statistcs for numeric values with columns removed and missing values filled with median.\n",
    "\n",
    "LoanTrainG.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Save this cleaned training data to a csv\n",
    "\n",
    "LoanTrainG.to_csv('LoanTrain_Clean_2019-04-23d.csv', index = False, header = True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}