Prediction_Feature4.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Inspirational Notebooks\n",
    "### Generating new festures according to these notebooks"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* https://www.kaggle.com/nuhsikander/lgbm-new-features-corrected\n",
    "* https://www.kaggle.com/rteja1113/lightgbm-with-count-features\n",
    "* https://www.kaggle.com/aharless/swetha-s-xgboost-revised\n",
    "* https://www.kaggle.com/bk0000/non-blending-lightgbm-model-lb-0-977"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/PP/Documents/7250/Python/anaconda/lib/python3.5/importlib/_bootstrap.py:222: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88\n",
      "  return f(*args, **kwds)\n",
      "/Users/PP/Documents/7250/Python/anaconda/lib/python3.5/importlib/_bootstrap.py:222: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88\n",
      "  return f(*args, **kwds)\n",
      "/Users/PP/Documents/7250/Python/anaconda/lib/python3.5/importlib/_bootstrap.py:222: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88\n",
      "  return f(*args, **kwds)\n",
      "/Users/PP/Documents/7250/Python/anaconda/lib/python3.5/importlib/_bootstrap.py:222: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88\n",
      "  return f(*args, **kwds)\n",
      "/Users/PP/Documents/7250/Python/anaconda/lib/python3.5/site-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline \n",
    "import matplotlib.pyplot as plt \n",
    "\n",
    "import math\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "from pandas import read_csv\n",
    "import sklearn\n",
    "from sklearn import preprocessing\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.cross_validation import train_test_split\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "plt.rc(\"font\", size=14)\n",
    "sns.set(style=\"white\")\n",
    "sns.set(style=\"whitegrid\", color_codes=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import keras\n",
    "from keras.datasets import mnist\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.optimizers import SGD\n",
    "from keras.optimizers import RMSprop\n",
    "from keras.optimizers import Adagrad\n",
    "from keras.optimizers import Adadelta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 200000 entries, 0 to 199999\n",
      "Data columns (total 57 columns):\n",
      "Unnamed: 0                            200000 non-null int64\n",
      "ip                                    200000 non-null int64\n",
      "app                                   200000 non-null int64\n",
      "device                                200000 non-null int64\n",
      "os                                    200000 non-null int64\n",
      "channel                               200000 non-null int64\n",
      "click_time                            200000 non-null object\n",
      "attributed_time                       348 non-null object\n",
      "is_attributed                         200000 non-null int64\n",
      "day                                   200000 non-null int64\n",
      "hour                                  200000 non-null int64\n",
      "minute                                200000 non-null int64\n",
      "second                                200000 non-null int64\n",
      "ip_confRate                           200000 non-null float64\n",
      "app_confRate                          200000 non-null float64\n",
      "device_confRate                       200000 non-null float64\n",
      "os_confRate                           200000 non-null float64\n",
      "channel_confRate                      200000 non-null float64\n",
      "app_channel_confRate                  200000 non-null float64\n",
      "app_os_confRate                       200000 non-null float64\n",
      "app_device_confRate                   200000 non-null float64\n",
      "channel_os_confRate                   200000 non-null float64\n",
      "channel_device_confRate               200000 non-null float64\n",
      "os_device_confRate                    200000 non-null float64\n",
      "ip_app_channel_var_day                134217 non-null float64\n",
      "ip_app_os_var_hour                    139465 non-null float64\n",
      "ip_day_channel_var_hour_x             151335 non-null float64\n",
      "ip_day_hour_count_channel             200000 non-null int64\n",
      "ip_app_count_channel                  200000 non-null int64\n",
      "ip_app_os_count_channel               200000 non-null int64\n",
      "ip_app_day_hour_count_channel         200000 non-null int64\n",
      "ip_app_channel_mean_hour              200000 non-null float64\n",
      "app_AvgViewPerDistinct_ip             200000 non-null float64\n",
      "app_count_channel                     200000 non-null int64\n",
      "channel_count_app                     200000 non-null int64\n",
      "ip_nunique_channel                    200000 non-null int64\n",
      "ip_nunique_app                        200000 non-null int64\n",
      "ip_day_nunique_hour                   200000 non-null int64\n",
      "ip_app_nunique_os                     200000 non-null int64\n",
      "ip_nunique_device                     200000 non-null int64\n",
      "app_nunique_channel                   200000 non-null int64\n",
      "ip_device_os_nunique_app              200000 non-null int64\n",
      "ip_device_os_cumcount_app             200000 non-null int64\n",
      "ip_cumcount_app                       200000 non-null int64\n",
      "ip_cumcount_os                        200000 non-null int64\n",
      "ip_day_channel_var_hour_y             151335 non-null float64\n",
      "ip_nextClick                          197456 non-null float64\n",
      "ip_app_nextClick                      163726 non-null float64\n",
      "ip_channel_nextClick                  138039 non-null float64\n",
      "ip_os_nextClick                       182531 non-null float64\n",
      "ip_app_device_os_channel_nextClick    86990 non-null float64\n",
      "ip_os_device_nextClick                181508 non-null float64\n",
      "ip_os_device_app_nextClick            120449 non-null float64\n",
      "prev_identical_clicks                 200000 non-null int64\n",
      "future_identical_clicks               200000 non-null int64\n",
      "prev_app_clicks                       200000 non-null int64\n",
      "future_app_clicks                     200000 non-null int64\n",
      "dtypes: float64(24), int64(31), object(2)\n",
      "memory usage: 87.0+ MB\n"
     ]
    }
   ],
   "source": [
    "# df_train= pd.read_csv('data/train.csv',nrows=200000, parse_dates=['click_time'])\n",
    "df_train = pd.read_csv('data/train_new_cols.csv',nrows=200000) #train data subset, original too large\n",
    "df_train.dropna()\n",
    "df_train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Unnamed: 0', 'ip', 'app', 'device', 'os', 'channel', 'click_time',\n",
       "       'attributed_time', 'is_attributed', 'day', 'hour', 'minute',\n",
       "       'second', 'ip_confRate', 'app_confRate', 'device_confRate',\n",
       "       'os_confRate', 'channel_confRate', 'app_channel_confRate',\n",
       "       'app_os_confRate', 'app_device_confRate', 'channel_os_confRate',\n",
       "       'channel_device_confRate', 'os_device_confRate',\n",
       "       'ip_app_channel_var_day', 'ip_app_os_var_hour',\n",
       "       'ip_day_channel_var_hour_x', 'ip_day_hour_count_channel',\n",
       "       'ip_app_count_channel', 'ip_app_os_count_channel',\n",
       "       'ip_app_day_hour_count_channel', 'ip_app_channel_mean_hour',\n",
       "       'app_AvgViewPerDistinct_ip', 'app_count_channel',\n",
       "       'channel_count_app', 'ip_nunique_channel', 'ip_nunique_app',\n",
       "       'ip_day_nunique_hour', 'ip_app_nunique_os', 'ip_nunique_device',\n",
       "       'app_nunique_channel', 'ip_device_os_nunique_app',\n",
       "       'ip_device_os_cumcount_app', 'ip_cumcount_app', 'ip_cumcount_os',\n",
       "       'ip_day_channel_var_hour_y', 'ip_nextClick', 'ip_app_nextClick',\n",
       "       'ip_channel_nextClick', 'ip_os_nextClick',\n",
       "       'ip_app_device_os_channel_nextClick', 'ip_os_device_nextClick',\n",
       "       'ip_os_device_app_nextClick', 'prev_identical_clicks',\n",
       "       'future_identical_clicks', 'prev_app_clicks', 'future_app_clicks'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.columns.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cols = ['ip', 'app', 'device', 'os', 'channel', 'click_time',\n",
    "       'attributed_time', 'is_attributed', 'ip_nextClick', 'ip_app_nextClick',\n",
    "       'ip_channel_nextClick', 'ip_os_nextClick',\n",
    "       'ip_app_device_os_channel_nextClick', 'ip_os_device_nextClick',\n",
    "       'ip_os_device_app_nextClick',]\n",
    "df_train = df_train.loc[:,cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ip</th>\n",
       "      <th>app</th>\n",
       "      <th>device</th>\n",
       "      <th>os</th>\n",
       "      <th>channel</th>\n",
       "      <th>click_time</th>\n",
       "      <th>attributed_time</th>\n",
       "      <th>is_attributed</th>\n",
       "      <th>ip_nextClick</th>\n",
       "      <th>ip_app_nextClick</th>\n",
       "      <th>ip_channel_nextClick</th>\n",
       "      <th>ip_os_nextClick</th>\n",
       "      <th>ip_app_device_os_channel_nextClick</th>\n",
       "      <th>ip_os_device_nextClick</th>\n",
       "      <th>ip_os_device_app_nextClick</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>83230</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>379</td>\n",
       "      <td>2017-11-06 14:32:21</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>5290.0</td>\n",
       "      <td>5340.0</td>\n",
       "      <td>5444.0</td>\n",
       "      <td>5307.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5307.0</td>\n",
       "      <td>5340.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>17357</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>19</td>\n",
       "      <td>379</td>\n",
       "      <td>2017-11-06 14:33:34</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>5177.0</td>\n",
       "      <td>5177.0</td>\n",
       "      <td>5177.0</td>\n",
       "      <td>5239.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5239.0</td>\n",
       "      <td>5547.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>35810</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>379</td>\n",
       "      <td>2017-11-06 14:34:12</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>5175.0</td>\n",
       "      <td>5175.0</td>\n",
       "      <td>6005.0</td>\n",
       "      <td>5205.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5205.0</td>\n",
       "      <td>5925.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>45745</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>478</td>\n",
       "      <td>2017-11-06 14:34:52</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>5108.0</td>\n",
       "      <td>5110.0</td>\n",
       "      <td>5137.0</td>\n",
       "      <td>5108.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5108.0</td>\n",
       "      <td>5110.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>161007</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>379</td>\n",
       "      <td>2017-11-06 14:35:08</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>18787</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>16</td>\n",
       "      <td>379</td>\n",
       "      <td>2017-11-06 14:36:26</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>4531.0</td>\n",
       "      <td>4531.0</td>\n",
       "      <td>4531.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>103022</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>23</td>\n",
       "      <td>379</td>\n",
       "      <td>2017-11-06 14:37:44</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>5005.0</td>\n",
       "      <td>5020.0</td>\n",
       "      <td>5005.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>114221</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>19</td>\n",
       "      <td>379</td>\n",
       "      <td>2017-11-06 14:37:59</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>4945.0</td>\n",
       "      <td>4955.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5006.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5006.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>165970</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>379</td>\n",
       "      <td>2017-11-06 14:38:10</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>4964.0</td>\n",
       "      <td>5503.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5052.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5052.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>74544</td>\n",
       "      <td>64</td>\n",
       "      <td>1</td>\n",
       "      <td>22</td>\n",
       "      <td>459</td>\n",
       "      <td>2017-11-06 14:38:23</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>5070.0</td>\n",
       "      <td>5111.0</td>\n",
       "      <td>5111.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       ip  app  device  os  channel           click_time attributed_time  \\\n",
       "0   83230    3       1  13      379  2017-11-06 14:32:21             NaN   \n",
       "1   17357    3       1  19      379  2017-11-06 14:33:34             NaN   \n",
       "2   35810    3       1  13      379  2017-11-06 14:34:12             NaN   \n",
       "3   45745   14       1  13      478  2017-11-06 14:34:52             NaN   \n",
       "4  161007    3       1  13      379  2017-11-06 14:35:08             NaN   \n",
       "5   18787    3       1  16      379  2017-11-06 14:36:26             NaN   \n",
       "6  103022    3       1  23      379  2017-11-06 14:37:44             NaN   \n",
       "7  114221    3       1  19      379  2017-11-06 14:37:59             NaN   \n",
       "8  165970    3       1  13      379  2017-11-06 14:38:10             NaN   \n",
       "9   74544   64       1  22      459  2017-11-06 14:38:23             NaN   \n",
       "\n",
       "   is_attributed  ip_nextClick  ip_app_nextClick  ip_channel_nextClick  \\\n",
       "0              0        5290.0            5340.0                5444.0   \n",
       "1              0        5177.0            5177.0                5177.0   \n",
       "2              0        5175.0            5175.0                6005.0   \n",
       "3              0        5108.0            5110.0                5137.0   \n",
       "4              0           NaN               NaN                   NaN   \n",
       "5              0        4531.0            4531.0                4531.0   \n",
       "6              0        5005.0            5020.0                5005.0   \n",
       "7              0        4945.0            4955.0                   NaN   \n",
       "8              0        4964.0            5503.0                   NaN   \n",
       "9              0        5070.0            5111.0                5111.0   \n",
       "\n",
       "   ip_os_nextClick  ip_app_device_os_channel_nextClick  \\\n",
       "0           5307.0                                 NaN   \n",
       "1           5239.0                                 NaN   \n",
       "2           5205.0                                 NaN   \n",
       "3           5108.0                                 NaN   \n",
       "4              NaN                                 NaN   \n",
       "5              NaN                                 NaN   \n",
       "6              NaN                                 NaN   \n",
       "7           5006.0                                 NaN   \n",
       "8           5052.0                                 NaN   \n",
       "9              NaN                                 NaN   \n",
       "\n",
       "   ip_os_device_nextClick  ip_os_device_app_nextClick  \n",
       "0                  5307.0                      5340.0  \n",
       "1                  5239.0                      5547.0  \n",
       "2                  5205.0                      5925.0  \n",
       "3                  5108.0                      5110.0  \n",
       "4                     NaN                         NaN  \n",
       "5                     NaN                         NaN  \n",
       "6                     NaN                         NaN  \n",
       "7                  5006.0                         NaN  \n",
       "8                  5052.0                         NaN  \n",
       "9                     NaN                         NaN  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    199652\n",
       "1       348\n",
       "Name: is_attributed, dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train['is_attributed'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZMAAAEFCAYAAAAmIwo/AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAHI1JREFUeJzt3XtsVGX+x/H3dFoQOq21EggNFItC\nlLvtCO5auqvYDGxE5FpgLa6oCMvFuguCFVr4tVzMsiWx3IvZkCrLRVgl0XihWWxasMhowRZhWRcq\nt0URlXYQ2s6c3x8bJ9u11NGnM4X280pIOg/fc/o9zcl85pxnzjk2y7IsREREDIS1dAMiInLjU5iI\niIgxhYmIiBhTmIiIiDGFiYiIGAtv6QZagtvtbukWRERuSElJSY2Ot8kwgWv/QUREpHFNfRDXaS4R\nETGmMBEREWMKExERMaYwERERYwoTERExpjARERFjQftqcF1dHZmZmZw5c4ba2lpmzJjBHXfcwYIF\nC7DZbPTq1Yvs7GzCwsJYvXo1e/fuJTw8nMzMTAYMGEBVVZVxrYiIhEbQ3nF3795NTEwMW7ZsYdOm\nTeTk5LB8+XIyMjLYsmULlmVRVFREZWUlBw4cYMeOHeTl5bFkyRIA41oREQmdoB2ZDB8+HJfLBYBl\nWdjtdiorKxk8eDAAKSkplJaWkpCQQHJyMjabjbi4OLxeLxcvXjSuTU1NbbI/XQUvItJ8ghYmkZGR\nANTU1DBnzhwyMjJ48cUXsdls/v+vrq6mpqaGmJiYBstVV1djWZZR7Y8xvQL+4JzpRstL6+R8aX1L\ntyASNC12Bfy5c+eYMmUKo0aNYuTIkQ3mMTweD9HR0TgcDjweT4PxqKgo41oREQmdoIXJhQsXmDp1\nKvPmzWPcuHEA9OnTh7KyMgCKi4txOp0kJiZSUlKCz+fj7Nmz+Hw+YmNjjWtFRCR0gnaaa/369Vy6\ndIm1a9eydu1aAF544QVyc3PJy8ujZ8+euFwu7HY7TqeTtLQ0fD4fWVlZAMyfP59Fixb97FoREQkd\nm2VZVks3EWput1tzJhIUmjOR1qyp905djCEiIsYUJiIiYkxhIiIixhQmIiJiTGEiIiLGFCYiImJM\nYSIiIsYUJiIiYkxhIiIixhQmIiJiTGEiIiLGFCYiImJMYSIiIsYUJiIiYkxhIiIixhQmIiJiTGEi\nIiLGFCYiImIsaM+ABzh06BArV66ksLCQZ599lgsXLgBw5swZBg4cyKpVq5gxYwZff/01ERERtG/f\nnk2bNlFVVcWCBQuw2Wz06tWL7OxswsLCWL16NXv37iU8PJzMzEwGDBhwzVoREQmdoIVJQUEBu3fv\npkOHDgCsWrUKgG+//ZYpU6bw/PPPA1BVVcWbb76JzWbzL7t8+XIyMjIYMmQIWVlZFBUVERcXx4ED\nB9ixYwfnzp1j9uzZ7Ny5s9Ha1NTUYG2WiIg0ImhhEh8fT35+Ps8991yD8fz8fB599FE6d+7MhQsX\nuHTpEtOnT+fSpUtMmzaN+++/n8rKSgYPHgxASkoKpaWlJCQkkJycjM1mIy4uDq/Xy8WLFxutDSRM\n3G5382+0tHnar6StClqYuFwuTp8+3WDsq6++Yv/+/f6jkrq6OqZOncqUKVP49ttvmTRpEgMGDMCy\nLP+RSmRkJNXV1dTU1BATE+Nf1/fjjdUGIikpyWj7Dm4uMFpeWifT/UrketbUh6WQTi68/fbbPPTQ\nQ9jtdgA6derExIkTCQ8P59Zbb+Wuu+7ixIkTDeY8PB4P0dHROBwOPB5Pg/GoqKhGa0VEJLRCGib7\n9+8nJSXF/3rfvn0888wzwH+C4Pjx4/Ts2ZM+ffpQVlYGQHFxMU6nk8TEREpKSvD5fJw9exafz0ds\nbGyjtSIiElpB/TbX/zpx4gTdu3f3v/7Vr35FSUkJEyZMICwsjD/84Q/ExsYyf/58Fi1aRF5eHj17\n9sTlcmG323E6naSlpeHz+cjKygJotFZERELLZlmW1dJNhJrb7TafM5kzvZm6kdbE+dL6lm5BJGia\neu/UBRkiImJMYSIiIsYUJiIiYkxhIiIixhQmIiJiTGEiIiLGFCYiImJMYSIiIsYUJiIiYkxhIiIi\nxhQmIiJiTGEiIiLGFCYiImJMYSIiIsYUJiIiYkxhIiIixhQmIiJiTGEiIiLGghomhw4dIj09HYAj\nR44wdOhQ0tPTSU9P56233gJg9erVjBs3jokTJ3L48GEAqqqqmDRpEpMnTyY7Oxufz/eTa0VEJHTC\ng7XigoICdu/eTYcOHQCorKzk8ccfZ+rUqf6ayspKDhw4wI4dOzh37hyzZ89m586dLF++nIyMDIYM\nGUJWVhZFRUXExcUFXJuamhqszRIRkUYELUzi4+PJz8/nueeeA6CiooITJ05QVFREjx49yMzMxO12\nk5ycjM1mIy4uDq/Xy8WLF6msrGTw4MEApKSkUFpaSkJCQsC1gYSJ2+0O1qZLG6b9StqqoIWJy+Xi\n9OnT/tcDBgxg/Pjx9OvXj3Xr1rFmzRqioqKIiYnx10RGRlJdXY1lWdhstgZjNTU1AdcGIikpyWj7\nDm4uMFpeWifT/UrketbUh6WQTcCnpqbSr18//89HjhzB4XDg8Xj8NR6Ph6ioKMLCwhqMRUdH/6Ra\nEREJrZCFyRNPPOGfNN+/fz99+/YlMTGRkpISfD4fZ8+exefzERsbS58+fSgrKwOguLgYp9P5k2pF\nRCS0gnaa638tXryYnJwcIiIi6NSpEzk5OTgcDpxOJ2lpafh8PrKysgCYP38+ixYtIi8vj549e+Jy\nubDb7QHXiohIaNksy7JauolQc7vd5nMmc6Y3UzfSmjhfWt/SLYgETVPvnbpoUUREjClMRETEmMJE\nRESMKUxERMSYwkRERIwpTERExJjCREREjClMRETEmMJERESMKUxERMSYwkRERIwpTERExJjCRERE\njClMRETEmMJERESMKUxERMSYwkRERIwF9bG9hw4dYuXKlRQWFvLpp5+Sk5OD3W6nXbt2vPjii3Tq\n1Inc3Fw++ugjIiMjAVi7di11dXXMnTuXK1eu0LlzZ5YvX06HDh3Yvn07W7duJTw8nBkzZnD//fdz\n8eLFRmtFRCR0gnZkUlBQwMKFC7l69SoAS5cuZdGiRRQWFpKamkpBQQEAlZWVbNq0icLCQgoLC4mK\nimLt2rU89NBDbNmyhT59+rBt2za+/PJLCgsL2bp1Ky+//DJ5eXnU1tY2WisiIqEVtDCJj48nPz/f\n/zovL4+77roLAK/XS/v27fH5fFRVVZGVlcXEiRN57bXXgP88Z3jo0KEApKSksG/fPg4fPszdd99N\nu3btiIqKIj4+nqNHjzZaKyIioRW001wul4vTp0/7X3fu3BmAjz76iFdeeYVXX32Vy5cv8+ijj/L4\n44/j9XqZMmUK/fr1o6amhqioKAAiIyOprq5uMPb9eE1NTaO1gXC73c21qSJ+2q+krQrqnMn/euut\nt1i3bh0bN24kNjbWHyDfz3Hce++9HD16FIfDgcfj4aabbsLj8RAdHe0f+57H4yEqKqrR2kAkJSUZ\nbcvBzQVGy0vrZLpfiVzPmvqwFLJvc73xxhu88sorFBYW0r17dwBOnjzJpEmT8Hq91NXV8dFHH9G3\nb18SExN5//33ASguLiYpKYkBAwbgdru5evUq1dXVfPbZZ/Tu3bvRWhERCa2QHJl4vV6WLl1K165d\nmT17NgD33HMPc+bMYdSoUUyYMIGIiAhGjRpFr169mDFjBvPnz2f79u3ccsst/PnPf6Zjx46kp6cz\nefJkLMvi2WefpX379o3WiohIaNksy7JauolQc7vd5qe55kxvpm6kNXG+tL6lWxAJmqbeO3XRooiI\nGFOYiIiIMYWJiIgYU5iIiIgxhYmIiBgLKExycnJ+MDZ//vxmb0ZERG5MTV5n8sILL3Dq1CkqKio4\nfvy4f7y+vj7g25aIiEjr12SYzJgxgzNnzrB06VJmzZrlH7fb7dx+++1Bb05ERG4MTYZJt27d6Nat\nG7t376ampobq6mq+v8bx8uXLxMTEhKRJERG5vgV0O5UNGzawYcOGBuFhs9koKioKWmMiInLjCChM\nduzYwZ49e4iNjQ12PyIicgMK6NtcXbt25eabbw52LyIicoMK6MjktttuY/LkyQwZMoR27dr5x/97\nUl5ERNqugMKkS5cudOnSJdi9iIjIDSqgMNERiIiINCWgMLnzzjux2WwNxjp37ux/wqGIiLRtAYXJ\n0aNH/T/X1dWxZ88eysvLg9aUiIjcWH7yjR4jIiIYMWIEH3zwQTD6ERGRG1BARyavv/66/2fLsjh+\n/DgRERE/utyhQ4dYuXIlhYWFVFVVsWDBAmw2G7169SI7O5uwsDBWr17N3r17CQ8PJzMzkwEDBjRL\nrYiIhE5A77plZWX+fwcOHABg1apVTS5TUFDAwoULuXr1KgDLly8nIyODLVu2YFkWRUVFVFZWcuDA\nAXbs2EFeXh5LlixplloREQmtgI5Mli9fTl1dHSdOnMDr9dKrVy/Cw5teND4+nvz8fJ577jkAKisr\nGTx4MAApKSmUlpaSkJBAcnIyNpuNuLg4vF4vFy9eNK5NTU392X8QERH56QIKk4qKCubMmUNMTAw+\nn48LFy6wZs0aBg4ceM1lXC4Xp0+f9r+2LMv/jbDIyEiqq6upqalpcL+v78dNawPhdrsDqhP5KbRf\nSVsVUJjk5uayatUqf3iUl5eTk5PDa6+9FvAv+u95DI/HQ3R0NA6HA4/H02A8KirKuDYQSUlJAffe\nmIObC4yWl9bJdL8SuZ419WEpoDmTy5cvNzgKGTRokH8uJFB9+vShrKwMgOLiYpxOJ4mJiZSUlODz\n+Th79iw+n4/Y2FjjWhERCa2Ajkxuvvlm9uzZw4MPPgjAnj17fvKzTObPn8+iRYvIy8ujZ8+euFwu\n7HY7TqeTtLQ0fD4fWVlZzVIrIiKhZbO+f9pVE06ePMnTTz/NN9984x/bunUrCQkJQW0uWNxut/lp\nrjnTm6kbaU2cL61v6RZEgqap986ATnMVFxfToUMH/v73v7N582ZiY2P9XxEWEREJKEy2b9/OX//6\nVzp27Midd97Jrl27eOWVV4Ldm4iI3CACCpO6uroGV7wHcvW7iIi0HQFNwD/44IM89thjjBgxAoB3\n332XYcOGBbUxERG5cQQUJvPmzePtt9/mww8/JDw8nClTpvi/2SUiIhJQmAAMHz6c4cOHB7MXERG5\nQen2uiIiYkxhIiIixhQmIiJiTGEiIiLGFCYiImJMYSIiIsYUJiIiYkxhIiIixhQmIiJiTGEiIiLG\nFCYiImJMYSIiIsYCvtFjc9i1axd/+9vfALh69SqffvopeXl5vPjii3Tt2hWA2bNn43Q6Wbx4MceO\nHaNdu3bk5ubSo0cPysvLWbp0KXa7neTkZGbNmoXP52u0VkREQiekYTJmzBjGjBkDwJIlSxg7diwV\nFRXMmzcPl8vlr3v33Xepra1l27ZtlJeXs2LFCtatW0d2djb5+fl0796dadOmceTIEU6fPt1orYiI\nhE6LnOb65JNP+Oc//0laWhqVlZXs3LmTyZMns2LFCurr63G73QwdOhSAQYMGUVFRQU1NDbW1tcTH\nx2Oz2UhOTmbfvn2N1oqISGiF9Mjkexs2bGDmzJkA3HfffTz44IN069aN7Oxstm7dSk1NDQ6Hw19v\nt9t/MBYZGcmpU6cara2vryc8vOlNc7vdzbxVItqvpO0KeZhcunSJEydOcO+99wIwduxYoqOjARg2\nbBjvvPMOUVFReDwe/zI+nw+Hw9FgzOPxEB0dzZUrV35Q+2NBApCUlGS0HQc3FxgtL62T6X4lcj1r\n6sNSyE9zffjhh/ziF78AwLIsHn74Yf79738DsH//fvr27UtiYiLFxcUAlJeX07t3bxwOBxEREXz+\n+edYlkVJSQlOp7PRWhERCa2QH5mcOHGCbt26AWCz2cjNzWXWrFncdNNN3H777UyYMAG73U5paSkT\nJ07EsiyWLVsG/GfSfu7cuXi9XpKTkxk4cCD9+/dvtFZERELHZlmW1dJNhJrb7TY/zTVnejN1I62J\n86X1Ld2CSNA09d6pixZFRMSYwkRERIwpTERExJjCREREjClMRETEmMJERESMKUxERMSYwkRERIwp\nTERExJjCREREjClMRETEmMJERESMKUxERMSYwkRERIwpTERExJjCREREjClMRETEmMJERESMhfwZ\n8KNHj8bhcADQrVs30tLSWLp0KXa7neTkZGbNmoXP52Px4sUcO3aMdu3akZubS48ePSgvLw+4VkRE\nQiekYXL16lUsy6KwsNA/NmrUKPLz8+nevTvTpk3jyJEjnD59mtraWrZt20Z5eTkrVqxg3bp1ZGdn\nB1wrIiKhE9IwOXr0KN999x1Tp06lvr6e2bNnU1tbS3x8PADJycns27ePL7/8kqFDhwIwaNAgKioq\nqKmpCbhWRERCK6RhctNNN/HEE08wfvx4Tp48yVNPPUV0dLT//yMjIzl16hQ1NTX+U2EAdrv9B2NN\n1dbX1xMe3vSmud3uZtwykf/QfiVtVUjDJCEhgR49emCz2UhISCAqKopvvvnG//8ej4fo6GiuXLmC\nx+Pxj/t8PhwOR4Oxpmp/LEgAkpKSjLbl4OYCo+WldTLdr0SuZ019WArpt7lee+01VqxYAcD58+f5\n7rvv6NixI59//jmWZVFSUoLT6SQxMZHi4mIAysvL6d27Nw6Hg4iIiIBqRUQktEJ6ZDJu3Dief/55\nJk2ahM1mY9myZYSFhTF37ly8Xi/JyckMHDiQ/v37U1paysSJE7Esi2XLlgGwZMmSgGtFRCR0bJZl\nWS3dRKi53W7z01xzpjdTN9KaOF9a39ItiARNU++dumhRRESMKUxERMSYwkRERIwpTERExJjCRERE\njClMRETEmMJERESMKUxERMSYwkRERIwpTERExJjCREREjClMRETEmMJERESMKUxERMSYwkRERIwp\nTERExJjCREREjClMRETEWEifAV9XV0dmZiZnzpyhtraWGTNm0LVrV55++mluu+02ACZNmsRvfvMb\nVq9ezd69ewkPDyczM5MBAwZQVVXFggULsNls9OrVi+zsbMLCwhqtFRGR0AlpmOzevZuYmBj+9Kc/\n8c033/DII48wc+ZMHn/8caZOneqvq6ys5MCBA+zYsYNz584xe/Zsdu7cyfLly8nIyGDIkCFkZWVR\nVFREXFxco7UiIhI6IQ2T4cOH43K5ALAsC7vdTkVFBSdOnKCoqIgePXqQmZmJ2+0mOTkZm81GXFwc\nXq+XixcvUllZyeDBgwFISUmhtLSUhISERmtjY2NDuWkiIm1aSMMkMjISgJqaGubMmUNGRga1tbWM\nHz+efv36sW7dOtasWUNUVBQxMTENlquursayLGw2W4OxmpqaRmt/LEzcbncQtlDaOu1X0laFNEwA\nzp07x8yZM5k8eTIjR47k0qVLREdHA5CamkpOTg7Dhg3D4/H4l/F4PERFRREWFtZgLDo6GofD0Wjt\nj0lKSjLajoObC4yWl9bJdL8SuZ419WEppN/munDhAlOnTmXevHmMGzcOgCeeeILDhw8DsH//fvr2\n7UtiYiIlJSX4fD7Onj2Lz+cjNjaWPn36UFZWBkBxcTFOp/OatSIiEjohPTJZv349ly5dYu3ataxd\nuxaABQsWsGzZMiIiIujUqRM5OTk4HA6cTidpaWn4fD6ysrIAmD9/PosWLSIvL4+ePXvicrmw2+2N\n1oqISOjYLMuyWrqJUHO73eanueZMb6ZupDVxvrS+pVsQCZqm3jt10aKIiBhTmIiIiDGFiYiIGFOY\niIiIMYWJiIgYU5iIiIgxhYmIiBhTmIiIiDGFiYiIGFOYiIiIMYWJiIgYU5iIiIgxhYmIiBhTmIiI\niDGFiYiIGFOYiIiIMYWJiIgYU5iIiIixkD4DPlh8Ph+LFy/m2LFjtGvXjtzcXHr06NHSbYmItBmt\n4shkz5491NbWsm3bNv74xz+yYsWKlm5JRKRNaRVHJm63m6FDhwIwaNAgKioqWrgjkZY1fd/Blm5B\nrkPrf+kM2rpbRZjU1NTgcDj8r+12O/X19YSHX3vz3G630e+0PfaU0fLSOpnuV83lqfa2lm5BrkPB\n3D9bRZg4HA48Ho//tc/nazJIkpKSQtGWiEib0SrmTBITEykuLgagvLyc3r17t3BHIiJti82yLKul\nmzD1/be5/vGPf2BZFsuWLeP2229v6bZERNqMVhEmIiLSslrFaS4REWlZChMRETGmMBEREWMKE/nZ\nfD4fWVlZpKWlkZ6eTlVVVUu3JNLAoUOHSE9Pb+k22oRWcZ2JtIz/vo1NeXk5K1asYN26dS3dlggA\nBQUF7N69mw4dOrR0K22CjkzkZ9NtbOR6Fh8fT35+fku30WYoTORnu9ZtbESuBy6Xq8k7YUjzUpjI\nz/ZTb2MjIq2XwkR+Nt3GRkS+p4+R8rOlpqZSWlrKxIkT/bexEZG2SbdTERERYzrNJSIixhQmIiJi\nTGEiIiLGFCYiImJMYSIiIsYUJiIiYkxhInINn3zyCS+88EKzre+/7177/PPPc+bMmR/UnD9/nqee\negqABQsWsGvXroDXX11dze9///uf1NOuXbtYsGDBT1pGpDEKE5Fr6N+/P0uXLm229R04cMD/c1lZ\nGY1d4tWlSxcKCgp+1vq//fZbjh49+rP7EzGhMBG5hrKyMtLT0/nLX/7Cww8/zCOPPEJWVlaTy9TX\n17Nw4ULS0tIYNmwYTz75JFeuXCE3NxeA8ePHs3HjRr744gumTZvG119/zQMPPEBGRgYul4vDhw/z\nwAMP+Ne3d+9exowZw8iRI3nrrbeAHx5NpKenU1ZWRm5uLl988QUzZ84E4PXXX2f06NGMGjWKzMxM\nrl696h93uVyMHTuWvXv3NuefTNowhYlIE+rr69mwYQM7d+5k165d2Gw2zp8/f836jz/+mIiICLZt\n28Z7773H1atXef/991m4cCEAO3bsYNq0aXTu3JmNGzdyyy23AJCSksI777xDbGxsg/V99913bN++\nnU2bNrFs2TK+/PLLa/7uhQsX0rlzZ9asWcPx48fZvn07W7du5Y033uDWW2/l5Zdf5vz586xcuZJX\nX32Vbdu2NbhRp4gJ3ZtLpAnh4eHcfffdjBs3jmHDhvHb3/6WLl26XLP+nnvuISYmhldffZV//etf\nnDx5ksuXL//o7xk4cGCj46NHjyY8PJwuXbowaNAgDh06FFDfZWVlVFVVMWHCBADq6uro06cPH3/8\nMXfffTedOnUCYOTIkXzwwQcBrVOkKQoTkR+xdu1aysvLKS4u5sknn2TlypUMHjy40dqioiJeeukl\npkyZwpgxY/j6668bnRv5X+3bt2903G63+3+2LIuIiAhsNluDddbV1f1gOa/Xy4gRI/xHRB6PB6/X\ny/79+/H5fP46PTJAmotOc4k04eLFi4wYMYLevXvzzDPPcN9993Hs2LFr1u/fv58RI0YwduxYOnXq\nxIcffojX6wUaPjzMbrf7x5vy5ptvYlkWZ86c4ZNPPqF///7ccsstfPbZZ1iWxalTp/z9hIeH+9c/\nZMgQ3nvvPb766issy2Lx4sVs3ryZpKQkDh06xPnz5/H5fP55GBFT+lgi0oTY2FiGDRvGuHHj6NCh\nA127dmX06NHXrB8/fjxz587l7bffpl27dgwaNIjTp08DMGzYMEaNGsWuXbv49a9/zbRp09i0aVOT\nv79jx46MGTOG+vp6/u///o/Y2Fh++ctfsnPnToYPH05CQgJJSUkA3HrrrcTFxZGenk5hYSGzZs3i\nsccew+fzcddddzFt2jTat2/PwoUL+d3vfkeHDh244447mu+PJW2abkEvIiLGdGQi8hMdPHiQnJyc\nRv9v48aNTU7Qi7RWOjIRERFjmoAXERFjChMRETGmMBEREWMKExERMfb/Bht0YSFsvxQAAAAASUVO\nRK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.countplot(x='is_attributed', data=df_train, palette='hls')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 50000 entries, 0 to 49999\n",
      "Data columns (total 57 columns):\n",
      "Unnamed: 0                            50000 non-null int64\n",
      "ip                                    50000 non-null int64\n",
      "app                                   50000 non-null int64\n",
      "device                                50000 non-null int64\n",
      "os                                    50000 non-null int64\n",
      "channel                               50000 non-null int64\n",
      "click_time                            50000 non-null object\n",
      "attributed_time                       88 non-null object\n",
      "is_attributed                         50000 non-null int64\n",
      "day                                   50000 non-null int64\n",
      "hour                                  50000 non-null int64\n",
      "minute                                50000 non-null int64\n",
      "second                                50000 non-null int64\n",
      "ip_confRate                           50000 non-null float64\n",
      "app_confRate                          50000 non-null float64\n",
      "device_confRate                       50000 non-null float64\n",
      "os_confRate                           50000 non-null float64\n",
      "channel_confRate                      50000 non-null float64\n",
      "app_channel_confRate                  50000 non-null float64\n",
      "app_os_confRate                       50000 non-null float64\n",
      "app_device_confRate                   50000 non-null float64\n",
      "channel_os_confRate                   50000 non-null float64\n",
      "channel_device_confRate               50000 non-null float64\n",
      "os_device_confRate                    50000 non-null float64\n",
      "ip_app_channel_var_day                34970 non-null float64\n",
      "ip_app_os_var_hour                    36809 non-null float64\n",
      "ip_day_channel_var_hour_x             38512 non-null float64\n",
      "ip_day_hour_count_channel             50000 non-null int64\n",
      "ip_app_count_channel                  50000 non-null int64\n",
      "ip_app_os_count_channel               50000 non-null int64\n",
      "ip_app_day_hour_count_channel         50000 non-null int64\n",
      "ip_app_channel_mean_hour              50000 non-null float64\n",
      "app_AvgViewPerDistinct_ip             50000 non-null float64\n",
      "app_count_channel                     50000 non-null int64\n",
      "channel_count_app                     50000 non-null int64\n",
      "ip_nunique_channel                    50000 non-null int64\n",
      "ip_nunique_app                        50000 non-null int64\n",
      "ip_day_nunique_hour                   50000 non-null int64\n",
      "ip_app_nunique_os                     50000 non-null int64\n",
      "ip_nunique_device                     50000 non-null int64\n",
      "app_nunique_channel                   50000 non-null int64\n",
      "ip_device_os_nunique_app              50000 non-null int64\n",
      "ip_device_os_cumcount_app             50000 non-null int64\n",
      "ip_cumcount_app                       50000 non-null int64\n",
      "ip_cumcount_os                        50000 non-null int64\n",
      "ip_day_channel_var_hour_y             38512 non-null float64\n",
      "ip_nextClick                          48959 non-null float64\n",
      "ip_app_nextClick                      39531 non-null float64\n",
      "ip_channel_nextClick                  32900 non-null float64\n",
      "ip_os_nextClick                       44728 non-null float64\n",
      "ip_app_device_os_channel_nextClick    21355 non-null float64\n",
      "ip_os_device_nextClick                44410 non-null float64\n",
      "ip_os_device_app_nextClick            29197 non-null float64\n",
      "prev_identical_clicks                 50000 non-null int64\n",
      "future_identical_clicks               50000 non-null int64\n",
      "prev_app_clicks                       50000 non-null int64\n",
      "future_app_clicks                     50000 non-null int64\n",
      "dtypes: float64(24), int64(31), object(2)\n",
      "memory usage: 21.7+ MB\n"
     ]
    }
   ],
   "source": [
    "# The ratio of df_train to df_test is 0.8 to 0.2 or 0.75 to 0.25\n",
    "df_test = pd.read_csv('data/train_new_cols.csv', nrows=50000,skiprows=range(1, 400000))\n",
    "df_test.dropna()\n",
    "df_test.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_test = df_test.loc[:,cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ip</th>\n",
       "      <th>app</th>\n",
       "      <th>device</th>\n",
       "      <th>os</th>\n",
       "      <th>channel</th>\n",
       "      <th>click_time</th>\n",
       "      <th>attributed_time</th>\n",
       "      <th>is_attributed</th>\n",
       "      <th>ip_nextClick</th>\n",
       "      <th>ip_app_nextClick</th>\n",
       "      <th>ip_channel_nextClick</th>\n",
       "      <th>ip_os_nextClick</th>\n",
       "      <th>ip_app_device_os_channel_nextClick</th>\n",
       "      <th>ip_os_device_nextClick</th>\n",
       "      <th>ip_os_device_app_nextClick</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>115115</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>145</td>\n",
       "      <td>2017-11-06 16:07:47</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>21633</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>19</td>\n",
       "      <td>178</td>\n",
       "      <td>2017-11-06 16:07:47</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>140.0</td>\n",
       "      <td>344.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>140.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>140.0</td>\n",
       "      <td>344.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>144498</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>178</td>\n",
       "      <td>2017-11-06 16:07:47</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>133.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>76919</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>237</td>\n",
       "      <td>2017-11-06 16:07:47</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>73.0</td>\n",
       "      <td>709.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>709.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1556</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>245</td>\n",
       "      <td>2017-11-06 16:07:47</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>50.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>50.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>50.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>67467</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>37</td>\n",
       "      <td>245</td>\n",
       "      <td>2017-11-06 16:07:47</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>37.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>37.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>37.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>20266</td>\n",
       "      <td>64</td>\n",
       "      <td>1</td>\n",
       "      <td>19</td>\n",
       "      <td>459</td>\n",
       "      <td>2017-11-06 16:07:47</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>689.0</td>\n",
       "      <td>36.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>31564</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>19</td>\n",
       "      <td>265</td>\n",
       "      <td>2017-11-06 16:07:47</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>609.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.0</td>\n",
       "      <td>461.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1732</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>134</td>\n",
       "      <td>2017-11-06 16:07:47</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>21.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>111114</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>245</td>\n",
       "      <td>2017-11-06 16:07:47</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>318.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       ip  app  device  os  channel           click_time attributed_time  \\\n",
       "0  115115    8       1  13      145  2017-11-06 16:07:47             NaN   \n",
       "1   21633    1       1  19      178  2017-11-06 16:07:47             NaN   \n",
       "2  144498   12       1  17      178  2017-11-06 16:07:47             NaN   \n",
       "3   76919    2       1   6      237  2017-11-06 16:07:47             NaN   \n",
       "4    1556   15       1  13      245  2017-11-06 16:07:47             NaN   \n",
       "5   67467   15       1  37      245  2017-11-06 16:07:47             NaN   \n",
       "6   20266   64       1  19      459  2017-11-06 16:07:47             NaN   \n",
       "7   31564   15       1  19      265  2017-11-06 16:07:47             NaN   \n",
       "8    1732    9       1  13      134  2017-11-06 16:07:47             NaN   \n",
       "9  111114   15       1  13      245  2017-11-06 16:07:47             NaN   \n",
       "\n",
       "   is_attributed  ip_nextClick  ip_app_nextClick  ip_channel_nextClick  \\\n",
       "0              0           0.0               0.0                   0.0   \n",
       "1              0         140.0             344.0                   NaN   \n",
       "2              0           0.0              11.0                   NaN   \n",
       "3              0           2.0               6.0                  73.0   \n",
       "4              0           0.0              50.0                   0.0   \n",
       "5              0           0.0              37.0                   0.0   \n",
       "6              0           0.0             689.0                  36.0   \n",
       "7              0           3.0              23.0                 609.0   \n",
       "8              0           0.0              21.0                   2.0   \n",
       "9              0         318.0               NaN                   NaN   \n",
       "\n",
       "   ip_os_nextClick  ip_app_device_os_channel_nextClick  \\\n",
       "0              0.0                                 0.0   \n",
       "1            140.0                                 NaN   \n",
       "2              0.0                                 NaN   \n",
       "3            709.0                                 NaN   \n",
       "4              0.0                                50.0   \n",
       "5              0.0                                37.0   \n",
       "6              0.0                                 NaN   \n",
       "7              3.0                                 NaN   \n",
       "8              0.0                                21.0   \n",
       "9              NaN                                 NaN   \n",
       "\n",
       "   ip_os_device_nextClick  ip_os_device_app_nextClick  \n",
       "0                     0.0                         0.0  \n",
       "1                   140.0                       344.0  \n",
       "2                     0.0                       133.0  \n",
       "3                   709.0                         NaN  \n",
       "4                     0.0                        50.0  \n",
       "5                     0.0                        37.0  \n",
       "6                     0.0                         NaN  \n",
       "7                     3.0                       461.0  \n",
       "8                     0.0                        21.0  \n",
       "9                     NaN                         NaN  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    49912\n",
       "1       88\n",
       "Name: is_attributed, dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test['is_attributed'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAEFCAYAAAAfRLtkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAFNRJREFUeJzt3X9MVff9x/HX4V60yo/ibaMpcTq0\nGktmpXCHSYesKW7IH9ZqcagN7dJV1q7qSNYORX7oxF9pg4la648ui7F2KoU4ky3tlNQS0UK9KVrZ\ndKZrmYqxP2w3uFoEzvn+sXq/ZRX9UDiA8Hz8hYfPvbyvubnPe+6591zLcRxHAAAYCOvrAQAAtw+i\nAQAwRjQAAMaIBgDAGNEAABjz9vUAbgsEAn09AgDclpKSkr61bcBHQ7rxDQcAdK6zJ9y8PAUAMEY0\nAADGiAYAwBjRAAAYIxoAAGNEAwBgzNW33M6ePVuRkZGSpNGjRysrK0urV6+Wx+NRSkqKFi1aJNu2\ntWLFCp05c0ZDhgxRSUmJxo4dq7q6OuO1AIDe4Vo0Wlpa5DiOdu3aFdo2a9Ysbdq0Sd/73veUk5Oj\nv/3tbzp//ryuXbumvXv3qq6uTuvWrdMrr7yi4uJi47UAgN7hWjROnz6tq1ev6qmnnlJbW5sWL16s\na9euacyYMZKklJQUHT16VJ9++qmmTZsmSUpISNCpU6fU3NxsvNYEnwoHgJ7hWjTuuOMO/eIXv9Dc\nuXP18ccfa+HChYqOjg79PiIiQufOnVNzc3PoJSxJ8ng839p2s7VtbW3yem9+M7r7ifDjS57p1uUx\n8Pg3bu3rEQBXdfZk27VoxMXFaezYsbIsS3FxcYqKitKXX34Z+n0wGFR0dLS++uorBYPB0HbbthUZ\nGdlh283W3ioYAICe49q7p9544w2tW7dOknTp0iVdvXpVw4cP17/+9S85jqMjR47I7/crMTFRVVVV\nkqS6ujpNnDhRkZGRCg8PN1oLAOg9rj1Nz8zM1LJlyzR//nxZlqU1a9YoLCxMzz//vNrb25WSkqIp\nU6Zo8uTJqq6u1rx58+Q4jtasWSNJWrlypfFaAEDvsBzHcfp6CDcFAgGOaaDHcUwDA11nj518uA8A\nYIxoAACMEQ0AgDGiAQAwRjQAAMaIBgDAGNEAABgjGgAAY0QDAGCMaAAAjBENAIAxogEAMEY0AADG\niAYAwBjRAAAYIxoAAGNEAwBgjGgAAIwRDQCAMaIBADBGNAAAxogGAMAY0QAAGCMaAABjRAMAYIxo\nAACMEQ0AgDGiAQAwRjQAAMaIBgDAGNEAABgjGgAAY0QDAGDM1Wh8/vnn+vGPf6wPP/xQDQ0Nmj9/\nvhYsWKDi4mLZti1J2rx5szIzMzVv3jydPHlSkrq0FgDQe1yLRmtrq4qKinTHHXdIktauXavc3Fy9\n/vrrchxHlZWVqq+vV21trcrKylRaWqqVK1d2eS0AoPe4Fo3169dr3rx5GjlypCSpvr5eycnJkqTU\n1FQdPXpUgUBAKSkpsixLsbGxam9v1+XLl7u0FgDQe7xuXGlFRYV8Pp+mTZum7du3S5Icx5FlWZKk\niIgINTU1qbm5WTExMaHLXd/elbU+n++W8wQCgZ68eQD3KQxarkSjvLxclmXp2LFj+vvf/668vLwO\newXBYFDR0dGKjIxUMBjssD0qKkphYWHGa00kJSV16/Yc37mjW5fHwNPd+xTQ33X2xMiVl6d2796t\n1157Tbt27dJ9992n9evXKzU1VTU1NZKkqqoq+f1+JSYm6siRI7JtW42NjbJtWz6fT/Hx8cZrAQC9\nx5U9jRvJy8tTYWGhSktLNW7cOKWnp8vj8cjv9ysrK0u2bauoqKjLawEAvcdyHMfp6yHcFAgEuv/y\n1JJnemgaDBT+jVv7egTAVZ09dvLhPgCAMaIBADBGNAAAxogGAMAY0QAAGCMaAABjRAMAYIxoAACM\nEQ0AgDGiAQAwRjQAAMaIBgDAGNEAABgjGgAAY0QDAGCMaAAAjBENAIAxogEAMEY0AADGiAYAwBjR\nAAAYIxoAAGNEAwBgjGgAAIwRDQCAMaIBADBGNAAAxogGAMAY0QAAGCMaAABjRAMAYIxoAACMEQ0A\ngDGiAQAw5nXritvb21VQUKCPPvpIlmVp5cqVGjp0qJYuXSrLsjRhwgQVFxcrLCxMmzdv1uHDh+X1\nepWfn6/7779fDQ0NxmsBAL3DtWi8/fbbkqQ9e/aopqZGGzZskOM4ys3N1dSpU1VUVKTKykrFxsaq\ntrZWZWVlunjxohYvXqzy8nKtXbvWeC0AoHe4Fo3p06froYcekiQ1NjYqOjpaR48eVXJysiQpNTVV\n1dXViouLU0pKiizLUmxsrNrb23X58mXV19cbr/X5fDedJRAIuHUzMUhxn8Jg5Vo0JMnr9SovL08H\nDx7Uxo0bVV1dLcuyJEkRERFqampSc3OzYmJiQpe5vt1xHOO1t4pGUlJSt27H8Z07unV5DDzdvU8B\n/V1nT4xcPxC+fv16vfXWWyosLFRLS0toezAYVHR0tCIjIxUMBjtsj4qKUlhYmPFaAEDvcC0a+/fv\n17Zt2yRJw4YNk2VZ+sEPfqCamhpJUlVVlfx+vxITE3XkyBHZtq3GxkbZti2fz6f4+HjjtQCA3uHa\ny1M//elPtWzZMj3++ONqa2tTfn6+xo8fr8LCQpWWlmrcuHFKT0+Xx+OR3+9XVlaWbNtWUVGRJCkv\nL894LQCgd1iO4zi3WrRq1SoVFhZ22JaXl6f169e7NlhPCQQC3T+mseSZHpoGA4V/49a+HgFwVWeP\nnTfd01i+fLnOnTunU6dO6ezZs6HtbW1tampq6vkpAQD92k2j8eyzz+rChQtavXq1Fi1aFNru8Xg0\nfvx414cDAPQvN43G6NGjNXr0aB04cEDNzc2ht8JK0pUrVzq8/RUAMPAZHQjftm2btm3b1iESlmWp\nsrLStcEAAP2PUTTKysp06NAh3t4KAIOc0ec07rnnHt15551uzwIA6OeM9jS+//3va8GCBZo6daqG\nDBkS2v7Ng+MAgIHPKBqjRo3SqFGj3J4FANDPGUWDPQoAgGQYjUmTJoXOOHvdyJEj9c4777gyFACg\nfzKKxunTp0M/t7a26tChQ6qrq3NtKABA/9Tls9yGh4crIyND7777rhvzAAD6MaM9jf3794d+dhxH\nZ8+eVXh4uGtDAQD6J6NoXP9ei+tGjBihDRs2uDIQAKD/MorG2rVr1draqo8++kjt7e2aMGGCvF5X\nvykWANAPGT3ynzp1SkuWLFFMTIxs29Znn32ml19+WVOmTHF7PgBAP2IUjZKSEm3YsCEUibq6Oq1a\ntUpvvPGGq8MBAPoXo3dPXblypcNeRUJCglpaWlwbCgDQPxlF484779ShQ4dC/z506BDfpQEAg5DR\ny1OrVq3SL3/5Sy1fvjy0bc+ePa4NBQDon4z2NKqqqjRs2DC9/fbb2rlzp3w+n2pra92eDQDQzxhF\nY9++ffrjH/+o4cOHa9KkSaqoqNBrr73m9mwAgH7GKBqtra0dPgHOp8EBYHAyOqYxffp0Pfnkk8rI\nyJAk/fWvf1VaWpqrgwEA+h+jaLzwwgt688039d5778nr9eqJJ57Q9OnT3Z4NANDPGJ8LZMaMGZox\nY4abswAA+rkunxodADB4EQ0AgDGiAQAwRjQAAMaIBgDAGNEAABgjGgAAY658Z2tra6vy8/N14cIF\nXbt2Tc8++6zuvfdeLV26VJZlacKECSouLlZYWJg2b96sw4cPy+v1Kj8/X/fff78aGhqM1wIAeo8r\n0Thw4IBiYmL04osv6ssvv9Sjjz6qSZMmKTc3V1OnTlVRUZEqKysVGxur2tpalZWV6eLFi1q8eLHK\ny8u1du1a47UAgN7jSjRmzJih9PR0SZLjOPJ4PKqvr1dycrIkKTU1VdXV1YqLi1NKSoosy1JsbKza\n29t1+fLlLq31+Xxu3AQAwA24Eo2IiAhJUnNzs5YsWaLc3FytX79elmWFft/U1KTm5uYO3wB4fbvj\nOMZrTaIRCAR68uYB3KcwaLkSDUm6ePGinnvuOS1YsEAzZ87Uiy++GPpdMBhUdHS0IiMjFQwGO2yP\niopSWFiY8VoTSUlJ3botx3fu6NblMfB09z4F9HedPTFy5d1Tn332mZ566im98MILyszMlCTFx8er\npqZG0n+/CdDv9ysxMVFHjhyRbdtqbGyUbdvy+XxdWgsA6D2u7Gls3bpV//nPf7RlyxZt2bJFkrR8\n+XKVlJSotLRU48aNU3p6ujwej/x+v7KysmTbtoqKiiRJeXl5KiwsNFoLAOg9luM4Tl8P4aZAIND9\nl6eWPNND02Cg8G/c2tcjAK7q7LGTD/cBAIwRDQCAMaIBADBGNAAAxogGAMAY0QAAGCMaAABjRAMA\nYIxoAACMEQ0AgDGiAQAwRjQAAMaIBgDAGNEAABgjGgAAY0QDAGCMaAAAjBENAIAxogEAMEY0AADG\niAYAwBjRAAAYIxoAAGNEAwBgjGgAAIwRDQCAMaIBADBGNAAAxogGAMAY0QAAGCMaAABjRAMAYIxo\nAACMEQ0AgDFXo3HixAllZ2dLkhoaGjR//nwtWLBAxcXFsm1bkrR582ZlZmZq3rx5OnnyZJfXAgB6\nj2vR2LFjhwoKCtTS0iJJWrt2rXJzc/X666/LcRxVVlaqvr5etbW1KisrU2lpqVauXNnltQCA3uN1\n64rHjBmjTZs26be//a0kqb6+XsnJyZKk1NRUVVdXKy4uTikpKbIsS7GxsWpvb9fly5e7tNbn891y\nlkAg4NbNxCDFfQqDlWvRSE9P1/nz50P/dhxHlmVJkiIiItTU1KTm5mbFxMSE1lzf3pW1JtFISkrq\n1m05vnNHty6Pgae79ymgv+vsiVGvHQgPC/v/PxUMBhUdHa3IyEgFg8EO26Oiorq0FgDQe3otGvHx\n8aqpqZEkVVVVye/3KzExUUeOHJFt22psbJRt2/L5fF1aCwDoPa69PPW/8vLyVFhYqNLSUo0bN07p\n6enyeDzy+/3KysqSbdsqKirq8loAQO+xHMdx+noINwUCge4f01jyTA9Ng4HCv3FrX48AuKqzx04+\n3AcAMEY0AADGiAYAwBjRAAAYIxoAAGNEAwBgjGgAAIwRDQCAMaIBADBGNAAAxogGAMAY0QAAGCMa\nAABjRAMAYIxoAACMEQ0AgDGiAQAwRjQAAMaIBgDAGNEAABgjGgAAY0QDAGCMaAAAjBENAIAxogEA\nMEY0AADGiAYAwBjRAAAYIxoAAGNEAwBgjGgAAIwRDQCAMaIBADDm7esBusq2ba1YsUJnzpzRkCFD\nVFJSorFjx/b1WAAwKNx2exqHDh3StWvXtHfvXv3mN7/RunXr+nokABg0brs9jUAgoGnTpkmSEhIS\ndOrUqT6eCOg7zxw93tcjoB/a+qDfteu+7aLR3NysyMjI0L89Ho/a2trk9XZ+UwKBQLf+pvXkwm5d\nHgNPd+9TPWXhUKuvR0A/5Ob987aLRmRkpILBYOjftm3fNBhJSUm9MRYADAq33TGNxMREVVVVSZLq\n6uo0ceLEPp4IAAYPy3Ecp6+H6Irr7576xz/+IcdxtGbNGo0fP76vxwKAQeG2iwYAoO/cdi9PAQD6\nDtEAABgjGgAAY0QDt2TbtoqKipSVlaXs7Gw1NDT09UhABydOnFB2dnZfjzEo3Haf00Dv++apW+rq\n6rRu3Tq98sorfT0WIEnasWOHDhw4oGHDhvX1KIMCexq4JU7dgv5szJgx2rRpU1+PMWgQDdxSZ6du\nAfqD9PT0m54VAj2LaOCWunrqFgADF9HALXHqFgDX8XQRt/STn/xE1dXVmjdvXujULQAGJ04jAgAw\nxstTAABjRAMAYIxoAACMEQ0AgDGiAQAwRjQAAMaIBiDpgw8+0PLly3vs+r55xtVly5bpwoUL31pz\n6dIlLVy4UJK0dOlSVVRUGF9/U1OTfvWrX3VppoqKCi1durRLlwH+F9EAJE2ePFmrV6/useurra0N\n/VxTU6MbfRxq1KhR2rFjx3e6/n//+986ffr0d54P+K6IBqD/PrBnZ2frD3/4gx555BE9+uijKioq\nuull2traVFBQoKysLKWlpenpp5/WV199pZKSEknS3LlztX37dn3yySfKycnRF198oYcffli5ublK\nT0/XyZMn9fDDD4eu7/Dhw5ozZ45mzpypv/zlL5K+vXeQnZ2tmpoalZSU6JNPPtFzzz0nSdq/f79m\nz56tWbNmKT8/Xy0tLaHt6enpeuyxx3T48OGe/C/DIEU0gK+1tbVp27ZtKi8vV0VFhSzL0qVLlzpd\n//777ys8PFx79+7VwYMH1dLSonfeeUcFBQWSpLKyMuXk5GjkyJHavn27RowYIUlKTU3VW2+9JZ/P\n1+H6rl69qn379unVV1/VmjVr9Omnn3b6twsKCjRy5Ei9/PLLOnv2rPbt26c9e/boT3/6k+666y79\n/ve/16VLl/TSSy9p9+7d2rt3b4eTTgLfFeeeAr7m9Xr1wAMPKDMzU2lpaXr88cc1atSoTtf/8Ic/\nVExMjHbv3q1//vOf+vjjj3XlypVb/p0pU6bccPvs2bPl9Xo1atQoJSQk6MSJE0Zz19TUqKGhQT/7\n2c8kSa2trYqPj9f777+vBx54QHfffbckaebMmXr33XeNrhPoDNEAvmHLli2qq6tTVVWVnn76ab30\n0ktKTk6+4drKykpt3LhRTzzxhObMmaMvvvjihscu/tfQoUNvuN3j8YR+dhxH4eHhsiyrw3W2trZ+\n63Lt7e3KyMgI7eEEg0G1t7fr2LFjsm07tI7T2aMn8PIU8LXLly8rIyNDEydO1K9//Wv96Ec/0pkz\nZzpdf+zYMWVkZOixxx7T3Xffrffee0/t7e2SOn5RlcfjCW2/mT//+c9yHEcXLlzQBx98oMmTJ2vE\niBH68MMP5TiOzp07F5rH6/WGrn/q1Kk6ePCgPv/8czmOoxUrVmjnzp1KSkrSiRMndOnSJdm2HTpO\nAnQHTz2Ar/l8PqWlpSkzM1PDhg3TPffco9mzZ3e6fu7cuXr++ef15ptvasiQIUpISND58+clSWlp\naZo1a5YqKir00EMPKScnR6+++upN//7w4cM1Z84ctbW16Xe/+518Pp8efPBBlZeXa8aMGYqLi1NS\nUpIk6a677lJsbKyys7O1a9cuLVq0SE8++aRs29Z9992nnJwcDR06VAUFBfr5z3+uYcOG6d577+25\n/ywMWpwaHQBgjD0N4CaOHz+uVatW3fB327dvv+mBcmAgYk8DAGCMA+EAAGNEAwBgjGgAAIwRDQCA\nsf8DoQTTGbbtqNgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.countplot(x='is_attributed', data=df_test, palette='hls')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['ip',\n",
       " 'app',\n",
       " 'device',\n",
       " 'os',\n",
       " 'channel',\n",
       " 'ip_nextClick',\n",
       " 'ip_app_nextClick',\n",
       " 'ip_channel_nextClick',\n",
       " 'ip_os_nextClick',\n",
       " 'ip_app_device_os_channel_nextClick',\n",
       " 'ip_os_device_nextClick',\n",
       " 'ip_os_device_app_nextClick']"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get columes names except the click_time (object), attributed_time (object) and is_attributed\n",
    "train_cols = []\n",
    "for each_value in df_train.columns.values:\n",
    "    if each_value == 'click_time' or each_value == 'attributed_time' or each_value == 'is_attributed':\n",
    "        continue\n",
    "    train_cols.append(each_value)\n",
    "train_cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Data_X\n",
    "X_train = df_train.loc[:,train_cols]\n",
    "X_test = df_test.loc[:,train_cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(200000, 12)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(50000, 12)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Data_Y\n",
    "y_train = df_train[['is_attributed']]\n",
    "y_test = df_test[['is_attributed']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(200000, 1)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(50000, 1)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "y_test_val = y_test.values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Algorithms"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def randomForest_pre(X_train, y_train, X_test, y_test):\n",
    "    X_train = np.nan_to_num(X_train)\n",
    "    y_train = np.nan_to_num(y_train)\n",
    "    X_test = np.nan_to_num(X_test)\n",
    "    y_test = np.nan_to_num(y_test)\n",
    "    RF_model = RandomForestClassifier()\n",
    "    # Train the model\n",
    "    RF_fit = RF_model.fit(X_train, y_train)\n",
    "    # Get the prediction of the test data\n",
    "    RF_predict = RF_model.predict(X_test)\n",
    "    # Compare the prediction with the known values\n",
    "    RF_acc = sklearn.metrics.accuracy_score(np.array(RF_predict)[:], \n",
    "                                         np.array(y_test_val)[:])\n",
    "    # Plot the data\n",
    "    plt.figure(figsize=(10,5))\n",
    "    plt.plot(RF_predict, color='red', label='Prediction')\n",
    "    plt.plot(y_test, label='Y_test')\n",
    "    plt.legend(['Prediction', 'Y_test'])\n",
    "    _ = plt.ylim()\n",
    "    \n",
    "    return RF_fit, RF_predict, RF_acc, RF_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# RF_fit, RF_pre, RF_acc, rf_model = randomForest_pre(X_train, y_train, X_test, y_test)\n",
    "# RF_acc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Gradient Boosting & AdaBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.ensemble import GradientBoostingClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def gradientBoosting_pre(X_train, y_train, X_test, y_test):\n",
    "    X_train = np.nan_to_num(X_train)\n",
    "    y_train = np.nan_to_num(y_train)\n",
    "    X_test = np.nan_to_num(X_test)\n",
    "    y_test = np.nan_to_num(y_test)\n",
    "    GB_model = GradientBoostingClassifier(n_estimators=100, learning_rate=1.0, max_depth=1, random_state=0)\n",
    "    # Train the model\n",
    "    GB_fit = GB_model.fit(X_train, y_train)\n",
    "    # Get the prediction of the test data\n",
    "    GB_predict = GB_model.predict(X_test)\n",
    "    # Compare the prediction with the known values\n",
    "    GB_acc = sklearn.metrics.accuracy_score(np.array(GB_predict)[:], \n",
    "                                         np.array(y_test_val)[:])\n",
    "    \n",
    "    # Plot the data\n",
    "    plt.figure(figsize=(10,5))\n",
    "    plt.plot(GB_predict, color='red', label='Prediction')\n",
    "    plt.plot(y_test, label='Y_test')\n",
    "    plt.legend(['Prediction', 'Y_test'])\n",
    "    _ = plt.ylim()\n",
    "    \n",
    "    return GB_fit, GB_predict, GB_acc, GB_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# GB_fit, GB_predict, GB_acc, gb_model = gradientBoosting_pre(X_train, y_train, X_test, y_test)\n",
    "# GB_acc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def logisticRegression_pre(X_train, y_train, X_test, y_test):\n",
    "    X_train = np.nan_to_num(X_train)\n",
    "    y_train = np.nan_to_num(y_train)\n",
    "    X_test = np.nan_to_num(X_test)\n",
    "    y_test = np.nan_to_num(y_test)\n",
    "    LG_model = LogisticRegression()\n",
    "    # Train the model\n",
    "    LG_fit = LG_model.fit(X_train, y_train)\n",
    "    # Get the prediction of the test data\n",
    "    LG_predict = LG_model.predict(X_test)\n",
    "    # Compare the prediction with the known values\n",
    "    LG_acc = sklearn.metrics.accuracy_score(np.array(LG_predict)[:], \n",
    "                                         np.array(y_test_val)[:])\n",
    "    \n",
    "    return LG_fit, LG_predict, LG_acc, LG_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# LG_fit, LG_predict, LG_acc, lg_model = logisticRegression_pre(X_train, y_train, X_test, y_test)\n",
    "# LG_acc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. SVM - Support Vector Machine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn import svm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def svm_pre(X_train, y_train, X_test, y_test):\n",
    "    X_train = np.nan_to_num(X_train)\n",
    "    y_train = np.nan_to_num(y_train)\n",
    "    X_test = np.nan_to_num(X_test)\n",
    "    y_test = np.nan_to_num(y_test)\n",
    "    svm_model = svm.SVC()\n",
    "    uniq = np.unique(y_train[9000:10000])\n",
    "    # Train the model\n",
    "    SVM_fit = svm_model.fit(X_train[9000:10000], y_train[9000:10000])\n",
    "    # Get the prediction of the test data\n",
    "    SVM_predict = svm_model.predict(X_test)\n",
    "    # Compare the prediction with the known values\n",
    "    SVM_acc = sklearn.metrics.accuracy_score(np.array(LG_predict)[:], \n",
    "                                     np.array(y_test_val)[:])\n",
    "    return SVM_fit, SVM_predict, SVM_acc, svm_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# svm_fit, svm_predict, svm_acc, svm_model = svm_pre(X_train, y_train, X_test, y_test)\n",
    "# svm_acc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5. ANN - Artificial Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def shallow_net_A(n=55,i=len(train_cols),o=2):\n",
    "    # Create simple one dense layer net\n",
    "    # Default 55 neurons, input 5, output 2\n",
    "    net = Sequential()\n",
    "    net.add(Dense(n, activation='sigmoid', input_shape=(i,)))\n",
    "    net.add(Dense(2, activation='softmax'))\n",
    "    # Compile net\n",
    "    net.compile(loss='mean_squared_error', optimizer=SGD(lr=0.01), metrics=['accuracy'])\n",
    "    return net"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def ann_pre(X_train, y_train, X_test, y_test):\n",
    "    ann_model = shallow_net_A()\n",
    "    ann_summary = ann_model.summary()\n",
    "    # Convert the values\n",
    "    X_train_ann = np.nan_to_num(X_train)\n",
    "    y_train_ann = np.nan_to_num(y_train)\n",
    "    X_test_ann = np.nan_to_num(X_test)\n",
    "    y_test_ann = np.nan_to_num(y_test)\n",
    "    # Conver the matrix, finally we have two classes (n_classes), the original one has oly one class\n",
    "    n_classes = 2\n",
    "    y_train_ann = keras.utils.to_categorical(y_train_ann, n_classes)\n",
    "    y_test_ann = keras.utils.to_categorical(y_test_ann, n_classes)\n",
    "    # Training the model\n",
    "    ann_fit = ann_model.fit(X_train_ann, y_train_ann, batch_size=128, epochs=99, verbose=1, validation_data=(X_test_ann, y_test_ann))\n",
    "    \n",
    "    # Evaluate: loss & accuracy -> Using Evaluation to get the accracy\n",
    "    ann_evaluate = ann_model.evaluate(X_test_ann, y_test_ann)\n",
    "    \n",
    "    # Using prediction\n",
    "    ann_pre = ann_model.predict(X_test)\n",
    "    # Convert value to boolean value\n",
    "    y_pre = (ann_pre > 0.5)\n",
    "    # Counting the boolean value, counting the accuracy by using basic calculation\n",
    "    from sklearn.metrics import confusion_matrix\n",
    "    ann_output = confusion_matrix(y_test_ann.argmax(axis=1), y_pre.argmax(axis=1))\n",
    "    ann_prediction_acc = ann_output[0][0]/(ann_output[0][0]+ann_output[1][0])\n",
    "    \n",
    "    return ann_summary, ann_fit, ann_evaluate, ann_prediction_acc, ann_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# ann_summary, ann_fit, ann_evaluate, ann_prediction_acc, ann_model = ann_pre(X_train, y_train, X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# ann_prediction_acc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6. MLP - Multi-layered Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def shallow_net_C(n=55,i=len(train_cols),o=2):\n",
    "    # Create simple one dense layer net\n",
    "    # Default 55 neurons, input 5, output 2, here we have more hidden layers with different activation\n",
    "    net = Sequential()\n",
    "    net.add(Dense(n, activation='sigmoid', input_shape=(i,)))\n",
    "    net.add(Dense(n, activation='relu', input_shape=(i,)))\n",
    "    net.add(Dense(n, activation='tanh', input_shape=(i,)))\n",
    "    net.add(Dense(n, activation='elu', input_shape=(i,)))\n",
    "    net.add(Dense(2, activation='softmax'))\n",
    "    # Compile net\n",
    "    net.compile(loss='mean_squared_error', optimizer=SGD(lr=0.01), metrics=['accuracy'])\n",
    "    return net"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def mlp_pre(X_train, y_train, X_test, y_test):\n",
    "    mlp_model = shallow_net_C()\n",
    "    mlp_summary = mlp_model.summary()\n",
    "    # Convert the values\n",
    "    X_train_mlp = np.nan_to_num(X_train)\n",
    "    y_train_mlp = np.nan_to_num(y_train)\n",
    "    X_test_mlp = np.nan_to_num(X_test)\n",
    "    y_test_mlp = np.nan_to_num(y_test)\n",
    "    # Conver the matrix, finally we have two classes (n_classes)\n",
    "    n_classes = 2\n",
    "    y_train_mlp = keras.utils.to_categorical(y_train_mlp, n_classes)\n",
    "    y_test_mlp = keras.utils.to_categorical(y_test_mlp, n_classes)\n",
    "    # Training the model\n",
    "    mlp_fit = mlp_model.fit(X_train_mlp, y_train_mlp, batch_size=128, epochs=99, verbose=1, validation_data=(X_test_mlp, y_test_mlp))\n",
    "    \n",
    "    # Evaluate: loss & accuracy -> Using Evaluation to get the accracy\n",
    "    mlp_evaluate = mlp_model.evaluate(X_test_mlp, y_test_mlp)\n",
    "    \n",
    "    # Using prediction\n",
    "    mlp_pre = mlp_model.predict(X_test)\n",
    "    # Convert value to boolean value\n",
    "    y_pre = (mlp_pre > 0.5)\n",
    "    # Counting the boolean value, counting the accuracy by using basic calculation\n",
    "    from sklearn.metrics import confusion_matrix\n",
    "    mlp_output = confusion_matrix(y_test_mlp.argmax(axis=1), y_pre.argmax(axis=1))\n",
    "    mlp_prediction_acc = mlp_output[0][0]/(mlp_output[0][0]+mlp_output[1][0])\n",
    "    \n",
    "    return mlp_summary, mlp_fit, mlp_evaluate, mlp_prediction_acc, mlp_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# mlp_summary, mlp_fit, mlp_evaluate, mlp_prediction_acc, mlp_model = mlp_pre(X_train, y_train, X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# mlp_prediction_acc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### 7. RNN - Recurrent Neural Network (LSTM)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.layers import LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def rnn_pre(df_train, train_rate=0.75):\n",
    "    # Set the dataset for train and test\n",
    "    df_rnn_train = df_train.loc[:,train_cols + ['is_attributed']]\n",
    "    df_rnn_test = df_test.loc[:,train_cols + ['is_attributed']]\n",
    "    df_rnn = df_rnn_train.append(df_rnn_test)\n",
    "    \n",
    "    pre_col_index = list(df_rnn_train).index('is_attributed')\n",
    "    dataset = df_rnn.values.astype('float32')\n",
    "    dataset = np.nan_to_num(dataset)\n",
    "    \n",
    "    # Normalize the dataset, set all the data of the dataset to be in the range between 0 and 1\n",
    "    scaler = MinMaxScaler(feature_range=(0, 1))\n",
    "    dataset = scaler.fit_transform(dataset)\n",
    "    \n",
    "    # Split into train and test sets\n",
    "    train_size = int(len(dataset) * train_rate)\n",
    "    test_size = len(dataset) - train_size\n",
    "    train, test = dataset[0:train_size,:], dataset[train_size:len(dataset),:]\n",
    "    \n",
    "    # Use this function to prepare the train and test datasets for modeling\n",
    "    look_back = 1\n",
    "    trainY = train[:, pre_col_index]\n",
    "    trainX = np.delete(train, pre_col_index, axis = 1) \n",
    "    testY = test[:, pre_col_index]\n",
    "    testX = np.delete(test, pre_col_index, axis = 1) \n",
    "    \n",
    "    # Reshape input to be [samples, time steps, features], here it changes the dimension from 2D to 3D\n",
    "    trainX = np.reshape(trainX, (trainX.shape[0], 1, trainX.shape[1]))\n",
    "    testX = np.reshape(testX, (testX.shape[0], 1, testX.shape[1]))\n",
    "    \n",
    "    # Create and fit the LSTM network\n",
    "    RNN_model = Sequential()\n",
    "    RNN_model.add(LSTM(5, input_shape=(1, len(trainX[0][0]))))\n",
    "    RNN_model.add(Dense(1))\n",
    "    RNN_model.compile(loss='mean_squared_error', optimizer='adam')\n",
    "    RNN_model.fit(trainX, trainY, epochs=10, batch_size=128, verbose=2)\n",
    "    \n",
    "    # Make predictions, trainPredict should be 1D array\n",
    "    trainPredict = RNN_model.predict(trainX)\n",
    "    testPredict = RNN_model.predict(testX)\n",
    "    \n",
    "    # Change the dimension from 3D to 2D\n",
    "    trainX_2D = trainX.transpose([1,0,2]).reshape(len(trainX),len(trainX[0][0]))\n",
    "    testX_2D = testX.transpose([1,0,2]).reshape(len(testX),len(testX[0][0]))\n",
    "    \n",
    "    # Append prediction back to the model\n",
    "    trainPredict_6cols = np.append(trainX_2D, trainPredict, 1)\n",
    "    testPredict_6cols = np.append(testX_2D, testPredict, 1)\n",
    "\n",
    "    # Invert predictions back to normal values\n",
    "    trainPredict_6cols = scaler.inverse_transform(trainPredict_6cols)\n",
    "    testPredict_6cols = scaler.inverse_transform(testPredict_6cols)\n",
    "    \n",
    "    # Calculating the RMSE\n",
    "    trainScore = math.sqrt(mean_squared_error(trainY, trainPredict_6cols[:, pre_col_index]))\n",
    "    print('Train Score: %.2f RMSE' % (trainScore))\n",
    "    testScore = math.sqrt(mean_squared_error(testY, testPredict_6cols[:, pre_col_index]))\n",
    "    print('Test Score: %.2f RMSE' % (testScore))\n",
    "    \n",
    "    final_prediction_train = np.where(trainPredict_6cols[:, pre_col_index] > 0, 1, 0)\n",
    "    final_prediction_test = np.where(testPredict_6cols[:, pre_col_index] > 0, 1, 0)\n",
    "    \n",
    "    # Change dimension from 2D to 1D\n",
    "    final_prediction_train = np.reshape(final_prediction_train, (-1, 1))\n",
    "    final_prediction_test = np.reshape(final_prediction_test, (-1, 1))\n",
    "    \n",
    "    # Counting the accuracy by using basic calculation\n",
    "    rnn_acc_train = sklearn.metrics.accuracy_score(np.array(final_prediction_train)[:], \n",
    "                                     np.array(trainY)[:])\n",
    "    rnn_acc_test = sklearn.metrics.accuracy_score(np.array(final_prediction_test)[:], \n",
    "                                     np.array(testY)[:])\n",
    "    return rnn_acc_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# rnn_acc = rnn_pre(df_train)\n",
    "# rnn_acc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train & Test Data (Call the function)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1. Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Random Forest accuracy: 99.876%\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlEAAAEuCAYAAACu4EdXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3XmAE/X9P/5njr2zsKzA7nLsAst9\nCRstrYpS/aB+tX7koxUon/Krn2prL61XtdqqaPkg1Pqpra3Wo9qWVgWPWq2tB17IDYFFQG72vi92\nN9kjm2R+f2STzZ3JZCaZyT4f/0AyM+97Jq+dTN5vnSAIAoiIiIgoJvpkF4CIiIhIixhEEREREUnA\nIIqIiIhIAgZRRERERBIwiCIiIiKSgEEUERERkQTGRGdosVgSnSURERGRZGazOeT7CQ+igPCFkYvF\nYlE8D4od+0V92CfqxH5RH/aJOiWiXyLd/OHXeUREREQSMIgiIiIikoBBFBEREZEEDKKIiIiIJGAQ\nRURERCQBgygiIiIiCRhEEREREUkgKog6ePAgVq9eHfT+Rx99hOuvvx4rVqzA5s2bZS8cERERkVpF\nnWzzueeew1tvvYWsrCy/9wcGBvDoo4/itddeQ1ZWFr7xjW/g0ksvxejRoxUrLBEREZFaRL0TVVxc\njCeffDLo/dOnT6O4uBgjR45Eeno6zGYz9u7dq0ghY9E/4MSeo53Y+t+34/ALr4fc55PfbULVtv3B\nG06fBl54AQBwassubH/2DSWL6sfR1493Hv0zOs7UoKOrD//aUQGnSwAAVHy8F589/ZqodFwuAf/e\nUYG2zl589tSrqPhkX1zlOlPXiW0H69DX78A7287gwPFm7DrcAABo6+zFy3+34J+P/hn7DtXBcqwJ\n72w7g+4ee8Q02zp78dDDb+D4TbcDnZ3Y+fybOPHudtFlag9oHwBoau/Be7sqIQgC4HAATz0FNDaG\nPP6Dx/+Ghv1fRMyj09qPd7adwYDDhf3HmnHkTBv2H2/G4dOtIfdvbu/BX/71xVAZRPrw1y+hbs+h\niPv8+oe/xQM/34STNR0Y6OnDO+v+hM6qegzYet3/r24QnZ+HJ52+hja/98929+MPb3yOf247g3e2\nnUFP3wAAQBAEPPCHHfhk+0m8s+5P6GntiDlPj5aOXvzsqW14cs1L6DhVjd62s7jzJy/h5NrfuPsu\nisOnW7H/WHPIbR//9hXU7CgXXZYjZ9qw72gTAGD/3/6F3/7ot6jfexiA/7kEAFv2VKG+1So67Wjs\nA05R50s4pz/cje3PvIF/76xES0ev9/3m9h68u7MSQlUV8MwzQIjx2Gd3n8/W3gHsPNSAj/ZV4xNL\nDQBgwOHCO9vOoNPaHzbvQ69twYGX35VU7hPVHd5rSCiVDV3YeqB26I3ycuDVVyXl5eF7rtQ2d+Oj\nfdWijjtW2Y49XwxdRzxjovVsr99+n59qwYHjocdkKBX1nX51rDzdiB/f9yp6WtpxtqIO7zz6Zzj6\nwre/x7aDdThT1+n33unas9h+sB47n/s7Tr63AwBQv/cwtvz6pYhp+Y73Hc+9gYdveRLOfjs+O1CH\ninqfPAQBeP55oKICjQeO4YPH/ya63oLLhRfWvoSXX93jv6G5Gfj974GBAf/9BQH/3lmJF94+gsaa\nFryz7k+wNflfs5JNJ4i44tfW1uLOO+/0+8pu3759+Otf/4onnngCAPCb3/wG48aNww033BAxLaXX\nzvvkUBc+OdTlfb1m1QS/7b11LdjwaX/IbWWLFkHndOLoxo2450AuAODBGwqhT1N+dZyjm7Zik3MK\npp2tRvfkUjR2DODaRaOwsDQHa15yn2w/vyYfxtzsiOkcqe7Bq9vakZOug83u7trAesbCk/d5U3Ow\n75Rt6P1VE/DbtxvR3h38oTd9fCZWXRL+jqTvcX86+gxunHVLTOV86l9NaD47gGVfHoUFU3IAAOtf\nq0OfXcD/d+lonL/rXUxauxa2WbNwbONGv2Pb9h7DkydNMDoH8PPVk8Pm8ecPW1DR1I/LF47E+wf8\nL1KhyvnL1+vR0+8CANz4H2MwaWxG1Hp0Ha3E/x0whk0TAGyn6/DY7qFT9HrhFF7XTcXss5WYOcKB\nN/RTMbejAl//4eKo+fk68vKneFUoxcyzVVj5gwu97//x/WbUtA59qJ83LQdfO38UvqjpxebPhi5e\nS7qPY8ktl8WUp8djb9TD1uduq9KzNcjINOKLzCIAwJPZ+9C2bFnE4z1jMrDNbJUNeGyHM+Q2MWl5\n/u957TmX8k1G3HBRPp55tzmmtKPxXKuinS/Ryg4AI7INuHOZuw09Y/HB9x/H+Yc/w8nf/hZdF1zg\nd+wH5Z3Y/kU35hRn4Uj1UEBw57IiHK7qwfsHOjG5IAPfumxMxLyltEW0Yz3b7/36OGSl62E+7zwA\nwP7t2yFkRD+vQin/y8d40zgN886ewaG8KQCAW79WgHNGpMVU1qM1vdj0WRvycgy4/doi0XUKl66n\njp7XRfazyO3pwom8YizXn8HslReHTcPhFLB2U11Qvr7jwrPN894d8+wYOW9KyPR8x3u71X19XpZW\ngzcHJvrlYdq/HzO++104c3Lw9Zv/BIchDT+a2o3RX5oVtd617+/D862FAIBbrhyLovx0AMD0m29G\nbnk5qu++Gy0rV3r3r27pxwsftPil8ZXOk7ji+1+NmpfcZF87z2QywWYb+lC12WzIzc2NqzBy+OS4\nBcBQEBWYV4PuCwAnQ5fD6b4AzyooANADACgzm2FIQBB1+JXdAICK3CI4OtzRuGlUAczmGcDgCTB3\n1mzkFJwTMZ0622kA7d4ACoizvQfz7nVlAhjqb7PZjPaX/hHykLO9+pB5etY48j3unIYGYFZs5Wwe\nPH5EfiHM5ukAgL7BchaOm4RJevcN1pyjR4PS3PdFIwAHHIa0iPk9/ua/AADp2fkA/IOoUMf1+Fy4\nCsdPgnn+uKj1ON7UA6A9bJoAUGkDgKG07UIGoAOqssZgkqsN0APVWaNj7uPyl3a5088e63fsus1v\n++1nF7JgNptR13MawFAQ1YZsyePK5tNWFSPGIV1wel9PMhoxKVq6g8cH5l/Vux9ATchtotLyKZfZ\nbPaeS+1WByaWTAXQHFvaUWw9sR9AV8jzRdR6YD7l7epxevf3jEWbQwcAmDZiBBCQ1nuH9gDohtXu\nf22bMXM2TrScAdCJ1m5X+DKE6QNRoh07uH3O3HkYlZvpfbvs3HOBnJzY8wOw88/uOzK1GUPXzylT\nZ2LqxDzRZbVYLBg1ehyANpy1Of3LH2t7BNZx8HVDeh7a9O4/lJ1CRsT0+vodwGAQFaosHr5juzB/\nDGaHSdN3vHvzEIba35tHZSUAwGCzwWFwB6HnmEaJqnvrR0PfAEwsmYpzpw8G6SdOAACKART7pDNw\nuAGAfxDVpDf55aXZtfNKS0tRVVWFs2fPwm63Y9++fVi4cKHU5IiIiIg0JeZbLG+//TZ6enqwYsUK\n/PSnP8VNN90EQRBw/fXXo6CgQIkyEhEREamOqCBqwoQJ3uehrrnmGu/7l156KS699FJlSkZERESk\nYpxsk4iIiEgCBlFEREREEjCIIiIiIpKAQRQRERGRBAyiiIiIiCRgEEXyi2HZEw1kQwB00CW7CCRV\nDCeKqk8pGU54we//qq6tIgSXyuuswYt6ygVRslzqfTvS5ZIjxRgM1SBwPIlak01rn3UynzTRLow6\nXWwNJKl0Cb4OxJWdyPYI3E2uKgpBCUtPOda+VQ2FPjiC2jbUPpG2JfvzTMb8ZRsZco+xCHXUVJAn\nYbD41S+GdlVbq6RcEBWNTqfOKmv1+q9KsjSm8h0i5kM/cBddhG2y4EAcogvz/xQQsZtTrK6aIzYg\nkbufxKYX1zUitmO1MBTVGVEQERERqRyDKCIiIiIJGEQRERERScAgioiIiEgCBlFEREREEjCIIiIi\nIpKAQRQRERGRBAyiSHaJmgxNU5PRaZwW5muhMGKZsTzpM2xGIEvZwk9mPCyovdJqL18IKRdEyTJr\nsU9HJvqi4pdbYN4iZk/X3PIccs9YHiW5WIeHlP5PRHDnPxu19D6PVlJv9QMaTpBpnAmBk98OoxnL\nPcVVarSI6aNIzZ3sjzM585drZCRwwnKVxBPiCiHpOul7SEwzlqvrPE+5ICoqvbo6YIhay6VBMlzp\nEvJ5LCaTCFOWxxMwhzsyGXGIWke+b/tqLD6LKtLY0Vowmmp0IgMXuftJ9PUkjnxjPVQLY3H4BVFE\npCqq+IObiEgCBlFEREREEjCIIiIiIpKAQRQRERGRBAyiiIiIiCRgEEVEREQkAYMoIiIiIgkYRBER\nERFJwCCKZJewmXY5wVDCaGDOOwonpmVfFCxHvGQonDDMx7Gql/UBVD4AQ2MQFYrvsi+uBC/74vNp\nFZizmBNAcx92ci/7Em2HGBtISukScR3wXfogng+GaEsoeJawSdiwiqfxVLsaQWieGaKT+7kRad2X\n5H6gyfqBL9PQkHuERaqjKsIJkYWQ0lV+h2jug2vIsAui1DqNvEqLpU0aWfZFzFgM3EeutRHDZZ2M\nYajWoe/bRppbkzKaCNVJsZpqjtj2l7ufRF/z4lr2JdZ1XyRnlTDDLogiIiIikgODKCJKKlV8bUFE\nJAGDKCIiIiIJGEQRERERScAgioiIiEgCBlFEREREEjCIIiIiIpKAQRTJL0GT9PFXXQnEicy0S4Oz\nQIckx4zlvpPUpkq7xED1VVZ9AYOlXBA13K/1w7z6Ual1slW100Kraa1vNVZcTZOtqYdZpyWsthpu\n15QLomThGw0nMTIOzDrRS9AkhNztK3t6CTomDtGWbokoysUrXHMqVkUN/iUaN4XqLIj4YIqU9TDs\nCQogegzEve6L4ocpZtgFUWr9a1WdpdIoOZZ9SUCP6ESs9Ra07Isu9P9jzjtM/ZJxeqh17OvCvtC+\niNVJsbpqjU5kQCL3uSo6ubiWfYlxf8k5Jc6wC6KIiIiI5BA1iHK5XHjwwQexYsUKrF69GlVVVX7b\nX3jhBVx33XW4/vrr8cEHHyhWUCIiIiI1MUbbYcuWLbDb7di0aRPKy8uxfv16PP300wCArq4u/OUv\nf8H777+P3t5eLFu2DEuXLlW80ESUOtT2jAMRkVhR70RZLBYsXrwYALBgwQIcPnzYuy0rKwvjxo1D\nb28vent7Vfu8EREREZHcot6JslqtMJlM3tcGgwEOhwNGo/vQoqIiXH311XA6nbjllluUKykRERGR\nikQNokwmE2w2m/e1y+XyBlBbt25Fc3MzPvzwQwDATTfdhLKyMsyfPz9imhaLJZ4yR9TW1h4xL1tF\nfdht5sF/T548CaAEAHDgwAEYsjJkL2cgq9UKZPq/19BQD4tlqO0PHz6MjOa6iOlU11iD3pOjvbu7\nu0WnaR8YCLs98P3e3l5RaYZSX1cPi8W/vhUVlZhWX4/xYdKsr60FMCZqfgOOAQBAS0tL0LZo5TxT\ncQbZrsYopQfaKyoA5ERMs+tUBYA072ubzQbkuL8C6+npAUziyhTIarMC2cHHOp0u//2s3bBYLKiq\n9m9np9Mp33ns82ukhoYG1ItMNzD/rpOV8FzSYi1b4P4Wi8XvXDpx/ETYfaVqbXVfq8KdL/HWwaOy\nshJtAds6OjoAAH19fX7vHzlyBM1N7muOmD6Opy2iHfv554cwItvgvS4fKC+Hy+cP+lj09vYBufAb\na8eOHYO1Vdy13VPWap9nguXoM08dfXlKaLPZIqbncA7VJdJ+vttqaqphD7NvqM+Onh4bkOGfzshT\npzA1YL/6hgZRdW9tbYXnonXy5EkIthoAwAKnEwYATU1NqPVJ51Rdb1AaLkEIeb4mS9QgqqysDB9/\n/DGuuuoqlJeXY/r06d5tI0eORGZmJtLT06HT6ZCbm4uurq6omZrN5qj7SPXZyf3AmZ6weTWnjwB2\nfhGxHNOmTQOq7ACAhQsXIi0nS6HSDjm6eQ/g8H+vqGgczOaZwEu1AIC5c+dixMTCiOk09p4B9p31\ney+u9h7MOzc3F2hp809zcFugtLS0kHlaLJag47KyhtpWdDkHjy8aNw5m8wy/9yZPnoTx48aFTVN/\nogVo7I+aX9pbLQD6MWbMGOCUzW9byON86jRl8hSYF44P3ifAqXY7cLw5Yllq+g3AwaELd06OO+jS\nAcjOzo5cpggOvbzbe7X2PdbwegPgcHpfm0y5MJvNaB2oBPYMjSuDwSB9XAWMG9/HAIqKilAULd3B\n4wPzr3WkAQcqQm4TlZZPucxms9+5NH3GdODDltjSjmL7qQPAmWqkhzhfvOeKiLL7ljnU+5NKSjAp\nIK0tR/YC1b3IzMwEuoY+PGfPno2a7irguDVyH4fpA1GiHTu4ff78eThn5ND1YeG55wJ5ebHnB2BP\n1g4A/nNnzZg5EzNL8kWX1WKxoLikBNh7Nrj8sbZHYB19+sxTwpycnIjpDTicwKa6sGXx8B3bEydM\nxLwwaYb67MjOzgGcPukAQG3wdX9cUZGound8egwYvA8wbdo0lM0c635hcAeSBWPHosAnHVdWI/Bp\nm18aep3OLy9R50qcIgVpUYOopUuXYvv27Vi5ciUEQcC6devw4osvori4GJdddhl27NiB5cuXQ6/X\no6ysDBdeeKGshY9VIub3UTU+lxYRn9uTSv3txr6lcOQaGsNthOkS9bMPDZ+7UYMovV6PRx55xO+9\n0tJS7/9vu+023HbbbfKXLJl8bvkKLleEHRXI2mcwCQEDOCXXepK5TnK3kLQJy5XvJzGzUYtKJ8HH\nRU84Bcd4FErVWMxM9hFnLE9yVyQ7/0RQc/vHUgYpn01Sr5NxrdCgAE62SURERCTB8Aui1HrbUKbl\nPAjyNGAi+kBMOSMsDRNXEcMezMHn5dM/qdYqkb76HPaPRGiG3Ou+iEwvrvWmYjtWC1/RD78gioiI\niEgGDKKIiIiIJGAQRURERCQBgygiIiIiCRhEEREREUnAIIpkl7D5TdQwkQqR2sVwnqj6jJLlfPf5\ntZeqK6sM1c81qPbyhcAgioiIiEiClAuiNDCthKKGe/2jYftIo4V208KcMr60Vl4tk2vuq+HWZQmr\nroYbNuWCKFn4LvuS4NuLgs6nSwKz1uCtzqjkXvZF5iaStJxBArrJN4t4loCJdmzCb/+n4hiPQrEq\nixoWkTJPbl8Mh6EQcdkXFXzfKLYEUvpKav8mv1X8DbsgShdhBuhk8v1LSZ0l1BAZ/qpJzITl0XMJ\n3EcX9kWMeYfNT3qaqca3LVLtrlGk2qRYVTUnYYv+BuYrtt/jGCCpOLSGXRBFREREJAcGUUREREQS\nMIgiIiIikoBBFBEREZEEDKKIiIiIJGAQRURERCQBgyhSQGJ+oqu2+UKIVCmWCXnUfFLJMHGU4Lvq\ni5rrqhAu+yI/BlEpJhXn4ZAVJ8GRRBOtptI54MLhUNSi4dVpnLE8OgZRRERERBIwiArFd9kXV/Ju\nLwat+pLEsihG9tu3yW+jRJRA8PkbUVDw78Vw3aNYnhq8nR8/Zeospo8iLjuS5K5Qw7InSotYRxVU\nX2wfJPRrQpXdtBp2QZRal2/wK5ZKy6gZciz7koAukLTsi+9SJHFcTcJlzZE3JKXbIqUrp226JEWv\noodEPMu+xHisFj4Kh10QRURERCQHBlFEREREEjCIIiIiIpKAQRQRERGRBAyiiIiIiCQYdkGUWmds\n9SuWSssoVqKKHzYfOWY2TkAdxIzFwH18X8bzE/DwUxeQR8q0RajO1mL/y3Fe+00Nos7aCqJ/kiah\n/BHaMNSWkCWJox+iXvMiXO/UKuWCKFmmMPBJQ5fEWZADcxZVFoV+E6qFn5q6RS5oIqa4SERT6Xwu\neToFPww8zaVks/mVPoE/n6bIzZ1KzSlXXeRuk4jTlKRQ+0cVS8OqLLBKuSCKiIiIKBEYRIXCGcuD\nBN5Wle1rUc5YLjGPZM9YnuAMU5hSVRbztVAyZyyP/tWOsvmrgdpnLBeNM5YTERERUSyGXRCl1ucm\nuOyLjLjsi4h0w7wvOUXp1Dra1VouWaR05bSNy77IklXCDLsgioiIiEgODKKIiIiIJGAQRURERCQB\ngygiIiIiCRhEkewS9VikWmccJlKVGB5UVuuKDgBk/xm9mquqFNXXWfUFDMYgioiIiEiClAuitPCT\nSCUpVX01tWs8ZVHrFBfqp+CEnjKlo9Np63LGsZg4crU0e0whGj4XtHXVISIiIlIJY7QdXC4X1qxZ\ng+PHjyM9PR1r165FSUmJd/unn36K3//+9xAEAXPmzMFDDz2k/b+wfL+XTeJ3tIHPJwiCK0klCbXs\ni0IJqys5aeklYszIdI4puWSMJBp8JiJeStVYTLqR9lG6JwQh8jAeFiNB7au+iCyElOdTpT5/p7Zr\nVtQ7UVu2bIHdbsemTZtw1113Yf369d5tVqsVjz32GP7whz/g1Vdfxfjx49HR0aFogYmIiIjUIGoQ\nZbFYsHjxYgDAggULcPjwYe+2AwcOYPr06diwYQNWrVqF0aNHIz8/X7nSykCtd8n8l/OguMjSx8r3\ngqRlX3zKFV81w637kvjRp9rxrpOrrdUn0pJBar1GksLE9nt8D6XGtrt6rw5eUb/Os1qtMJlM3tcG\ngwEOhwNGoxEdHR3YvXs33nzzTWRnZ+O///u/sWDBAkyePDlimhaLJf6Sh9HS4n8nLDCvnuqmsNvM\ng/+eOnUKwBQAwMGDB2HMzZa9nIG6urqBjCIAQ7c5GxsbYbH0evf54sgXyOxojphOVbU16D052ttq\n7fZPc3/4NAcGBsLmGfh+X19f2G3heO4CNzQ0wGLp8dtWWVmJuro6jA+TZmNNDYD8qPkNDNgBAK2t\nrUHbopWzoqISuYjcTwDQceY0gKyIaXafrIbv3zpWmxUYHI42mw3Icd9xj7WPrVarJ2u/Y50OR9B+\nFosFlVU2v/edTpds57Hvbf2GxkbUi0w3MH/rydqw22JNy2KxoLpq6Fw6fvy45LTD8Vyrwp0v8dbB\no6qqCq0B29rb2wAA/X39fu9/8cUXaGpyn1MuEX0cT1tY9lugj/CheujQIdSajN7rcvnBg3Dm5UnK\nq6e313uueBw/fhx9HRniyjpYz8qqqqD3Qu0n1qHDh5CXE/pjuKenJ2J6DudQbSLt57uttrYWzjD7\n+o533zIEXidGnDqFaQH7NTY2iap7S0sLgGIA7s9ZfW8dAOBchwNGAM3NzajxSedUfV9QGoJLCHm+\nJkvUIMpkMrkv1oNcLheMRvdheXl5mDdvHsaMGQMAOO+883D06NGoQZTZbI64PR47z5QDp4fKG5hX\nW04FsO3ziOWYOnUqUOt+/ujcc89F5qgRCpV2yInX9wKD1zOdTgcIAgoLC2E2zwZecn84zJ4zG6Om\nTIyYTou9Ethz1u+9uNp7MG+TKRdobhtKs8wMvFwX8pC0tLSQeVosFvf7Lw192GVmZsZcTt3LtRAE\noKioCGbzLL9yTpo0CeNPjffuG5jmoYoOoNYWNb+0f7YCvX0YPXq033gKe5xPnSZPngSzOXI/AUBF\nlwv4oj5iWepdGYDltPe1Kcfk/STIyckB4L6TE2sfH9m0B3AiKG/D35uAgYGh/EwmmM1mdDirgV1D\nf6AYDHrp48qnrQD/Ox9FhYUoipbu4PGB+Tfqc4C9x0NuE5WWT7nMZjOa7ZXAXve5NGPGDOCDltjS\njmJXxUHglC3k+eI9V0SU3bfMod4vKSlBSUBaHx+1AFW1yMjMAKxDgfPs2bPR1FsLHD0FfaQ+DtMH\noniOLTNDrw8RRA1unzdvHgryh/6AXXDuucDo0bHnB2D/X3cC8L/rOWPGDMyZco64sprNsFgsmFRS\nAuzu8L4Xaj9RPHWcOw9j87OD+gwAsrOzI6Y34HACm+rClsXDd2xPmDABC8Kk6Tvefcvgud5482hs\nDDq2sLBAVN07t50Eqt3/nzp1KsyzC90vBmOKsWPHYqxPOrqcZuAT/z9kdXqdX16izpU4RQrSon6d\nV1ZWhq1btwIAysvLMX36dO+2OXPm4MSJE2hvb4fD4cDBgwfdAQgRERFRiot6J2rp0qXYvn07Vq5c\nCUEQsG7dOrz44osoLi7GZZddhrvuugs333wzAODKK6/0C7JomErclOVEFE2q/OJRhnoIw/x5L1XP\nSA9ocqxGDaL0ej0eeeQRv/dKS0u9/7/66qtx9dVXy18yIiIiIhXjZJspRqk/tNT0B1x8P0pTUUU0\nRBPNFur5GhXTVmk1TqbG1sR5oEUablgGUUREREQSMIgiIiIikoBBVCg+D7cJruQttRLElcwlaAJe\nK5WwupKT9CBmInrJN494HpaNdmy4+itWRw0+WBo3paosZlxEWnZE4a6IlrzqH4KWQcRld1RRfXGF\nkFJWqdVTRbP4GH5BlGqfm0jd2ZETToYGTEgfiMkkcLwq/GwHh94Qnd//U6tlIg291Kqp9uiSFCaI\n7vc4Lo4xH6qBwTj8gigiIiIiGTCIIiIiIpKAQRQRERGRBAyiiIiIiCSIOmM5UayEBD0YqbZfaRCp\nUgw/nVLHL8LCkGPZF58nlYfDr/8CJaLOu3fvxu233+5dR7e/vx/XXHMNVq9eHf1gn/L96le/giF7\nNPo6e2FrOoJzpi8NecjevXsxYcIE6PV6/P73v8eaNWvkqIZoKRdE6Yb9T9uUqb+qmlWnk3xBVVU9\nNEQL7aa5c19jxdUy+X5dyU4T48tf/jJ+/etfAwDsdjuuvPJKXHvtteEPiHDuZo4ch8yR48Juf/fd\nd7F06VKUlpYmPIACUjCIIiIiGu7+59M/4cKT24HXf4znO3oAAHl/0QH3Z4Xc/5J+Bxb22P3ey4IT\n18PgfvH6j4EbbgC++tWYymG1WqHX63HjjTciow+o7ezHuPO/jed+twG/6WyGy+XC7UYjFgF4r7kZ\nTy9bhvz8fAwMDKDsK5ehp/U0Oqt3oajsv9FZvQcNZz7DsmXv4NJLL8X8+fNRVVWFe++9F4899hju\nvfdebN68Gdu3b8cTTzyBjIwM5OXlYd26dTh69Ciee+45pKWloba2FldddRW+//3vx9yugRhEERER\nkWx27dqF1atXQ6fTIS0tDQ888ACef/55LCyeAquhDGcrd2LGtJF47qlfo6OjA988/3y8CWD9yZN4\n45NPkJeXh+9+97t+aTr6rWjUczToAAAgAElEQVQ/9TG+Yv4mnn3u+3j88cdx/vnno6SkBBs2bEBa\nWhoA91eWDzzwAF5++WUUFBTgz3/+M55++mksWbIE9fX1eOutt2C327F48WIGUURERBTsxUtuxIuX\n3Ii3H78WN9/1DwDAw+Z0lK36fyH3/3RnJX7/2kG/9/6frgH/FooAAG8/Pvh13L/+FTVv36/zPJ5/\n/nkU5J0DdAP93Y04aGn0PiflANBiNGKk0YhRo0YBABYuXIh+n+MHetqQnlsIgyENOp0Od999d8i8\nOzo6YDKZUFBQAAA4//zz8X//939YsmQJpk+fDqPRCKPRiMzMzKj1EIO/zgvFd9mXJD58GLTUiorK\nItsTqLLXKfkPiyaim3yXa1Eyu3BpC0o9GzIcH/ZVqAfF9FHEvBVf92X49XUg1T/cLrZ4IuuhH7xu\npZvG4CsXXYaNGzfiueeew5V2O0Y7HOhyONDe3g4AOHTokN+xadnnYMDWApfLCQC47bbb0NTUBJ1O\n59eOo0aNgtVqRXNzMwBgz549mDRpEgBlnpscdnei1PrwqX+xdFBDMKBZciz7IkMxouYhYgmiwPGq\nC/P/mPMOm18ciaYY37ZIuXaJtOxLqtVVY3RJuvSL7ve4ln1xHzuy+Muor/sE3/zmN2G1WrHK5UI6\ngAdnzMBNN92EkSNHwmj0D0+MGSaMKl2CAwdewYoVH+OrX/0qCgoKMH36dNxzzz34xS9+4c1j7dq1\nuPXWW6HT6TBy5Eg8+uijOHnypORyRzLsgigiIiJSxqJFi7Bo0aKg9zdu3IhPntwEdAJ6gxHf+/HP\n8KU5he6NY8cCAJaMHo0lTz3lPebA8WZ8dHonskeXAgBGTjwPs0eV4A/P3OzdZ/ny5TCbzQCAzZs3\nAwAuuOACXHDBBRHLtX37dhlqy6/ziIiIiCRhEEVEREQkAYMoIopK7c+/EpH8eNpHxyCKkotPsWqS\njpdX2cg3m3ZguuwjSpBhfB1nEEVEREQkAYMoIiIiIgkYRBEREZEsbrvtNjzzzDPe11arFVdccQWO\nHTsWcv9NmzZhIMY86uvr8dFHH8VRSvkwiArF9ynapM4SLkR8nUyylUTmOsndRNLSS0Q/+c5YLv15\nBCHaswxhqsIZy+WT1BnLI01YLmNZkpG+1ik1LmIrg9gdh/Zcs2YNXnnlFZw6dQoA8Mtf/hIrVqzA\nzJkzAw5xH/PMM8/AFWOZdu3ahf3798dwlHI42SYREVGKumnt+97///qQHek+r3319juC3vsUY/zS\nufDc8fh2lKghPz8fDzzwAH7+85/jjjvuQG1tLR5++OGQ+7766qtoaWnBHTk5eKqlBY+fOoV93/gG\nXC4XbrzxRhROMeNs5Q501VoA6JCZNxFFJV/Gs88+i76+PixcuBB5eXnRG0FBw+5OlGqXffH9vzqL\nqB1yNGACOkFMFjqdPuB1bMeHTzjmDcOQLuR/U0GKVSfFJP8OVEQiLjyXXnopJk+ejPvuuw+PPvro\n0OduwKE33HADxowZg1/bbPg0Oxu1vb14+eWX8Ze//AV/+MMf0GPrRmfNPoyduwzFF/0I6aaxEATg\nu9/9Lr72ta/hsssuU6CCseGdKCIiohT1x59fjmvu+gcA4I556ShbdXnI/d7bVYXfvVru994laMF7\nKPKmAwB4t0ZUvsuWLUNfXx8KCgpE7X8iIwNHuruxevVqAIDD4UBLcyMKz12OjjOfYqCnHZmjSqC2\nIJNBFBERESWFTqeDC8AUux2LRo3CLzZuhMvlwlNPPYWCwvHorP4jxs67DnpDGmp3P4+u3ELo9QVw\nuWJ5kko5w+7rPFJeop4NVtOD9kSqFcN5oupzSu4fociamkaosH/PO+88fNdkwqU2G7INBqxatQrX\nXXcdACArOxsZIwpRs+Np1Ox8BoZ0E3JzizB9+nR8+OGHeOedd5Jcet6JIiIiIpktWrQIixYtirrf\nhg0bgBdfBADcN20a8PTT3m3lJ5oxsngRRhYPpWOwtmD27Nl47733AAAWi0Xmkscm5YKo4f7A5HB4\nKD2u56mHQwMpQAvNprW+1VZptU2uoaGxIaYamzZtwj//+U+crWtGTa/7C7B1FX/Dmgfuw8KFCzXd\nsCkXRBEREZF6rFixAitWrMAnv9uExysyAQD3/8+XsHBuUZJLFj8+E0VEREQkAYMoIiIiIgkYRIXi\n8wsGwaWiXzOoqCyy/chD5b+4kbL0QiJ+AOOXRRzPEwi6yJeAcPVXrIoq/PWQ0pSqsphkI+2jdFdE\nS384DIWIy+5oqP5SftUptXpqaxYGUUREREQSDL8gSq/SXwH43E1QaQm1Q4ZfeiTkxyJiMgkar77j\nRHohwx2r4R/JyE6Xuqu+RPwlo9Z+5ZhqktX6ors9jvER6zUrnmtcogy/IIqIiIhIBgyiSHZa+i6f\nKOXFMmO5gsWImwwXFsH3LoqqK6sM1V+bVV/AYAyiiIiIiCSIGkS5XC48+OCDWLFiBVavXo2qqqqQ\n+9x88814+eWXFSlkTNT/FaqihkP143pkg897SKSBdtNY3/LZI+1hlylEww0bNYjasmUL7HY7Nm3a\nhLvuugvr168P2ueJJ55AV1eXIgUkIiIiUqOoQZTFYsHixYsBAAsWLMDhw4f9tr/77rvQ6XTefYiI\niIiGg6hBlNVqhclk8r42GAxwOBwAgBMnTuCf//wnfvzjHytXQiIiIiIViroAsclkgs1m8752uVww\nGt2Hvfnmm2hqasK3vvUt1NXVIS0tDePHj8fFF18cMU2LxRJnscNraemImFdffWvYbebBf0+fPg1g\nGgDg80OHkF6bK3s5A3V1dQFpBQAAl+ACADQ2NsJi6fPu88XRo8i2tUdMp7LKFvSeHO1ttXb7vd6/\nf3/YfR1OR9g8g/qjvz/stnBcg7/gaGhohMXS67etqqoKtbW1mBAmzeaaagB5UfOz99sBAG1tbVHr\nEKiyshIWfUvEfQCg89QZAOkR07SeqvV/bbUCWYP/t1mBbHFlCtRt7QYyioKO9fyB5GGz2WCxWFBZ\n6T+uXC6XfOfx4HgHgMamJtSJTDcwf1tlQ9htsaZlsVhQ5XMuHTt2THLa4TQ3u69VDkfo8yXeOnhU\nV1ejJWBbW5v7OmL3Of8A4OjRo2hsdJ9TghC9j+Npi/3798NoCP8szOHDh1Gfa/Relw8ePAhHXZ2k\nvHp6erznisfxE8dh7wx+xjcUTz0rKiqD3gu1n1ieOvoZvL719vZFTM/hHPolW6T9fLfV1dUBYfat\nCvHZ0dvTG3SNGXHy5OCn45CmpmZRdW9ubgYwEYD7czatvx4AMN/hQBqAlpYWVPukc6axLygNQRBC\nnq/JEjWIKisrw8cff4yrrroK5eXlmD59unfbPffc4/3/k08+idGjR0cNoADAbDZH3Ueq3ZUHgZND\ngyEwr45RNcAn+yOWo3TKFGDwejxv7lyYCkcrU1gfp/5uAQbjAb1OD8CFgoJCmM1zgJfcH6SzZs7E\n6FlTIqbT7qgCdvsHknG192DeJlMu0DwUUJSVLQQ2hb6gGQ3GkHlaLBb3+y8NBQaZ6ekxl1O/qQ4u\nCCgqKoTZPNuvnMXFJZhQNcG7b2CaR2q6gMquqPml//t9oKcX+fnnAGd6/LaFPM6nTiUlk2A2F0et\nR1WvDjhUE7EsjUYTsGfoA9xkMgFO90+1TTkmQHD/UjvWPj726l5gAEF5G//RDAwGkACQnZ0Ds9mM\nTqEG2Dk0rvR6vfRx9ZJ/YOj7gHXh2LEojJbu4PGB+TdnnAJ2HAm5TVRaPuUym81o8zmXZs6cCbzf\nElvaUeyt+hw4WQFDiPPFe66IKLtvmf3eH2zX4uJiFAektfXEfqCiB+np6YBt6A+RWbNmod1eD3zR\nDZ1OF74MYfpAlMFjy8oWIs1oCLt9ztw5GDd66FuQc889FygsjD0/AOUv7Qp6b8b0GZg3Ncq13VvW\nMuzfvx+TJ08CdrnHhF/dY22PwDr69uVgv2VmZUZMb8Dh9F6DQ5XFw3dsjx8/PmyabSE+OzKzsvzT\nAYDWVgQaO3asqLpbd54GKtz/nzKlFOZ57j/kMHhjZsyYMRjjez060QJ85J+fTud/7RF1rsQpUpAW\nNYhaunQptm/fjpUrV0IQBKxbtw4vvvgiiouLcdlll8la0ETQRVkrLFn8/h7T8C8VVEGOGctlKEbU\nPESUM3Afv5c6SJ7rJlzWyRh6ah3tfk09jM7JYVRVVdIlaQIr0d0ez4zlsR6rgbEYNYjS6/V45JFH\n/N4rLS0N2u/WW2+Vr1RENGxob3o9IiI3dd6WISIiIlI5BlGkgMTcW9DgCgFEiRfLiaLmc0qOZV/8\n/q/myipDUPtFU+3lCyHlgigNfIWqqOHxPEMc38nrh0UDyU4L40prfaut0mqbfG3NXlOEFi4wYaRc\nEEVERESUCAyiiIiIiCRgEEVEREQkAYMoIiIiIgkYRIXi+wsBlyv8fkoXI/C1in65IFtRZK+TGtoo\nsWUQFHwoM1xNFKuhisZ44ihTZ0HEQ9CRcla6K6Kmr6KhoFhbROwAhfKMgegiSGogaRUUVPYMOoMo\nIiIiIgmGXRCl1p9B+95M0PCvPdVBjgZMQCdIWvYlzP9jzzvM+0n4Cbdah3sqn4epXDet0yXpDpTo\nMRHXsi/K7p8Mwy6IIiIiIpLDsAuiBJcKvmgOwfcrZa0/FpKo8od9RkyOAiSgEmKecQvcRwjz/9jz\nDvN+Eh7EUOtw1/p56BWiImrqf9FkmbF86NaGWvtX7DM/0h5DCn+Q6PTiaLjoz8EJkV6q0rALoohI\nXcQ8AE1EpEYpF0SJec5ERCLxpyGDwFKIq5syZVdJkwCIVpbIBZVlfESV2MbSKfjnmi7gX8XF9byF\nigapGCoobqQiaK05I5GrLrK3ScQOkDkvCZQtgrQHgZP1zFg4KRdEERERESUCgygiIiIiCRhEERER\nEUnAIIqIiIhIAgZRofg8qJvMpVbUtOxLYNaylYTLvsSfW1KWfVEoTy38pllmSlVZzLhI6rIvcW5P\nJOWWOUpGpuIpO+sBl30hIiIiGraGXRDFZV+GAY1Mc8FlXzx5quBP7hBS+TxM5bppHZd9kSWrhBl2\nQRQpT50fiUTDVAzftaj621Q5Cuf7qazmuiokmY+EiKL28oXAIIqIiIiCaTCoSbSUC6I0cPdPUUrd\n/lTTbdX4vsZSUUW0RNGH1+VJW2t9q7XyaplcX1OzxxSi4XMh5YIoIiIiokRgEEVEREQkAYMoIiIi\nIgkYRBERERFJwCCKiIiISAIGUaH4LvviSuZSK/55J7csga9lKovMP6GV+xe5UuqZiF8F+y/poeAv\nWxL9E+dh+JPqZNY4UnMrXa5o55aq5jRSqCwqX/UlhmVfEnedVGzJKYmGXxCl1p9S+pRLpSXUDhn6\nOCF9IGb2/MC6yDazfdgpyxNOrTOW+zaGWi8bUkX6yT+nXki2ZJ0PIvs9nvER47HJWEEhVsMviCIi\nIiKSAYMokl+CbsOr6W4/kWqlyokiQz0Ev/+nSLvEINYmTHgLaXCspl4Qpf67f4oaDjOWx3c7Wb5i\nDCeaaDaVLi4ejrZKq3FyNbaqLoQpRMPtmnpBFBFpitoeFCUiEotBFBEREZEEDKKIiIiIJGAQRURE\nRCQBgygiIiIiCRhEEREREUnAICoU32Vf1DRvRRLLEpS1XEWRe9kXWVOTll4ieknQDZ26SuYXLm3F\n8lTT+ZYoSi0pIuZn4xHXfVG4L6Ikr6ahoFRRIn2+qKH6YvtAypxbUuunhnbxxSBKJfxWQuMvvuMj\nx7IvCegDMctr6ALmPvJfBkF6IcNlnYyhp9ZlX3zbSAvLT8Qi0tBLrZpqT7LaX/Q1L46LY8yHamAw\nGqPt4HK5sGbNGhw/fhzp6elYu3YtSkpKvNv/9Kc/4Z133gEAXHLJJfjRj36kXGlJExL1kajOj14i\nlYnhlo6q7rwHkmPGcp9PcTVXVTFqr7TayxdC1DtRW7Zsgd1ux6ZNm3DXXXdh/fr13m01NTV46623\n8Morr2Dz5s3Ytm0bjh07pmiBo+HimcrUX03NGt/6lyqqiIZoodk017caK66WyTZhuUzpUACtnbs+\not6JslgsWLx4MQBgwYIFOHz4sHdbYWEhnn/+eRgMBgCAw+FARkaGQkUlIiIiUo+oQZTVaoXJZPK+\nNhgMcDgcMBqNSEtLQ35+PgRBwC9/+UvMnj0bkydPjpqpxWKJr9QRNDedjZhXX3NH2G3mwX9PnzkD\nYCYA4PCRI8hsqZe9nIE6OzsB41gAgNPpAgA0NTXBYun37nPs+DHU2LsiplNZaQt6T4727u7u9nt9\noPxA2H2dTmfYPAPf77fbw24Lx+Vtn0ZYLH1+26qrqlBTU4OJYdJsraoCkBs1v/5+d7u3t7VHrUOg\nqqoqWIytEfcBgK6TlfCcguHStFU2+L3utnYDGUUA3OcmssSVKSjvrm4gvTDo2IGBAb/9emw9sFgs\nOFPZ4/e+yyXIdh77foXU1NSEWpHpBubfW9cSdlusaVksFr9z6eixo5LTDsdzrXKEOV/irYNHTU0N\nmgPPg1b3uLYH9PexY8fQ0OA+pwQRZYinLQ6UH0C6MfyXIUeOHEFzbZr3uvz5559joLlZUl5Wm817\nrnicPHkSLmuNqOP3798Pg16HiooK73ty9NmRI0fQVJPm957nfOjr7Y2Y3oBz6LyJtJ/vtvqGhrD7\nhvrs6OvrA3L808k9cQLTA/Zrbm4RVfempmYA4wEAZ06fRuaA+/o2b2AA6QBaW1tR5ZNOZVN/cCJC\n8LVHyZgimqhBlMlkgs021LgulwtG49Bh/f39uP/++5GTk4OHHnpIVKZmszn6ThJZag4BJ6xh8+qs\nbgC27IlYjtIpU4DBc3XunDkYWVykTGF9VPxjPzDYzAaDHnA4UVBQALN5LvBSLQBg5oyZGDtvWsR0\nOoUaYGeH33txtfdg3rm5uUBzm/fthQsWAptDB5cGgyFknhaLxf3+YJoAkJGeHnM59a81AE4nCgoK\nYTbP8StncUkJJjZM9O4bmObxph7gVHvU/DLe+wCw9iD/nHwgIIAIeZxPnUpKSmA2lwTvE6DWkQYc\nqIhYlpas08COobu/uaZcYPBzz2QyAc4IZYrg5Bv7gL7gY9PebgH6hi5c2TnZMJvNsOlrgR1DAaVe\nr5M+rnzaCvD/Gq6goAAF0dIdPD4w//YRVcCn5SG3iUrLp1xmsxlnXdXALve5NGvmLODd5tjSjmJ/\n7SHguBXGEOeL91wRUXbfMod6f+LEiZgYkNb2UweAM9VIT0uDdxABmDlzJrpdTcCRbugQoa5h+kCU\nwWMXLliIzIwQH0GD2+fMmYOJBbnet+fPnw9MnBi8vwiHX9kNuPzfmzZtGhbOGCuqrGVlZThYfsB9\nk2BHiOtHrO3hU8cJY3P9+sxzPmRmZUVMzz7gBDbVhS2Lh+/YHldUFDZN3/HukZmZ6Z8OAJz1v1EB\nAGPHjhFV9969FcDgfYAppaUwzx/nfpHmDiRHjx6N0T7ppJ9uBT5s8U9E53/tEXWuxClSkBb1maiy\nsjJs3boVAFBeXo7p04diUEEQ8IMf/AAzZszAI4884v1aj4iIiCjVRb0TtXTpUmzfvh0rV66EIAhY\nt24dXnzxRRQXF8PlcmHPnj2w2+347LPPAAB33nknFi5cqHjBiYiIiJIpahCl1+vxyCOP+L1XWlrq\n/f+hQ4fkLxURERGRynGyzVB856pQ0SzhyZzDJagsSiWsuvSSX4So+Sn48+BwdRGU+rG3BueJiZdi\ns2GL6KNIeSvdE1rqaaWGZcR0VdBAooug1qUdEoBBFBEREZEEDKJUwn+iQO1OPAYk7mZC2Dtzciz7\nkoA+ELXsS8A+fkuRxDPpaNj8pKcplVpHe8qckrHMWA6ot66yXFh8ZiyXITUl6BS8gEb6NiMxy75E\nOTagfGodir4YRBEREVGQYfjtesxSLojSQuSqJKXuJMiZbrxpxXO45pYGUQktLMCrtb7VQpumCrmG\nhsaGmHZouGFTLogiIiIiSgQGUUREREQSMIgiIiIikoBBFBElFZ9dJSKtYhBFREREJAGDKCIiIiIJ\nGESF4jM5huBK4lIrAV90qGrZF7nKInOd5G4hacVLoS+oEl2VJI1xNZ1bsqUr4lfjkfJWukmitXky\n+yRY4suiptpHI6WvAj/fxB+nrukQGESRArR0+hOluFg+4NR86soQVAlhXwwPqopLQ1F9AYMxiFIJ\nuZbzIMjTgAnog7iXfYkv83Ab4klVEvUOd12I/6WGSGOPk4AOV8H9HjKkiWu9qdiO1cIEuqkXRKm/\nzRWlVPVlnbE83uPjm7I8ztyHKS00m14LhRzCoZhI8jQ2A0yFaPhkSL0gioiIiCgBGEQRERERScAg\nioiIiEgCBlFEREREEjCIIiIiIpKAQRQRJZX2ZoYhInJjEEVEREQkAYOoUHyXfRFcSSxHwOtkLkGj\nVNZyL/siczmlLE2Q6El3BQXnWAlXf8WqmLRlX5KSrSd3hdIVMy7Uex9QTSVTbGmeCAlLXRZFTmLr\nLaV9pLapmOWMEolBlAqpbIyEFe6zO1EfSGHzkSGoSEgfiJkcMmgfeaYsDztfeRIGn1rHu/8qAmot\npQghTpRI565qqyrHsi8+lVNDkBKKTsFyRQraQvd7iP3jGCBRD+WyL8k37GeUVegKKGuycScWz0k8\nzMeHRGw20jLZxi/PA2Vo+AKTckEUERERUSIwiCIiIiKSgEEUERERkQQMooiIiIgkYBBFREREJAGD\nKCIiIiIJGEQRUVJpb2YYIiI3BlGh+E74lcxZwgNfJ3H29MA50GSbE032KcZln7I8EYfERdApdxqH\na05BqQlzkjVjeVJyHcxbqdmwRfRRMuc2jJq3iqJrxRZsSEamMRBdBElTlsd+iJu65pRiEEWyS9S5\nr9YZh4lUJYYPOFWfUypfIkoT1F5ptZcvBAZRKqHzX2MieQWJgWpLKceyLwmonJjZ0wP38Rsmcc3c\nHuZ9ySlKp9Zx5FsujZySshhGVVWlZLV/qHxDxjTxLPsSY+20cN6lXBClhUZXklLVl7Nd4170JZ41\n48SsV0dBhv1ySgrgEkSJI9uqL+wyZWi4YVMuiCIiIiJKBAZRRERERBIwiCIiIiKSgEEUERERkQQM\nooiIiIgkiBpEuVwuPPjgg1ixYgVWr16Nqqoqv+2bN2/Gddddh+XLl+Pjjz9WrKBEREREamKMtsOW\nLVtgt9uxadMmlJeXY/369Xj66acBAC0tLdi4cSNef/119Pf3Y9WqVbjwwguRnp6ueMGJiIiIkkkn\nCJGnCH300Ucxf/58XH311QCAxYsX47PPPgMAfPjhh/j000/xyCOPAAB++MMf4pZbbsH8+fPDpmex\nWGA2m+Uqf5DfvbQX71nqva+/M6rdb3tf3wA29haE3IY3Xnf/O28+nhu9CADwzcxGZGUpHxRur+7B\nF7kT/N4rMg7ga7ndeK4jHwBwg6EeeSMyI6ZzqC8Tu3qz/d4LqmcMPHkbIMDpM9vKt/I68Oezo8Ie\nFyrPjrMdGJU3ypsmAHzn4+fw3Fe/E1M5PcePMw7g6txuv/fmZ/Zi0cndwKmT7p2vu97v2MY2G97W\nT4yanye9XL0T3S5D1Lr51mlBZi/Oz+qNWo+zXX141TkuYll6eu34W1+h9/XU7nqcynUfM6W7AWdy\ni6LWJZSt1b04njs+6Fjfenh8Z1Q7DvdnYGdPTtD7UoTKw5vm538HLrlE1PGB+ff22vHXwbaKdSzd\nPKodz/uOy1Ht+LwvE7sHz6Vrc7vwj+4RMaUdzbtWE2oG0kOm6TlXxJTdt8y+78+pPYILTu4AJk8G\nFpb57ftm1wi0OIP/hr7S1I2T9nSctmeELFdg3lLawnPst/I6kK4L/vjxbF82ohNjDM6h6/LlVwAm\nU8z5AcCW6n5UDJ4rHktyrJiWbhdV1v/J60B3ZzvqswqxY/A8CHXexDru/mtEJ0YbnCHPifyes7h+\nfPilvpwAXgiRb6hx4Xnvop4qzBqfGzI93/HukdtvRXeGyT+Phnpg5053XoPX7vOt1VgwMXrfHK3r\nxrbsEgDAV7J6MDezz73B08e5ucDSy7371w6k4d/W4PJ6ymIw6JE3txAXXnNF1LzjESluiRpE/exn\nP8Pll1+OSwYvbEuWLMGWLVtgNBrxj3/8AydOnMBPfvITAMA999yDZcuW4YILLohYGCV98MYhbO+L\nfPEhIiIi7ftP+wmU3Xip4vmEC6Kifp1nMplgs9m8r10uF4xGY8htNpsNubmho1wxhZHDuXPmIu+x\nV1DUWAf92LE4Z1xwhF9f04YRI7JgGukfdaPfDjQ2AiXF6OnuRcfZHoyfeI5iZQ1UcboZEybkQ5+e\nhiobMDlHgE4H9Fj70N5mxYSS0aLSqbLpUJQJtDW0wWTKRG5edvSDwuhx6NAxAIzLFHDGpsPoDKDH\nARRlCXC6dKiwuqBvbkbexAK4oEPXAFCSDRj0wbF5Y2MTCgsL4HTpsK/WhhntlcibNxPNzV3ISDdi\n5Dni/sp0unSo6gGmmIbyGHDpUNcLTMoZfK+yChg/HkgLHuI1Va0YMzoXmTkZYfNwCUCFTYcpOQLa\n7e47cDqdexmEczKC6zbg0qHCBqTrfcogQl11K0blm5BtCn+H8cgX9ejNzMG0iXkYmSbgzOkmFE8c\nDWO6we//sao43YyMDAfGTRjnfc8p6HCsC8hNAwZc7jHomeR9T7sOk7MFdNU2YfLkMdAbpP0uxeHS\nYW8HkHO2DbMmjIAxzYDPTnbiPEMnsktLok4v3TmgQ78TGJsZ3M51NW3IG5mNnBFZosrSNaBDrxMo\nyBTQ1tyJuhYrppWcg6zB/vCcS+kGATU97vGfZZBvfa8zVl3I88VzrkTS092Ljg4bHOeMQWEmkDFY\nrgGXDrW9wOT0AaCuDphUEnSsMDi+S3KAtn73epd2FzAxW/CWqzgbMIY4jwGgs82KfrsDY4vyYq6z\n1aFD94D7GhJKr1OH1sYkxV4AAAduSURBVP6hsqCnF+joAMaPC7m/WJ5zxWkwoqkfKM6O3o/dAzrY\nHEBhluDtE98x4XHWroPdFXpMhtLr1KGtH5gwWAa7E9h2pgtLppggCEBlZQumlBZEnci7sU+HbAMw\nIm0oX881O62jHRmZaRiZb0J/rx2NjZ0omTwmYnqeuvWc7cIX1Z34yoIJaOzTwWQEcn3yQE0tMGYM\n7Do9Guo7UDJ5rKh6A8Cpila4RuZher7PtdnpAqqq3HdNA+pc0+Pug9IcATUVTSiZNAYGo/vaYzAY\nIExepGhMAUS++RM1iCorK8PHH3+Mq666CuXl5Zg+fbp32/z58/HEE0+gv78fdrsdp0+f9tueDMbM\nDJx/1VyYzd9Kajmk8L1/t1imdOR0YYj3Yimn7y3ReOonx/FiXBTj/pG/hJIusD8viLAt1nRD3aYO\n17ZyjquLA16HGltqoNS5FC19pR97ACK3udL1ToZ46+TpE6XaZonP/xNxfQvFt25XJSAPOSj97VY0\nUYOopUuXYvv27Vi5ciUEQcC6devw4osvori4GJdddhlWr16NVatWQRAE3HHHHcjICP/XPREREVGq\niBpE6fV674PjHqWlpd7/L1++HMuXL5e/ZEREREQqxsk2iYiIiCRgEEVEREQkAYMoIiIiIgkYRBER\nERFJwCCKiIiISAIGUUREREQSMIgiIiIikoBBFBEREZEEURcglluyp2gnIiIiikW4ZZgSHkQRERER\npQJ+nUdEREQkAYMoIiIiIgkYRBERERFJwCCKiIiISAIGUUREREQSGJNdADm5XC6sWbMGx48fR3p6\nOtauXYuSkpJkFytlHTx4EL/61a+wceNGVFVV4ac//Sl0Oh2mTZuGhx56CHq9Hr/73e/wySefwGg0\n4v7778f8+fNj2pfEGxgYwP3334+6ujrY7XZ8//vfx9SpU9kvSeR0OvHzn/8cFRUV0Ol0ePjhh5GR\nkcE+UYm2tjZcd911eOGFF2A0GtkvSfZf//VfMJlMAIAJEyZgxYoV+N///V8YDAZcdNFF+NGPfhT2\nc768vFz0vrISUsh7770n3HvvvYIgCMKBAweE733ve0kuUep69tlnha997WvCDTfcIAiCINxyyy3C\nrl27BEEQhAceeEB4//33hcOHDwurV68WXC6XUFdXJ1x33XUx70vivfbaa8LatWsFQRCEjo4O4ZJL\nLmG/JNkHH3wg/PSnPxUEQRB27dolfO9732OfqITdbhd+8IMfCJdffrlw6tQp9kuS9fX1Cddee63f\ne//5n/8pVFVVCS6XS7j55puFI0eOhP2cj2VfOaXUnSiLxYLFixcDABYsWIDDhw8nuUSpq7i4GE8+\n+STuueceAMCRI0fwpS99CQBw8cUXY/v27Zg8eTIuuugi6HQ6jBs3Dk6nE+3t7THtm5+fn7Q6as2V\nV16JK664AgAgCAIMBgP7Jcn+4z/+A0uWLAEA1NfXY8SIEdixYwf7RAU2bNiAlStX4tlnnwXAa1iy\nHTt2DL29vfj2t78Nh8OBW2+9FXa7HcXFxQCAiy66CDt27EBLS0vQ57zVahW9r9xS6pkoq9XqvRUI\nAAaDAQ6HI4klSl1XXHEFjMahGFwQBOh0OgBATk4Ouru7g/rD834s+5J4OTk5MJlMsFqtuO2223D7\n7bezX1TAaDTi3nvvxS9+8Qtcc8017BMVeOONN5Cfn+/9gAV4DUu2zMxM3HTTTfjjH/+Ihx9+GPfd\ndx+ysrK828O1s8FgCNv2iYgJUupOlMlkgs1m8752uVx+H/SkHL1+KB632WwYMWJEUH/YbDbk5ubG\ntC/FpqGhAT/84Q+xatUqXHPNNXjssce829gvybNhwwbcfffdWL58Ofr7+73vs0+S4/XXX4dOp8PO\nnTtx9OhR3HvvvWhvb/duZ78k3uTJk1FSUgKdTofJkycjNzcXZ8+e9W73tHNfX1/Q53yotg+3r9wx\nQUrdiSorK8PWrVsBAOXl5Zg+fXqSSzR8zJ49G7t37wYAbN26Feeddx7Kysqwbds2uFwu1NfXw+Vy\nIT8/P6Z9SbzW1lZ8+9vfxk9+8hN8/etfB8B+SbY333wTzzzzDAAgKysLOp0Oc+fOZZ8k2d/+9jf8\n9a9/xcaNGzFr1ixs2LABF198MfsliV577TWsX78eANDU1ITe3l5kZ2ejuroagiBg27Zt3nYO/Jw3\nmUxIS0sTta/cUmrtPM+T+CdOnIAgCFi3bh1KS0uTXayUVVtbizvvvBObN29GRUUFHnjgAQwMDGDK\nlClYu3YtDAYDnnzySWzduhUulwv33XcfzjvvvJj2JfHWrl2Lf//735gyZYr3vZ/97GdYu3Yt+yVJ\nenp6cN9996G1tRUOhwPf+c53UFpaynNFRVavXo01a9ZAr9ezX5LIbrfjvvvuQ319PXQ6He6++27o\n9XqsW7cOTqcTF110Ee64446wn/Pl5eWi95VTSgVRRERERImSUl/nERERESUKgygiIiIiCRhEERER\nEUnAIIqIiIhIAgZRRERERBIwiCIiIiKSgEEUERERkQQMooiIiIgk+P8B/EUWBfBoFEkAAAAASUVO\nRK5CYII=\n",
      "text/plain": [
       "<Figure size 720x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "RF_fit, RF_pre, RF_acc, rf_model = randomForest_pre(X_train, y_train, X_test, y_test)\n",
    "print('Random Forest accuracy: {}%'.format(RF_acc * 100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2. Gradient Boosting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gradient Boosting accuracy: 99.57000000000001%\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlEAAAEuCAYAAACu4EdXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzsvXeAHNWV7//tyVGTZzRZM6OcNQ3G\nBmNYsGwvPL8fNrsk/7TrXfy8ttfGb81zXIO1WE8I44AX24AJwsgGSxiTbCNAIBAIhFBLI2k0Go0m\n55xz6H5/dJiq6uqu6urq7uqe7+efmaq699xzY52+VXWOyWaz2UAIIYQQQnwiKtQKEEIIIYSEIzSi\nCCGEEEI0QCOKEEIIIUQDNKIIIYQQQjRAI4oQQgghRAM0ogghhBBCNBAT7AItFkuwiySEEEII0YzZ\nbJY9H3QjCvCsjF5YLJaAl0F8h/1iPNgnxoT9YjzYJ8YkGP3ibfOHj/MIIYQQQjRAI4oQQgghRAM0\nogghhBBCNEAjihBCCCFEAzSiCCGEEEI0QCOKEEIIIUQDNKIIIYQQQjSgyog6ffo0duzY4Xb+zTff\nxI033oibb74ZBw4c0F05QgghhBCjouhs89FHH8VLL72ExMRE0fm5uTnce++9+NOf/oTExETceuut\nuOaaa5CdnR0wZQkhhBBCjEL0zp07d3pL0N/fjx07duD111/HP/7jP7rOX7x4EefOncNNN92E6Oho\nNDQ0wGq1YtWqVV4L7OrqQkFBgS7KyzEzt4C/vHUBU68cxlhiKnILMu0XXn8daGsDVqzAWyfbYXr+\neaR/ZBvwxS8C6en2NA0NwAsvANu2of7QMZw/dBwlbReAb34TkNmJk+XwYbuc0lLgkUeAwkIgNRUY\nHASeeALYtAmoqQGOHAFOnwaiooDMTMw/+Cu88pX/Qs6ZDzG9Zj0O//BBlI93I2rjBjQd/hDVB99H\n6aXrxWX9+78D588Dl1/uOmW12nDw/WZkLkuA5dApWN9+GxnmTfaLQ0N2HZYvB37/e2DrViA62mt1\nGjtGcK5pALkZSXj1WDMmpubQsvsBFD3/NAZu/We8GFuGljMNGK2tR9drR3CqfwH5r7+EePM2e92e\nfBJITgayslx9PzAyhfv2nUBBdjKyn3sa7w+aMBWTgKy0RKCuDvjyl4GyMnvbyTB4sQWH73kY5aeP\nIursWcBsRk/fGI4++AwqKvJgSkwErrsOeOcdICsLKC4GmpqAq68GPvc5vH5uAEkJsUhNigPGx4F/\n+Adg2TJg9Wp7ATYbRvb8DG/s/RvKWmtw2pSBwWf+jPbXj2LgX76C3M9uBzIzRTr1Dk7iuZ8dQPcf\n/oyK6z8Bk8lkvyCov4vz54FXXwU2b8YbB08h/uDfsOzSbcDICPD448DGjUBsLGC1Ao88gl8cbMLf\njjaheEUOlp2x4OCrZ5C3phTRnZ04uOtx5H37G0iYngBycuxj64UXgNpae/862bcPiI8HHD9y5uat\nOPheE7AwgRUli+08PDaDJx87hO5v/QAX+6ZRdMl6xMZEw/bii7j7xUZEvXcU54ZtKCrJRmyMYOzY\nbMCnPgVMTAAWC1BZCTjbQELf0BR2P3kcNU2DWL08Gfjv/8Z3X2xBeUkGsrKXLSYUzEcAwNQU8Nvf\novp8F9rPNSJ/s2OtGR52tdvhM92IiY5CWkq8e8E2G/Doo/Z2SksDAJx74TBaP6xBwZbVOFnbi2ff\nqEPx1ABSX/8brPPzOPiN3ci8dDOSMpbh0M/+gITn/4TUpx4HqqqAv/s72foBABYWxPO/vx/YuxfY\nsgWIsf92nR2bwMGfPY382RHEf/C+fW1woGadbGgfxvmmQZw7WY/0gy8j+dJtgMmE3r3P4J3GMVS8\n+hxMf/g98MlPLs7z06eBI0cwfegwXn32CAq2rIalaQQX24bQ0jWKFQVprrGRl5mEhLgY+zj86lft\na8arrwItLTh7oRudCZnIz072qqOIvj7gySdRl7UCFztHUZSbKr7+2mtAezua4zNx5t2zKD36mn0M\n/+53wLFj9vlRXu69jJ/9DHj6aeAznxGNv7n5BRx8rxl5y+LQ/90f4cSHDSi76pLFfM3NwLPPuo3b\n2uZBNHaOoDAnBV1dXVh+4gQO7juEjN3/haQLNfY1pbYWZ+76GTrH5pG/aSVwzz3AX/4CbN/ucQ40\nvXUCZ/96FKUf2WAvvmsUP9rzF1yVZ8JE4jK8caIN5YVpiDp7xn6v2LBBVs67+9/E/LkaZGxc7Rpz\nDZ2jOP/aMXQkZWF6dh5ZaYno7BvHB+e6UT4/DBw4YB+HDz8MzM8Dt94KbN0K63N/xsHjbcisPYOq\nX/0BTxysw5XvvYijF4dgPX0GGZUb7YXabPb5dvw4uvvH8f6ACRVF6fL90dVlHzfJycBVV8H24YfY\n+3Ybao9UYdOF44v3oEceAR58ELjhBlGb2Ww2HDzWgneqOpCfnYy3T7aj8O2DiIs2AffeC5w8ia6V\nKwNqU9ir4Xk+mmw2m01JQHt7O771rW+JHtmdOHECv//97/HAAw8AAH75y1+ioKBAZGjJEejYeW+d\nHcVbZ0ddxztvKwIAmC+xT5i33vkAP3u+CwDw8s9vsOt04gQAoPKyy2BaWMD5ffvwnVP2Cf7CLz6P\naJsVdb/+NcYuu0yxfGc5jbt2ofyHP8RUWRlqnn0W5f/n/yDjrbfQ+b/+FwoefVSUp/muu3D+xeP4\nzSe/ijWdtZhJTkVzWiG++ep/I+OJe7Fzv13fH342EzGpSQCAhPp6bLjlFpH+AHCudRLPvjuIpPgo\nTM5YAQD/95oYzC1fjrLvfQ+Zhw650rZ961vove02r/XZ+XQ7AOCSlck4UT/hOv/yz2/Av/3Lr9GZ\n4W7ofKThOHZsisLExo2yOv73y90YHJt3yfnst16wl3Vbkav9pHmEPPHwcbQuK8B/vPIArjn/Fhp3\n7cL3B9diEjG469hvkXtNJVbs2iWS45Rbl7cSd37hp4iKAu6+pQjF992H3GefFZWXfPo0fv+3dpwu\n3YJ/fXsvnrjqX0Tlv/zzG9x0+8lzna72/mrJEPI+vsljHzl1eXP/S/jFB/Y8PytvR/aLLyLz9dfR\n9cUvovPrX0f6oUNI2f1zfOlLv3Xlvf2tx/H41bdjRW48rnz1Gezb9nlsbanCj5/b6dZO5w4cwHR5\nOeLa27HpBvFY/+DCOF6xDKMkJw7/uj3Xlefx13rR1j/rOv5Y1gw+/ekKzNzyddz7P7/nOn/JymT8\nj49kuI7TDx9Gxbe/7TpuuP9+DHswMu7/cycmpu31LsUYMloaUFVqN/ic8xUQzMff/Q6TGzYg/5FH\nUPDoo6LxAgBl3/8+Ml9/HdX/+lV8P/3TbnKcLDt6FKu++U3MZmfj7MGD9nSO8b3ztiLX/4C9j99d\ndTnu++x3sHyiHzeZk/DftUmua8K2lCPzlVdQdtddmCovR82BA1j1ta9h2fHjaL/jDvT80z/Z8z95\nGC/HrcJHGo7jrhd34/Rrr2FeYpx7Q6hv9lg/fljRj/GtW3H/C90YSUrHPX/6Eba1nhbNc+fYe/Lj\nO/DcR27EZcMX8UH64g/fb92Qj+qWSbx2agRlefH452tz3PoWgFsfqMHZBp7yOnVzXn/m119Az3/d\nhYrvLY47b20eMzCALZ+293/jrl0Y+sxnXNferRnFoapRrDMN47zNfsO/02xF6poSAMDWq65C9MQE\nLvz2txivrHTlE44PAJi95d+x+39+H3nD3Xjsia+41nmnzvdeacPGz30OAFD72GOYEP6QEeCU+4PP\nLENc5jLXcdFAG0xrKtDWP4vrLknHV2+7GgBw8t13YUtIEMmYX7Bh1/4Ol34ZBw+KdHGVJRjbDzz/\nn6hoOofB7duR+frronTO8Z4/1ImuDLvBIFz/fnxdChbS05Fy8iTWfPnLAIDP3fEs5mNi8aVP5aIo\nO86tnutuuw1JdXWu4+Nll+DHn/uhXZd9/4HYf/lH9N10k6vv63/6U4xcfbUrfWvfDJ54vU8k8+rz\nb+HOVx5wHZ8+eBDzQXgCpnvsvJSUFExMLN5UJyYmkJqa6iWHsjJ68NYFC4BFI0pa1pq1GwB0yeuz\nsAAAWJeXB2BSlGb1smWAD3qXx9kHVGJTk11+aysAoGBqyi3tiuhovL0sBwDQkFuB+ZhYAEB/SjY+\naTYD+/8CANi4bj2S8xw7GpOL+gnr2DHRAGDQdUMHgM3l5cC6dUD74qILAMVWK4qV6uSYfFPWBAAT\noktyBhQANGeXYkVMl33HS6CjM8bR4NMvyuaT9pWncbJzmV2ngRR7W5QnJGDSMZRn+kewIkr8qp9Q\nzmiifafDanWcHxhwT9fejsZcex/0Lls0MLzpNim4oaUlLrNfHx/3WpfCnAIA9nyr09OBDvuCmD8+\njnyzGXj3XTTHiR+jO/XpG7NiwGRfVBtzymR13FBUZB+zgvZw6lHVUQ1gGN1DcyLddh94WSRjwpoA\ns9mMl1KzROdnbIniOh09KrpekZzscb5MCNqqzZaCnvw1bvoBEM9Hs9m+yyXAldbRbqlTs0C6jBwn\nZ84AAOL6+xevO3Qxm82u/50MptgNmu7kbOQuSwAwLV++HI4fK4mNjfZ0jY0AgKL5eRQ58h15wt5m\nzdmlAIAtq1fbd7ChMh6YQN/+1GxUJE0CK1diJMmu50iSfbdNbp53Om6S3dHiNXvN2vWo62sEMIL+\nMatdh2PHPKrg01peX+9T3rnoGFRIdnK85mlqcv1bnpAgGn/vN1YBGEW3Lcl1rihnOVY60zjG1prM\nTPG4FYwPi8WCwWT7mOhJt69tznXeycbiYtf/a7OzPd8zHHLXVqxERnmx67g9qxhxw/YfmImpi4ZB\n5ebN9h1NAdMz84DDiDKbzcCbb8oWJRzbMzP2OZXpGI9CnOPdaUABQGd6vuv/rWvX2ndWm5td55z3\nqsKScpjX5bkXLjCgAGAoZfGH13hCKrYAKBG00cqUFFGbzVV3ARAbUa1ZJaLjqOnp8IydV1FRgZaW\nFgwPD2N2dhYnTpzANue2OyGEEEJIhOPzTtTLL7+MyclJ3Hzzzfje976H22+/HTabDTfeeCPy8mQs\nUUIIIYSQCESVEVVUVOR6H+qzn/2s6/w111yDa665JjCaEUIIIYQYGDrbJIQQQgjRAI0oQgghhBAN\n0IgihBBCCNEAjShCCCGEEA3QiCKEEEII0QCNKLUoO3YnLkxBaS+b0xFfJPSNwetg8lU9g9cnZBi8\nXQynnc7tJZRms8rI1rM8A8qyQT4MjWEw+PyQI+KMKF2GiFxH+tq5HmImKcoRZLOZxOlVROiRbwBn\nPqlOgR6wntoggNgUyjRJbxNy6YWxm7RUQXW7ekjnqb88lOGxzir0UKuptN1satpRBarKV9MeAEyK\ncyuI41FalteydfoxIMmvNBcA7zfV0N/P/OgvifKmQPW9v40kZ8hpFR3o8a23QSu5v6nOJxkXivM+\nwEScEaVECO7rqgr2+Zd+yCoSBhilbRQNOuX00gXCJPo/AIuH1rYLZZsHrGxB+xplTCmhUk+vNx6p\nCL3qrkWOL3mM0Ed66KBWhu7VVbme+FNHHw0eA/SoIkvOiCKEEEII0QMaUYQQQgghGqARRQghhBCi\nARpRhBBCCCEaoBFFCCGRTOg/syMkYqERRQghhBCiARpRhBBCCCEaoBGlFm6Jq0arEzWfy3H9EwF9\nY9Q6OP1c+qqfUesTakLRLj749VHl0DeYBFAf2boa0Mu4rrKM4EvLG0YbfyqIOCNKF8+0ct6gg+Sx\n3CZxpyhKb7UqFyvnnmwpeSxXcM/mZgwoeSzX4O5N9Y3IUzoVHrp98vTtNY22y26itXosV5MvQj2W\nu5YW5yW9PZarGLveSgz17UzV2PCY2bOTWn9wG2N+9pm3tcJnySH0WK7F+LZpDA/mFkWCHsuXCIoD\nXCcjjejSNrqEElDSQ5VnaM+Lts9e7kVFyeumuen8aHO/eytAc0HYvgELG6I3aj2We1lv3OpKj+VB\n1UG1w3Kd66t6PQniXA+HeUcjihASUkK940EIIVqhEUUIIYQQogEaUYQQQgghGqARRQghkUwYfvFE\nSLhAI4oQQgghRAM0ogghhBBCNEAjihBCIpkw+EyckHCFRhQhhBBCiAZoRKmFL2eqRqsnWp/R6k3e\niBi2Dna9GPZFJwzeLoZTT2eFRF7cl2DYF6N1rxuGG4DK0IiSQxj2BRpv1FrDvghDjpjExoga1/ry\nTq+XUNgXX8sMSNgXtQkVBHgL+yLI67HOKhRRSuG87s3DtT1BAPtaZdgX2BTCIhkt7Ivjr+Y1RoqW\n/N6aJMQ3NL9Kdwvv5I8woZgghn3R656jF3qGqIH7/U09Qb6PKbDkjKiQuZH3N+6Xj/KWNHqEXtDj\nN5tSn6tILx0XYr206+hJM80tF4lhXwTtGw7hJwCobwsvQ0fNuNQEw76oE6FzOvXlqlxP/JnrPt/n\nNBcVNJacEUUIIYQQogc0ogghISX83oIIM8LwPRNCwgUaUYQQQgghGqARRQghhBCiARpRhBBCCCEa\noBFFCCGRjBG+WiMkQqERRQghhBCiARpRauEXLurR7ETNN3RzUmgEjFoHp1r0WK4PbBff0N1juVD0\nEvRYbvRdyTCcHxFnRBlmjIRIEa+lBlsnw3TGIm7O3hQ8locUo+gBFT7vgqGrr85LfcyvKz54LCeB\nR7e2NvJNPgCOSXVxOhzhRJwRpQvCiWKSOaeXbKWkkvQ2q0YdPJVp5AXBKGhZfdW2q1/9IhOayJMc\nrwumQhgiT+eDOXb8bU+jo1cMSEl+NbsO3sIahbw1/TF6gzUW/C1H65ruLyr1Vow1uJgwoHqIsgQ7\nfJkCS86ICtmvP39/Rfsob0mjR+gFPealUp9LL6vYrRDqpTqEgkw6T6ppbrpIDPsi+jEVJvNNTk+5\n/vcl7gvDvgRVB7Ui9K6u6vXEr7nu28JqgB5VZMkZUYQQQggheqBoRFmtVtx99924+eabsWPHDrS0\ntIiuP/HEE/j85z+PG2+8Ea+//nrAFCWEEEIIMRIxSgkOHTqE2dlZ7N+/H1VVVdizZw8eeughAMDo\n6CieeuopvPbaa5iamsINN9yA7du3B1xpQkjkEPJ3bwghRCOKO1EWiwVXXnklAGDr1q2orq52XUtM\nTERBQQGmpqYwNTUFkxGeSRNCCFmE6zIhAUNxJ2p8fBwpKSmu4+joaMzPzyMmxp41Pz8f119/PRYW\nFvBv//ZvgdOUEEIIIcRAKBpRKSkpmJiYcB1brVaXAXXkyBH09vbijTfeAADcfvvtqKysxObNm73K\ntFgs/ujslYGBQdmyzI7js9VnPerjTHPx4kUApaI0zS0tGFCht1NGW1sbigXyN0xPIwHA4OAgMiV5\nOjs7Pco7efKk6//q6mrE93YAAJJra7FWoj8AtLaNu8k4V1OD6dlZrJ+aQqLgfG9vL9pU9sXY2Jiq\ndE66urowIqOjt763WCyu9lNKK6S9vR2IW+s67ujsRKEXucLzK0dHkSYpb1l9PYB0r2V6062ntxcW\ni8VjHzl1aWxsBBw90tTcjLypKSQBGBoeRqPFgpzWVo9fpywsWL3qBwAXLlzAeFISEmtrsV6iR3f3\nsGxdpHKnp6Zk6zo+Pi46n93SIpoxra2t6FPZf8I6yrXTxfp6jFosWDEwgCyZtOsc7TY6OiYrx0lm\nUxPKPFxXGmttba2AZOZ6y5Pb3i6a/5vn5hALoK+vD62OfNMzM0D8Yp6z1dWYHZbvFzW0tbVh+Kz7\n+tbT04N2yRrnROo25dy5c+jtsa/3CwsLsFgsbn0rxBcdN8/PI9ZLXrk52t7ejiKV5cV2d8N552lv\nb0ePIG1f3xAA+/3L+fyluaUF45YkUdmNjY0YkinDU7kdHR2itaampsY11xoaGjCs0D41588jaUJ8\nz3LOwZ6eHte5qqoqLCxbJko3v7DYdxaLBbmCe46S7tMzM0jwqpk7Z8+exWx/P9Lq67FScq2+vh7R\n0x1ueeT6VIhzbDrTSdeN+o4pVboF0qZQQtGIqqysxOHDh3HdddehqqoKq1evdl1LS0tDQkIC4uLi\nYDKZkJqaitHRUcVCzWalptXOOxdPAo2THsvatHET8GK3V31WrVoFtMyKzq0oKcEKH/QuLl4czmaz\nGUiwD9nMTKkJBRQUFADVdbJyKisrgT8dBABs3LgRy4qX2y/MLuon1L97qhE4MSySsWH9emDTJiAx\nUXQ+NzcXuUp1erodAJCamgr0DXhP68AGIH/5cuSvXTRszGaz3Zgxm10ypUj7weM4ceS3Oe6+RYWF\nQN/i5cKCAlVyzGYzIFiYXOm6u4Hz/fJle5IpqFNebq79+vS0Vx3Ky8uB810AgLLSUlf/ZKSn29Mf\nO4Y2D+VHRyt/WLtmzRrAbAaiFtM69TjbdQ44X++mW/RzXcD8gus4ITERZrMZr0reXEpJSRHXSbKI\nlRQXo0Sh/+SQa6dVK1fa65GVJZ/W0W7LlqV6lYOaGvfrDl28jUsAKC4uASQ/ULyuY2+9JU4Xazcf\ncrKzkePIdzT+qCjLpo0bgYoKAFicK96Q6FtcXIziTZuAw1Wi83l5ecjzIMsUJTbT169fj7axFuDC\nOKKjo+06nDjhUQWf1vIY8e1GTd6iwkLRsdc8bYuzpaioCEWCtMebTwMXJ2ASzIXSkhKslcgrLyuz\njzUngvEhd6MulOi3fv161/8V5eViWUIcctevW4estWWivoyOjgLmF5CXl+c6t3XLFkBy75ibXwD2\nd7j0w5EjskWJx7a9vxPi42XTemPTpk1AaSnQ7j5PVq5cCfOG5T7LlI5N6bphTewG3la+7wTSpgC8\nG2mKRtT27dtx9OhR3HLLLbDZbNi9ezf27t2LkpISXHvttXjvvfdw0003ISoqCpWVlbjiiit0Vd5X\nTEbxLBGq9xC8lUuP5e5+SpaCx3I95Cj5kKHHcu9lyY6z4KhCoNv7uobuMnosDwmKRlRUVBTuuece\n0bkKx68lALjjjjtwxx136K9ZKBF6Cdc7PpuCHKE3VpskBp1mL9FLyGO53rGhvHl09phHbbP60S/i\nJAoey73I03tIBYQI91ju0lpvj+Uq1i5vJYa6ObXMvcXM4eGx3NuaHtAq6OyxXMu9SXp/U53P5xyB\nhc42CSGEEEI0sPSMqFDtxyrukCza1yab8kvDhnnkZER0aRsdfu/4qoeqkB2CceK7RmqU0JgthLsG\nQZgLYTPd1D6m8dLmbq9EMOxLkHVQK8OAr4wo4eNUDwe3SUvPiCKEEEII0QEaUYQQQgghGqARRQgJ\nLWGwZR/WsH0JCRg0ogghhBBCNEAjipBIhzsRhBASEGhEEd3R6v/Dd3T24RVKAlkHf2RrzRoJfRII\nQtEuPpRpuF7Tvb2Efo8CXJ4BZdmM/nsqDNcNGlFqCcPOJYQQQkjgiDgjyjBPLkKkiNdiGfYFJqkt\nvBTCvuiAIUKqRFjYF8OEqFoC6NXShg6DwrAvISHijChdEIZaMen8yEgx7IuwSySPxRj2RRG/QkXI\nydNw41UdAsFTMjVhX4T/e1JRh/41xAiJ9LAveq0x0vyqhq63RKFtzyUf9iWQ7a932BctKkjvb77k\nE+DNcWwwWHJGVMh+/Sn9ihYMBFWDwkC7FIZDh7bRZWIq9bn0suxuhc3zsR86evIErLnpItBjuWhO\nhst802GHQc241AQ9lgdLhLZy1c5DPxSMxJ2tJWdEEUIIIYToAY0oQgiJZIywQ0NIhEIjihBCCCFE\nAzSiCIl0uBNBCCEBgUYUIZGO0b9ao5EXWIze/4SEMTSiCCGEEEI0QCNKLfw15wPBCfuiuw+vUGLY\nOujsm2ypY/R2MZp6OreXyL+anGwDhmrRU5befvR0x+jzQwYaUYEiVB7LvV6kx3K3uwQ9lqtC0YeM\nATyWK1oAhvNYToKGXo1t5Js8PZaHBBpRhBBCCCEaoBElhzDsi8w5vWQrJpXqYtUYvmUJhX3RHw1h\nX9T+evOjX0TRgDzp6GP4GF+uqw5towcRHvbFNcZ0Dvui5tGNtxJD3ZwewxmpyhwmYV88relAYB+t\n6h32RYuuJo2vfUh30Rj2JbiE7AmJv8FTfZS3pNEj9IIOavjc53KPfKRPHwXH/my1e1JNc70jMeyL\nsH3DZb4FQk+GfQmqDiG7RaldT/zoE1/DaRmhS5VYckYUIYQsKcLhTkRImEIjipBIhzdRQggJCDSi\nCIl0Qv1yCwkt7H9CAgaNKEIIIYQQDdCIIoQQQgjRwJIzokK2s61QsM9qGXiLXuqaIaDlyJWlQ9m6\naO9Ln3tIK/3MW3jsj/dhj94VNEtUWYAc/r6zFaCxpvbzbt+EBnhe+PvJfQBk+iXHlzw+uvQIiMdy\nke8RbbLU94EmvwKOP+5zTm49kf1iz48+sSnNdWl6497mXEScEWXS4yVagQzdP/n2QT+TVJcoFXm9\neeBW4UXZF7GByRRY3BYFRY/lvs9ik14fKXtpP+Elj58mq2h/pRQm19/ArWaKkj2NX2kypdoYzGO5\nm7r+6qchv7cchpi+Wu+i0k/rdaqMGrckPsnztqb7KjoQ7iCEza/3gLDZtMmUjokQD9SIM6IIIYQQ\nQoIBjSg56LFcMWkY7LLqBD2Wa8mnK6rHaXiOSt3WGF8fncB7iwX+yaNSASF04BqkcozusVzUBV7z\naPQ8rofH8hBDI4qQSMdgiw4JMux/Eskw7EtwYdiXJYBRQi8o9bmKd2KWQtgXv9uaYV8WYdiX0GKU\ntUdTuQz7ooUlZ0QRsuQIh09cSOBg/xMSMGhEEUIIIYRogEYUIYQQQogGaEQRQgghhGiARhTRHZvW\nT1e1lANExjsfRq2DVrWMWp9QY/B2UXY7EGQCqI+sewE9yzOgLGkEBMNhtPGnAhpRagnDziWEEEJI\n4Ig4I8own0SGSBGvpfrzGbqBwr6YbFY/8kpPKIV9CSFG0QNQ3pHyQ1fVP0+UXEYEUEefURH2xUC9\nG/Ho1da+fqIfVAId9oXIEnFGFCFEgpGMMUIIiSBilBJYrVbs3LkTFy5cQFxcHHbt2oXS0lLX9bff\nfhu//vWvYbPZsGHDBvzoRz/SLdhjyBD92tD5vRtfQq1I3i2yedqBCUnYF2P2sZpQF77J05JJbTo/\n+kWYxlOdVclRTqJrPk1lRXgSa47rAAAgAElEQVTYF73e7ZOGfXH+42VO6BzIwyeU4s/6NZfDJOyL\nt/wBrYHOYV+0NIM0rJlP+QyE4k7UoUOHMDs7i/379+POO+/Enj17XNfGx8dx//334+GHH8azzz6L\nwsJCDA0NBVRhQoiPGPkRBAk87H8SyRg97IvFYsGVV14JANi6dSuqq6td106dOoXVq1fjvvvuw223\n3Ybs7GxkZmYGTlsdMGzYF8FAUPXcPdx3+wKJLm2jw8RUDPWj5r0Zm8djxXeAtKC17SIw7ItwcTZF\nhcl8U9kW3tYYtycJDPsSXB1CVQ/Vu1PB6xO3NdKAKD7OGx8fR0pKius4Ojoa8/PziImJwdDQED74\n4AO88MILSEpKwhe+8AVs3boVZWVlXmVaLBb/NfdAX594J8xZltlxfObMWY/6ONPU19cDKBelaW1t\nRZ8KvZ0yWltbUSKQv356GokAhoaGkCHJ09nZ6VHeqVOnXP/XnKtBwlAvACDp/HmsE+ofHQ0AaGkd\nd5NRU1ODKasV6yYnkSQ439fXh1aVfTE+PqYqnZPu7m4MSXWE9763WCyu9lNKK6SjowNI2iQ6LvQi\nV3i+YmQE6ZLyUi9eBJDotUxvuvX29sFisSCpttat/sDiGGlsagRgn1vNLS3ImZxEMoDhkRE0WCzI\nbmlBrIcyrCperr9QV4fxZcuQWFuL9RI9urtHZOuyMD8vkjE9PS1b1/GJCdH5zKYmCGd9W1sbejXM\nc7l2qq+vx4jFgtL+fmTLpF3raLfRsTFZOU4yGhtds1p6XWmstbS0AK6Ropwnp61NNP83zc4iDkB/\nfz9aHPmmp6cg7ODq6mrMjC/OX1/Xyfb2dgyeOeN2vqe3F+2SNc6J1KVBTU0NenomAQDWBSssFguy\nmpuxwkOZvui4aW4OccK8Jy2IEtxk5eZoe3s7ilSWF9Pfjy2O/zs6OtAtSNvTOwwAsNpsLqu9ta0V\n05J2aWpqwqBMGZ7K7ejsFK01586dwwbH/42NjRhSaJ/aC7Vomx0VnXPOwd6eXte506dPY16yQTG/\nsNh3FosFOYJ7jpLu0zMzSPCqmTtnq6sxOzyMZfX1WCW51tDYiPi5Lrc8lfD+I8k5Np3tL1036jun\nVekWSJtCCUUjKiUlBRMTE65jq9WKmBh7tvT0dGzatAk5OTkAgEsuuQTnz59XNKLMZrnpog/vN1YB\nDYv6SsvavHkT8IK4s6VpVq5cCbSLb1IlJSUo8UHvkpLF4Ww2m4EE+5DNyJCaUEBBQQFwZsTtPABs\n27YNeP4QAGD9hvXIKC+2X7Au6mc2m11GVN9sM3B8WCRj/fr1wLZtQFKS6HxOTg5ylOr0dDsAICUl\nFegd8J5WwPLly7F83TrXsdlsthszZrNLphRpP3gcJ39oFR0WFhYCQ5JjFXLMZjOQluaebmAAOCOv\no0eZgjrl5ubYrwsMEjkdysvKgQv2hXJFaamrf9LT0uzpLRZ4Mq+jTMrfhKxZvRowm4GoxbROPc71\n1ADnxtx0i36+B5ibcx0nJCTAbDbjkER2SnKyuE41NaLrxcXFKPbUf9L+F9zI5dpp5cqV9npkZ4vO\nu9I62m1ZaqpXObh40f26Qxdv4xKA/T3QZvEc9bqOHT0qThdnNx+ys7OR7ch3LOE9UZaNGzcCa9YA\nwOJc8YZE36KiIhRt3gwcOiE6n5ebizwPsqS7TuvXr0fPVDtwvh5R0VF2HaqqPKrg01oeK/5JYK40\nI0phh6+oqEh07LW8rsV1vbCwEIWCtCfbzgIXxkVGW0lxCTZI5JWVlaFMeE4wPuRu1IUFBaLjDRs2\nuP4vLy+3j1s5HHLXrlmL3E2rRH0ZHRMDzM0hNy/XdW7Lli1Abq5IxNz8ArC/w6Uf3n9ftii5sZ0Q\nHy+vlxc2bdwIVFQA3d1u1yrKy2HeXCCTyzvSsSldN0zJvcBb/YpyAmlTAN6NNMWVuLKyEkeOHAEA\nVFVVYfXq1a5rGzZsQF1dHQYHBzE/P4/Tp0/bFzxCCCGEkAhHcSdq+/btOHr0KG655RbYbDbs3r0b\ne/fuRUlJCa699lrceeed+NKXvgQA+MxnPiMysshSJTgey3X/cjKUREIdhERaffSC7eIbOreX6Kti\nOdkG9DKupyyjflXtIgznh6IRFRUVhXvuuUd0rqKiwvX/9ddfj+uvv15/zYxGGHYuIQCM8cItIYRE\nIHS2GShC5bHcW7ER47HcH4NWkpcey1Uh/UowNEootIfSuKDH8qWLXh8YGmEeeIIey0MCjShCIh2j\n76IayFgkhBBfoBFFCCGEEKIBGlFyCEOt6BWSQUa2MpIXtK0aw4QEIuyLQXcP9H5xUos8qe8dLwl9\nOy9M4uF/f+T4ct0WzH1+o++k+U1gwr6o2+HzEhImwM2udeypEx4eYV+8rRWq1xFtBatLpzbsixYV\nNH6A5HbvMbrHcqITiguawDuymiFpUCPGEOjQNrpEa1fSQ3pZNr3UelVlPiniKb6l5pYLZZyzQL17\nJyoiTOabWo/lXsaOmwR6LA+qDiELqqE6ofY+8fmdMgN0qRI0ogghhBBCNEAjihBCCCFEAzSiCCGE\nEEI0QCOKkEjHCO+KEEJIBEIjSi1G+FIiTLCZEJQ6uUqIhPYzah206mXU+oQag7eL4dTTPeyLUPQS\nDPsShB9UH3zwAT72sY9hx44d2FFQgJuKi7EvPV1dZkE9f/rTn+LtN/6G6ZFODNS97jHLsZoa9PT0\noK+vDzt37vRTe9+JOCPKMF/RhEwPL+VGiMdyf3D76m4peCzXYQHW5WtFv5Xw3h6KOhrMY3k4fHkU\nKZj0amwDTAOPGMhj+Uc/+lHs27cP+zo78fv2duzNyMBolDZzIyGtAFmrt3u8/tdjxzA+Po6cnJyQ\nGFGKsfNIEDDKTZsQQkhE8C9vP4krLh4FnvsmHhuaBACkTwzbLzY1uaW/qvYItrWcEp1LnJ3CjSf+\nbD/YHwfceivwd3/nkx7jJhOibDZ8sagI8dYhtB/7LQou/Vc8Gj2FXx4/Duutt+J/JybisqkpvNrY\niIduuAGZmZmYm5tD5ceuxWR/A0ZajyG/8gsYaT2OroYjuKGkBNdMTGDz9DSaurvx3e9+F/fffz++\n+93v4sCBAzh69CgeeOABxMfHIz09Hbt378b58+fx6KOPIjY2Fu3t7bjuuuvw1a9+1feGlRBxO1GE\nEEIICR3Hjh3Djh078E8FBfh2fj7u6utDstWKS02JKProlzHa9iFSbSb84SMfwW9+8xvck5uLOQB7\n3n8fe/fuxeOPP46EhASRzPmZcQzWH8a2jf+A51tbMWsy4dLJSZQtX4777rsPsbGxAOyPae+66y78\n6le/wu9//3tceumleOihhwAAnZ2dePDBB7F//3489thjutSVO1GEkNDCnVhCdGfvVV/E3qu+iJd/\n9v/hS3e+CAD4r+d2orKlCigrAy5eFKV/e+0n8OvtXxOd+/uqV/DK1r8HALz81Q3AypXA3/6mWPZH\nP/pR/OIXvwBiYoCFBQDAYxkZyEM0AGBmrBuno+ax4/hx4I47MG8yoS8mBmnx8cjIyAAAbNu2DTMC\nmXOTA4hLXY7o6BiYAPyf/n7ZsoeGhpCSkoK8vDwAwKWXXoqf//znuPrqq7F69WrExMQgJibGzUjT\nCnei5BCGfZE5p5dsxaSSF7Q9hgEIQdgXw6L3/VjDDV51W/kT9kWQxOPLojp0msewL8EcD+obNKBq\nBAqbs/t0DvuiJmSRzVuSgMd9CeA7bBEQ9iWgw1nnsC9qdY1yCIxLycHHrLHYd+mlePTRR/GZsTFk\nz89jdHYWg4ODAICzZ8+K8sYmZWFuog9Wq90ouyM/Hz0xMTCZTKJ2zMjIwPj4OHp7ewEAx48fx4oV\nK+zVCcAPtiW3ExWyF88VX4oVHKgZkPz17hmjhF5Q6nMV6U2SsWDyNTyQ2rI9q6BSYCSGfRG0dbjM\nNx1eGHYTwbAvQdUhZLcoteuJP2FfHHM9reSj6Dz7Cv7/Dz/E+C234Lb5ecQBuPuKK3D77bcjLS0N\nMTFi8yQmPgUZFVfjVPVzuLm4GH83MYG8+XmsLS7Gd77zHfz4xz92FGnCrl278I1vfAMmkwlpaWm4\n9957cVGy86YXS86IImTJYYSbCyFkSXDZZZfhsssuczu/r70db60tBwBERcfgK9YkfOTSS4EHH3St\nUVeXluLqX/7SlefUhV682fA+krIrAABpxZdgfXI+Hn7y311pbvvkJ7Hh5psBAAcOHAAAXH755bj8\n8su96nX06FE9qsvHeYQQQgghWqARRQghhBCiARpRhBiFQL0Mq8cL5kqPBMPmywNCiFrUfJyw1KER\nRUio8GSYhME7TGGgYtgg/XhAN7lh+rWiCxrm4cMSXg9oRBFCCCGEaIBGFCGEEEKIBmhEEUIIIUQX\n7rjjDjzyyCOu43GTCZ9esQK1cXGy6ffv3485H8vojInBm8nJfmipHzSi5BA+i3e+/BEKj+UwGcZj\nuVtWg74Uo/eLkJpaSG27+tUvQq/62j2Wax0ChvRYHqbv0ITUY7mXNIFuTUX5S8BjOaye8we0Bir1\nFq3zKj2W79y5E3/84x9RX18PAPhJTg5uHhnB2tlZ9yw2Gx555BFYfehrmwk4lpSEk4mJqvMEEjrb\nJCTSMajBSwgJPLfves31/y8+878RNz8LxMYAV8+L0k3FuceSe3vtJxbl/KEeV1wyh39VsBoyMzNx\n11134Yc//CH+IyEB7bGx+C9HCBYpz3Z0oK+vD/+Rn4/fdHbiZx98gBO33gqr1YovfvGLWF5uxnDz\nexhttwAwISG9GPkFlfhtZiamTSZsm5pCfoh/PC25naiQ3U4UQ4D4GM6DN0bP6NE2ekxMPcK+SMaC\n8Esuv77q8qiaxrYL5XgMVNmiHenAFKE7KtvCpy/3GPYlyDoYoB7eUFHHa665BmVlZfh+Tg7u7enx\nWKN/LCxETk4OftHVhbeTktA+NoZnnnkGTz31FB5++GFMToxhpO0EcjfegJKPfx1xKbmwAfjy4CD+\nx9gYrp2Y0LVqWuBOFCGEEBKhPP7DT+Gzd74IAPiPgw+gsqUKqKgAGhpE6V7d+En86lNfF527qvYI\nXt38abucL6wEVq8GDrapKveGG27A9FNPIW9+XjkxgLr4eJzr68OOHTsAAPPz8+jr7cbyLTdhqPFt\nzE0OIiGjFEgtVCUvWNCIMgJh+i4HCRM4vpY27H9iYEwmE6wAymdncVlBAX68bx+sVit+85vfIG95\nIUZaH0fups8jKjoW7R88htHELETZbLCGWnEHNKKI7khfiA9YOXq/9B9KIqEOQiKtPnph8Hbx+PFK\nqAigPrKi9SzPqLIMxiWXXIIv19XhqfZ2HI+NxW233YbJyUl88pOfRGJSEuKXLUfbew8hKiYeMQlp\nSE1djtWzs3goKwsbZmZQFmL9aUSpJZCD2AjP8gkhkQnXFxICLrvsMlzW16eY7r777gN+8hMAwPc/\n9jHgzjtd16rqepFWchnSSi5znYse6sD6mRm82twMAKjRV22fiTgjyjDLRYgWLq/FBlunAJVn8sOg\nVfMyt2FuOkbRA/61uX5KKLyor6RiMNtTWpbsRwMkWOj2bryRQ+kY+CX8/Wlp+EtqKoatA2h772EA\nwO7ocewcHsa2oGqiP0vu6zxCCCGEBI+bR0awr70dd0Zlofjyr6D48q/gBwsp2JaeHmrV/IZGFCGR\njoF2tAghJJKgEUUIIYQQogEaUXIIQ61A5y/AfJFjknzl5ilEQCjCvhj0jQ69w9Foqaf6L5y094so\niScVdRizHoeO35J1UMI9YUDVCBS6rTFuYV/UlK1anO4oLlv+rDFhEvbFZvP8oX5Av5RULVtd2Beb\nhrlnk97f1OYz2L2HRhQhhBBCwhOGfQkyoTJilXZIBAOBYV/8RIe20eUrHF/1kEvvtkD4OE5kZQAm\nDxNBc9NFYNgXoVRTuMw3VWPI+9eWbnVl2Jeg6hCqaqgu1o8+8fUrX0/rlJFYekYUIYQQQogO0Igi\nuqP1Wbfv0GO5YYm0+uhFKNrFh50Dw/Wazu0lkiYn26hexnWSpfc7o7oThusGjSi1hGHnEkIIISRw\nKBpRVqsVd999N26++Wbs2LEDLS0tsmm+9KUv4ZlnngmIkj5hFEM7VB7LvV6MEI/l/mSWGsP0WB4+\nKL5X6Gd+PVHjsZzdG3YYwnO/J8Lt/bEIQdGIOnToEGZnZ7F//37ceeed2LNnj1uaBx54AKOjowFR\nkBDiJ1wwCSEkICgaURaLBVdeeSUAYOvWraiurhZdP3jwIEwmkysNIYQQQshSQNGIGh8fR0pKius4\nOjoa8/PzAIC6ujr85S9/wTe/+c3AaUgIIYQQYkBilBKkpKRgYmLCdWy1WhETY8/2wgsvoKenB//8\nz/+Mjo4OxMbGorCwEJ/4xCe8yrRYLH6q7Zm+viHZssyO4zOnz3jUx5mmoaEBwCpRmvb2dvSo0Nsp\no6WlBaUC+eumppAEYGh4GBmSPJ2dnR7lnaqqcv1fc/48kiYGAQCJtbVY7zh/8uRJ2OLiAADNLRNS\nETh//jwmo6KwdmICyYLzff39aFXZF+PjY6rSOenu6cHg+fMuHZ1t7K3vLRaLq/28pxW/l9DZ2Qmk\nLsYCb29vR5EXucLzFcPDSBccA0BqXR2AaI96etfN3q4WiwVJ589jnUx6py6NTU0AlgGwj5dsR/+M\njI6i3mJBVnMzEjyUocabcV1dHcYyM0VjxalHV9eIbF2cP5CczMzMyNZ1cmJCdD6jqQnlgutq5wsg\n9ncl104NDQ0YtlhQ0teHHEE+Z1rnuB4TjFE5ndMbGlDh4brSmtTc0gIgVXWenNZWlAjSbZydRTyA\ngYEBNDvyTU1OiUSeO3cO09PTqnWS0tHRgf7Tp93O9/T0oF2yxrmQjKPz58+ju3vKcckKi8WCzOZm\nlHko0xcdnW3g5OTJk4iJXnzULDdH5eayJ2KGhrDF8X9nZye6BGl7eoYBOOaNo8jWtjbMStqlubkZ\nAzJleCq3s6sLBYLj6upqbHT839TUhEGF9rlQdxGdtinROecc7Ovrc507c+YM5rq6xOkWxPMmW3DP\nUdJ9embG49riierqasyMj2PZxYuSuyPQ2NiIxPlutzzbbDavOzW9fX1oE6zP0nWjsXtaPqOEQNoU\nSigaUZWVlTh8+DCuu+46VFVVYfXq1a5r3/nOd1z/P/jgg8jOzlY0oADAbJabLvrwQfNp4OKiISEt\na/OWzcDz4sEoTVNRXg44kjg/CS0qLESRD3qXli4OZ7PZDCQmAgAy0tLc0hYUFABVg27nbTBh25Yt\nwEtvAwDWrV2L7HWOW1XU4tCs3LYNSLBPicH5FuADsSG5bu1awGwGkpNF53OyspCjVKen2wEAKSmp\nQO/A4nmF12yW5+Vh+bp1rmOz2Ww3Zsxml0wp0n7wOE72NYsOCwoKAMf90wagqKhIdN2THLPZDAii\niLvSjYwAlga7PA/vE7nJFNQpOztbVV3KV5QBdfY2LS0tdfVPWmqqPf3p03BflhwI9PIUBmH1qlX2\nfheMFacetX21QPUFN91iXuwFZmZdx3Hx8TCbzXhTIjspKUlcpwsXRNeLioo8zxcP/S/VxUlFebm9\nHjk58mkd7ZYq2DGX7fOmJvfrDl28jUsAWFFaCtSL56jXdeyDD8TpHD9ysrKykOXI92HSewAWx9iG\n9euBjfZbsGuueEOib2FBAQq3bAFePWY/4ZCbl5uLPE+yJON73bp1GJztBGrGYDKZ7DpIXuEQ4tNa\n7mgDJ5WV2xAb4/nHis2kfi4DAARGR0F+PgoEaas6qoHacZEj0ZLiYmySyFuxYgVWCM852riyshIn\nT550K7Jg+XLR8caNG13/l5WVoUxhDqxetRIF5g2ivoyJiQFmZpEtGO+bN28GCgpEIubmF4D9HQAc\n7fLhh7JFyY3tBElfeMImGB4bN2wA1q4F+vvd0pWVlcO8tdBdgJf3MW0wITc7G7mCNpKuGzF1fcCb\nkvKkMm22gNoUgILxrpR5+/btOHr0KG655RbYbDbs3r0be/fuRUlJCa699lpdFQ0GIfOA6u3lXpvk\nNqjmAxC+LOwZPbwG6/ERjoIebpflvuCSHgt3Dvz4UsiTaqHwWO53WwfqK1BhFIFwmW+B8JhNj+VB\n1SFkHsvVrif+eCz31ROZAbpUCUUjKioqCvfcc4/oXEVFhVu6b3zjG/ppRQhZMtjCYKEkhBA56GzT\nCBjhFxQhJDLh+kJIwKARpRYjO1kzIkFoL9cORiT0TSDrEIqbaCT0SSAwersYTT3dw74I3iVcimFf\njP58zOjzQ4aIM6IMM0RC5bHcW7ER4rHcn5Xe7Zk8PZarwud3GQKihMI7ZkoLsNE8lhtntYp4wuad\nNn8It/fHIoSIM6IIIYQQQoIBjShCCCGEEA3QiCIk0gnD9wyIjrD/CQkYNKIIIYQQQjRAI0qOQH61\n4YMcm0mc3mOoDyWZWvN5E2nUl2KN8MKk2nb1p1+E48JTX+gwZj1JCOrexlLZSfG3npL8auaotzSB\nbnZl+X7MZb2/6vM4Efwsx+olfyDbX6XeovHhLY8WXSX3N7UY7d5DI4oQQggh4UmIf2QtOSMqZBsV\nPnyerepzciPsuBgVXdpGh4mp1Ocq0ks/2zeJ/vfH1YOn8xrbLhLDvgjbN1zmm0qXHapDfHiSqQWG\nfVEnIkQ7LarXE3/Cvvho8BihS5VYckYUIUuOcFiJCCEkDFlyRlTIdv4UCrYJbnSqnvka+D0Rm8kU\nHI/lznaSlqVL2ToYHkp9riKtTWIA2UT/a9fR87tO0rZUK1B7H/gdOy9AY031+yA+CQ35i0b2ZB4M\na7f+90GmcqEa5PiSR0Va0fwJxLuvOgQIV9sHmsR7+UGlej3xo088jTuP6Y17m3Ox5IwoQoixMNqL\nosRAhMNdNBzh7rRuRJwRpYt7/0DK8EG29F0RdXXz8k6EilAUHqVqaZJAvavi9SsR74uuW95AhH0J\nxjskgmse32XQ4/0M59AJ5c3M0/iVJlMrJxhomWshGHfe3oMJ6/us9F0cvcSqCRvlC1He5riPsgLw\n/lhA3wu0aZOpex/4ScQZUYQQCfw1v7Rh/xMSMGhEEUIIIYRogEYUIYQQQogGaEQZAW63E0IIIWEH\njSg5hCE1nC+tLfGwL25fsBv1rVMj6GW0sC9+tEkAIgbpp4TWdAZDtzXG18/JEeKwL4rXDRT2JVDl\nREjYF1m3DEow7AvRDSPc+AkhkQnXFxLJMOxLcGHYlyUAw74oYqiwL/62NcO+LKL203WGfQkMDPvi\nNS3DvhCiAhuC47Ecej9qDSWGrYNGvQxbnxBj8HYxnHqBVCgQHssNLsto3euG4QagMjSi1BKGnUsI\nIYRoJhy2gkJMxBlRhunyEA0+r8X688jFSB7L9cwbCI/leqGXHgb/AaD6RdEI81geNHUM3v/BQLcn\nkkZuy3B79OnEyG2qgogzogghhBBCggGNKEIIIYQQDdCIIoQQQgjRAI0oQiIdI73/QAghEQSNKEII\niWRoRBMSMGhEySEXUiMUYV8k/pZsnkIEhCLsi+qcwUXvkABawtuoblY/+kWUwpOKAfWxEzjR7mVF\neNgXvdYYLSE0vAzvQLemxzBWzuv+TGW9x0KA4h95a4OAtr/qsC/q8mhpBq3+BI02y2lEBQvFm7HA\nY7magcVfl57Rw2uwHouwkh7Sy7LppXoIx4kWpbyVJaOTv/LUZDWox3Kxw/IwmW9qPZZ7aXO3uoaL\nx3IjoIe+oaqy2mkYRFcKofLe7gs0ogghhBBCNEAjSi1h+qggNAQn7Ivuj1pDSbDDWwSaSOiTQMC+\n8A2ddRdKkxVtwFAtesry9XUHvV+PUC4w/MZq5BlRRtn9o8fyAD5m8WeiSfIuBY/lOmAIT80+PBLX\nll9H1HgsD5IqZIkQrh7Lw5zIM6IIIWFF0H/tEkKMgxF+oPkBjShCIh3+6iSEkIBAI8oIhLklTggh\nhCxFaEQRQgghhGiARpQR4OMWQkig4PpCSMCgEUUIIYQQogEaUXKIwr64n9NLtjISf0taQw8EIOyL\nUT/Q1vvtMi1fjimFsxAI9yRARRlCMR50dCbyY+x6VDGYwRf8bU/DE5iwL66QRV7leov7EuAGVVq2\n/FljAuhfSs9yvIZ98UW2r3qoDfsi3MXUOUSNzaQx7ItkZzXU7ldoRAULhS11YRgGVUsHt+g9o0fY\nFx3UUO5z5fTS0C7iBUP74qF31JdIDPsimpNRkTXfvLqTczvBsC+qMcrao6lclfPQD39UPhs8YdD9\nMUoJrFYrdu7ciQsXLiAuLg67du1CaWmp6/qTTz6Jv/71rwCAq666Cl//+tcDp20o4Rd0qrGZEByP\n5c4JFgl9Y9Q6aFXLqPUJNQZvF592P4JBQD2Wy8g2oJdxXWUZ3Sg12vhTgeJO1KFDhzA7O4v9+/fj\nzjvvxJ49e1zX2tra8NJLL+GPf/wjDhw4gHfffRe1tbUBVVgJwwQKDZkeXsqNEI/l/uxcuP0Sosdy\nVfi9W6SLEup3c7Xk1xU1QXyN070Rj173hVA/OvIKPZaHBMWdKIvFgiuvvBIAsHXrVlRXV7uuLV++\nHI899hiio6MBAPPz84iPjw+QqoQQTXDBJISQgKBoRI2PjyMlJcV1HB0djfn5ecTExCA2NhaZmZmw\n2Wz4yU9+gvXr16OsrEyxUIvF4p/WXujtGZYty+w4Pn36tEd9nGkaGhsBrBWl6ezsRJcKvZ0ympub\nsUIgf+3kJJIBDI+MIF2Sp7Oz06O8KoG+tRdq0TY7CgBIqK/HBsf5U6dOwZqU5Ch3wk1GbW0tJuLi\nsGZ8HCmC8/0DA2hR2RdjY2Oq0jnp6e1Ff02NS0dnG3vre4vF4mo/pbRCurq6gIzF47a2NhR7kSs8\nXz405MrqLC/lwgXFMr3pNjAwAIvFgsS6OqyXSe/UpbGpCU7FW1tbkenon9GxMVy0WJDZ1CTqL1+5\nePEiRnNzkXjhgpsend6BQXYAABhHSURBVB2jsnWZm5sTyZiZnZWt6+TklOh8RmMjygXXOzo60K1h\nnsu2U2MjhiwWFPf2IlcmrXNcC8eonM5p9fVY6eG60lhramoCkKw6T3ZLC0oF6TZMTyMBwODgIJoc\n+SYnJiHs4JqaGkzNz6vWSUpHZyf6ZNa3nt5etEvWOCfSR1q1tbXo6pq2X3PokNnUBE+rui86bpyZ\ngfAn9qmqU4iLWXwYIjdH2zs6UKSyvOjhYWx1/N/V1YVOQdru7hH7Pza4dgDb29thlbRLS0sL+mXK\nOHnyJKJl3ovr6upCvuD4bHU1Njn+b25uxoBC+1y8eBE9MeI555yD/f39i3LPnsWs4BgA5hYW+85i\nsSBLcM8RItdm0zMzSPCqmTvnzp3D9PQ0UuvqsFpyrampCam2Hrc8W61WRHuR2dfXh1bB+ixdN5p7\nZlTpFkibQglFIyolJQUTE4s3ZqvVipiYxWwzMzP4wQ9+gOTkZPzoRz9SVajZLDdd9MHSdhaoG/dY\n1pYtW4A/d3nVp6K8HOgVyy0oKECBD3qvWLFCLN9h5KSnpbmlLSgoAE71ycrZumUL8LejAIC1a9Yi\nd9Mq+4W4OFeabdu2AQ5Dd8TWBrw/JJKxdu1awGx2pXGSnZWFbKU6Pd0OAEhNTQV6B7ynFZCXm4u8\n9etdx2az2W7MmM0umVKk/eBxnPyuXnSYn58PTC8eFxcXi657kmM2m4GMDPd0k5PA+zXyZXuSKahT\nVlaW/bpgnsjpUF5WBjTYjf6SkhJX/yxLTbWnr6mB/KhQx6pVq+z9Hr24jDn1uDh4ATg76qZb7Mt9\nwPTiwhUfFwez2YwjEtlJSYniOjU0iK4XFhai0FP/eeh/qS5OysvL7fXIzZVP62i31NRUr3LQ3u5+\n3aGLt3EJwP7j8IJ4UfC6jp08KU6XYL9lZWZmItOR72Ty+6Is69evB7ZsAYDFueINib6FBQUoFKwX\nTvJyc5HnQZb0MdfatWsxZu0Bzo3B5NTdyysaPq3lkqcU27ZuQ0K891tQUWGh+vIGFten/Px85AvS\nVnfXADVjokeoRUVF2CaRV1pailLhOUcbV1ZW4nTVKbci8/PzRcebNm50/b9ixQqsUJgDq1atQpF5\ns6gvY2NjgekZZGdnL8rdtAkQvIsMALNzC8D+DgCOdpExoF3XJGMlQcMTow0bNgAbNgDDw27XysrK\nYK4scs8U5f2NoZycHORUVrqOpetGXEM/8IbyKhhImwLwbqQpvhNVWVmJI0fsS2hVVRVWr160QW02\nG772ta9hzZo1uOeee1yP9QghhBgEPs4lJGAo7kRt374dR48exS233AKbzYbdu3dj7969KCkpgdVq\nxfHjxzE7O4t33nkHAPCtb33LvjNCCCGEEBLBKBpRUVFRuOeee0TnKioqXP+fPXtWf60IIfph5C+K\nSOBh/xMSMOhsUw7hoqPK669G2UpJJf6WPPpwCYHHcqnXWMOgt14axKluVh36BfDSFzqMWT+cquuH\nao/l4Wks6ObvTOqxXMXg9VZioFtTUb4/c1lv/1IBmgi6+eUKscdyze2gKZ+x7j00ogghhBASnjDs\nyxJByVGg2JWu3/JCiU0a8y+Q5QDuZekSekEH/ZX6XHjZZpMP+yLRQ5hCteM/Wbmqk2orw4f+l4a2\n8btsnRC1r15lGGW3zIMeNsB9cIQy7IvuLOoQEI/landuVIrwV5abGG/l+rGeqE2ruK7aPK93RoVG\nlFqMsvgRQgghQcCvINBLhIgzogzT5SH61eW1WIOEffG3afzJriq4qiF+McM4esAg4S58DejsY35d\nURH2xWSc1Sri0W0zzQjhjzzBsC8hIeKMqLDECDcoQgghhPgEjShCCCGEEA3QiDIC3FolhAQKri+E\nBAwaUYREOgZ/XGzjPT6wGLz/yRInzMcnjShCCCGEEA3QiCKEEEII0QCNKDmEoVY8OXTUQbZiUonT\nSkOFfVGdM7jo7ddEmzw/w5SEKhSEUQlC2Beb1ao5r7/otsZIw76oGLrexnegh49SyBO/itdd+QDN\nVavn/D6JDlDYF9Hw8JJHSytodcpstMf/NKKI/piC47Ecehu4oSQS6iAk0uqjF0ZvF4Or5y+i6gXC\nY7nBZRk25qkTLfVk2JcwQc9wAHKXBdNblUM3o0+GUKJH2+gxMX11Dqkm7ItNOE78wGNmjVL9ceRq\n0LAv4iIia755W2PcnIBGVNiXAKNLHY3TTrI7lUF06hkO8y7yjCijtHmoPJZ7vWgQj+WatXDk98fA\nURNnzygT1yh6GAXF9lAYF0bzWM7uDTsM4bnfE/RYHhIiz4gihBBCCAkCNKIIISSS4a4DIQGDRhQh\nkY6RH0GQwMP+JyRg0IgihBBCCNEAjShCCCGEEA3QiCKEhBSjOc8jhASRMH/cTCOKEEIIIUQDNKLk\nEIZaMbmf00u2IlLP355CBIQg7ItRv/jR2yOvFnmqm1WnsC8eddRhzHpWMYi/HlWHffGjCC/hNzxn\n0qkNnP2nc9gX4zjN04Zfc1nn8elRnJ/leJtHPs2xAIV9EfWBV119K94lW0vYF+m4psfyMCHAHsuF\n8lV5cDaAEWOyyccbswFBGdgeDVwd2kaV13hFIT7oYbPJp3drR5uH/33DzSu187zWppNm9KH/jeqx\nXBRFQK8yDPLowlObyw7DCPJYLpo9cm2g5zqvUZZsM+kV9sWLYa16zfPDqaeis1KDzA9fiDgjytPN\nIej4egMNRrkG8VgeysXUbaGgx3JVGMJTs7/tYTCP5SR4hEP4EL+hx/KQEHFGFCFEghEMIEIIiUBo\nRBkB/ioghBBCwg4aUYQQQgghGqARRQghhBCiARpRhBBCCCEaoBFFCCGEEKIBGlGEkJDCsC+ELGHC\n/OthGlFyiDpVJ2/CsrIVkkrS2zw4twyFx3JvTtuWOuo9DevlsdzDNNbDY7mP5wOCao/l2rXS5IFd\nNweIOsmT5FczR/X28O8LytU1kMfyAJWjm+f/QHksh1rnoRrroaX+BvuanUYU0R2t7vx9LkdvAzeU\nREIdhERaffTC4O1iC655HFJkDRg9+8eIsoxlf7hj8PkhB40otQQ47Iv4qoqyDGCNG8KLtRx6hH3R\no2o+9bl8ercIHDbh/9p3aDw6k1cnUSajH97wjRr2RRiKyQDzTV88N7qacamJiGtDGXQJORUa5NYT\n2Z1Kf8K++Gikh8OQiTgjyjCNHiJFvJZqkLAv/raMP3Ht3G7YDPuiCl1iCfqtRGSFfTFO70Y+erW1\nYX84Agz7EiIizogihBBCCAkGNKIIIYQQQjRAI4oQQgghRAM0ogghhBBCNEAjihBCCCFEA4pGlNVq\nxd13342bb74ZO3bsQEtLi+j6gQMH8PnPfx433XQTDh8+HDBFCSGEEEKMRIxSgkOHDmF2dhb79+9H\nVVUV9uzZg4ceeggA0NfXh3379uG5557DzMwMbrvtNlxxxRWIi4sLuOKEEEIIIaFE0YiyWCy48sor\nAQBbt25FdXW169qZM2ewbds2xMXFIS4uDiUlJaitrcXmzZsDp7ECU+NTouOXdu21/7PtegDA9C+e\nAZBhv+Y4B0kaHDwLZF8GAHht03Ykzk4CJ7oW03nDKeP1msX/d+0F5nLsx/0xi+edWLpxrOIy1+FE\nQgoA4ESZGct/9TyAArvIP76N9L99YE80Mrwo5+f7AYfhenY6AUCSuA32vwO8XQ+MpYrLbplXUadM\nAMD5hn4Iva28suXTsEZFy+aYiU3ASzXjwBMHRW0wNDyEjlfOYMGauaibQJ+Xdu0V6+dBt8k4e/4T\n5WbkjfYAZweAdfZr7636GBbeaXSX4zjuTlsuLq/NKu4nAOjpQUemXeCp0q1u5b+07XoZ3Rbr9F7D\nCOZ27QWGh9xlA65zw387BSSssss82gyMptivjTrSX7yIyY3bRaV8UHGp6//j5Yv/vyQdUwDw4gng\ndD8w5K7HBxPJAOIX28FB/0imSETzdDRe2rUX1Ws/ITrf2j0myoeGBnGbW7q9jC1xGRMJyYv1kGkn\nvFoNNO0FakZEZbz04yfs/m4c43pqLElejpOWFpn+sOvylz1PAchazL/tepwpXlzH3nj5OGAqcl0D\nAPzfJz3726k9Ly4rvgzYVgZ0LJZ9YjwWSAGm4xLtMve9CbxyBgBcc8U74nZ86WQP8Ms/AcgHALy/\n6qMwwQpcnHBb406u2GYvJzFNJOPwk6/g4mwcRGOjvt59zXKWqWZNdJK8Cti2ynX4yk//gDih4zZJ\nGYc2XIOcE12q1gQAwOzsYtoz/aK0x8dTAMTBKljD3j7WgM4Wydr/9kVgQFiGvY3/umcfxkYG0bnm\n464rL2273r72CMfkwy8vHr95AejxPgfeeP59ZL91GsK+7B+ZBgBYLI2LY+23fwVSU0USFgRyXrr3\nKaCuTraf7H1kT3dkzZXoS8121Hm1KJ1wvDs5XbJ47qXfHQIyTwJdnW7lvPvCUYwffMO9mhVXig7P\nF6xz/f/Omo9jqKEN2LNvUd6JTlG/tc/FAhDXuz8129Uu0VYrCnqH3csNIiabQvCe//zP/8SnPvUp\nXHXVVQCAq6++GocOHUJMTAxefPFF1NXV4dvf/jYA4Dvf+Q5uuOEGXH755R7lWSwWHdV35/U/n8XR\n6YyAlkEIIYSQ0HPD6Bls/cp1AS/HbDbLnlfciUpJScHExITr2Gq1IiYmRvbaxMQEUiXWsi/K6MGW\nDRuRfv8fkT85hqjlechKcuyWDA0BViuQlYXOKROWjQ4g5b0jwN//PZBg/9WFmVmguxsoLcHk2BSG\nhidRuCwWOHcOuPxj6hQYHgHmZoGcHKClFcjPB+JiAasNaG4GVqwApqft+phMQFoakJwENDWjqW0I\nRdmJiFq9Gi0X2lGWEQtTQT4mx6cxODCOotJscVkX6oDYWKC8THS6ZcKE/ARgYHwWKSP9SC2x/zJ1\n6VBSArS2AmVliq58J+dNGJoDChJsaJwwITsemGxuQ/7COBbq6tG09QpExUQjPRawjo5iNLsApQOt\niC5bYZfd3gFkZQGJCeju7sHy5XlYsJpwYghYkwqk97ajd1kO4hPikRZns/dBdTVQUQGkp8nqtDC3\ngJb6LpQnLthDmpStwJzVhI7mHqwozgRiY4ATFiAlBcjLBTIygLl54OhR4Ior0DYXi5x4ICHaZo9+\ncfKkvU1yFtvX2tqGplErylOAwdxioKsLptgY2OrqkPXxjyyOGQdzVhOaOoYRN9iPFVsqFi8I6u9i\negbo7QVKitExtoCM/k4klRWLx0iUo2NaWnEuKgNTiMaq5clImxpF46gVJQXpiFmYQ2NzH0paLyBm\nVYV9zM3NASMj9nYpKV4ss7vH3g7xi4/amyZMiBvtQWF+7mLb2kyoHZhFas0ZzBWVoKwsx65KXz+O\nz6aibHYIoxk5KEuLdqno4sMTQHExMDUFlK3wOKbmrSZ8OAQkRwPrlgExzU14Zy4Nl5SnIylWIHR2\nDujqAkpLFs81NmFkWRZmZheQW+D4sSRot46ZKKTHAskxHn4btrYBeXmudhgdmsDU1CzyCjIwMGNC\nxxSwKn4WiX3dQFYWWs41I3/rGsTFRaOtpR/Z0yNItM3bZa1d47GOAIDmFqCgwDH/rfZjyZxrrO9B\naV4KokeGgaJC13nnXPGGc27Ozy1g+UA74svs7TTX2Y32uAyUTfcDExPAGsGOw+QUMDQEm9WKpnEb\nSlcVYmA+GjYAs1agOMnebo3jJpQkATFRjnasvWAfWwCQkYERUxxmsnKRm+CDB29HG4wXl2Fs3oT8\nREnewSHAZsNUehb6x+dQPNJlH8Mdnfa1MjZWNEdlae8AhoeBjRvcLjnrtHDxInpiUlBSsbgrjdk5\noLMTWFEqyjM2Z8LEPLA80Wbvk+hotIwuIH+oE3Gpyfa2nZ7BcEMrZjNzkJufbu/nqSlg3VqPak6N\nT2NAsKbPWk14t2kcV+dHwZaYhOYJoDzZBtOUvb9QWCArp3tgEkmzk1iW72iXllZMZmRjaHQasdlZ\niI8G0mJtmFkwoXsaKI2bAzo67PVsbrGvGVVVwLZtQHc3WhKykB8zh8meAdRgGT6WOIHu1BykzEwi\ntUTQXm3tQHQ0ZuMS0JWYidJkD+NgfsF+ryksBD78EEhPR31MOqwxsVgdNbE4Hzo6gYEBYPMmNxFt\nk/Y+qEgB2iaB0rFuRKemAF1diI6Jhu22qwJqUwDeN38Ud6JeffVVHD58GHv27EFVVRV+9atf4bH/\n1979hTTVx2EAf87OyszpxS4jJnMVFBEh0tWyoD92kUWSS4TdrCL7S4Y1Z0laazTqzm6St7qwbmJF\nXUV1E0PNLqQFSn8IxMhFZBa0pc3tfN+Llxa9vr5th+XZu/f53HnOF/3tPOjv4Wx4/vgDwF+fifJ4\nPAiFQkgkEqivr8fdu3dRVFQ06/cbHByckxf8u38GZY+55B9mkp+YS/5hJvnJ6E7xyztRmzZtQl9f\nHxoaGiAiCAQCuHbtGmw2GzZs2AC3243GxkaICJqbm/+1QBEREREVil+WKJPJhDNnzvx0zOH48XaF\ny+WCy+XK/cqIiIiI8hj/2SYRERGRDixRRERERDqwRBERERHpwBJFREREpANLFBEREZEOLFFERERE\nOrBEEREREenAEkVERESkwy8f+5Jrv/sBxERERES5NNtjX+a8RBEREREVAr6dR0RERKQDSxQRERGR\nDixRRERERDqwRBERERHpwBJFREREpIPZ6AXkkqZp6OjowMuXLzF//nz4/X6Ul5cbvayC9ezZM1y8\neBE9PT0YHR1Fa2srFEXB0qVLcfr0aZhMJly6dAmPHj2C2WxGW1sbVq1aldUsZW56ehptbW0YGxtD\nIpHA/v37sWTJEuZioFQqhVOnTmFkZASKoqCzsxNFRUXMJE98/PgRdXV1uHr1KsxmM3Mx2I4dO2Cx\nWAAAixcvxq5du3Du3Dmoqgqn04lDhw7Nus9HIpGMZ3NKCsj9+/fF6/WKiMjTp0+lqanJ4BUVru7u\nbtm6davU19eLiMi+fftkYGBARETa29vlwYMHMjQ0JG63WzRNk7GxMamrq8t6ljIXCoXE7/eLiMin\nT59k3bp1zMVgDx8+lNbWVhERGRgYkKamJmaSJxKJhBw4cEA2b94sr1+/Zi4Gm5qaku3bt/90bNu2\nbTI6OiqapsmePXtkeHh41n0+m9lcKqg7UYODg1i7di0AYPXq1RgaGjJ4RYXLZrOhq6sLJ06cAAAM\nDw9jzZo1AIDq6mr09fXBbrfD6XRCURQsWrQIqVQKExMTWc1arVbDXuN/zZYtW1BTUwMAEBGoqspc\nDLZx40asX78eABCNRlFWVob+/n5mkgeCwSAaGhrQ3d0NgH/DjPbixQtMTk7C4/EgmUzi8OHDSCQS\nsNlsAACn04n+/n58+PBhxj4fi8Uyns21gvpMVCwWS98KBABVVZFMJg1cUeGqqamB2fyjg4sIFEUB\nAJSUlODLly8z8vh+PJtZylxJSQksFgtisRiOHDmCo0ePMpc8YDab4fV6cfbsWdTW1jKTPHD79m1Y\nrdb0Bgvwb5jRFixYgN27d+PKlSvo7OyEz+dDcXFx+vxs11lV1Vmv/Vx0goK6E2WxWBCPx9Nfa5r2\n00ZPv4/J9KOPx+NxlJWVzcgjHo+jtLQ0q1nKzrt373Dw4EE0NjaitrYWFy5cSJ9jLsYJBoNoaWmB\ny+XCt2/f0seZiTFu3boFRVHw+PFjPH/+HF6vFxMTE+nzzGXu2e12lJeXQ1EU2O12lJaW4vPnz+nz\n36/z1NTUjH3+n679bLO57gQFdSeqsrIS4XAYABCJRLBs2TKDV/T/sWLFCjx58gQAEA6HUVVVhcrK\nSvT29kLTNESjUWiaBqvVmtUsZW58fBwejwfHjx/Hzp07ATAXo925cweXL18GABQXF0NRFKxcuZKZ\nGOzGjRu4fv06enp6sHz5cgSDQVRXVzMXA4VCIZw/fx4A8P79e0xOTmLhwoV48+YNRAS9vb3p6/z3\nfd5isWDevHkZzeZaQT077/sn8V+9egURQSAQgMPhMHpZBevt27c4duwYbt68iZGREbS3t2N6ehoV\nFRXw+/1QVRVdXV0Ih8PQNA0+nw9VVVVZzVLm/H4/7t27h4qKivSxkydPwu/3MxeDfP36FT6fD+Pj\n40gmk9i7dy8cDgd/V/KI2+1GR0cHTCYTczFQIpGAz+dDNBqFoihoaWmByWRCIBBAKpWC0+lEc3Pz\nrPt8JBLJeDaXCqpEEREREc2Vgno7j4iIiGiusEQRERER6cASRURERKQDSxQRERGRDixRRERERDqw\nRBERERHpwBJFREREpANLFBEREZEOfwLRBqkcH/awEgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 720x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "GB_fit, GB_predict, GB_acc, gb_model = gradientBoosting_pre(X_train, y_train, X_test, y_test)\n",
    "print('Gradient Boosting accuracy: {}%'.format(GB_acc * 100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3. Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression accuracy: 99.824%\n"
     ]
    }
   ],
   "source": [
    "LG_fit, LG_predict, LG_acc, lg_model = logisticRegression_pre(X_train, y_train, X_test, y_test)\n",
    "print('Logistic Regression accuracy: {}%'.format(LG_acc * 100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 4. SVM - Support Vector Machine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SVM accuracy: 99.824%\n"
     ]
    }
   ],
   "source": [
    "svm_fit, svm_predict, svm_acc, svm_model = svm_pre(X_train, y_train, X_test, y_test)\n",
    "print('SVM accuracy: {}%'.format(svm_acc * 100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 5. ANN - Artificial Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_1 (Dense)              (None, 55)                715       \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 2)                 112       \n",
      "=================================================================\n",
      "Total params: 827\n",
      "Trainable params: 827\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Train on 200000 samples, validate on 50000 samples\n",
      "Epoch 1/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0035 - acc: 0.9983 - val_loss: 0.0020 - val_acc: 0.9982\n",
      "Epoch 2/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0019 - acc: 0.9983 - val_loss: 0.0019 - val_acc: 0.9982\n",
      "Epoch 3/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0018 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 4/99\n",
      "200000/200000 [==============================] - 2s 8us/step - loss: 0.0018 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 5/99\n",
      "200000/200000 [==============================] - 2s 11us/step - loss: 0.0018 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 6/99\n",
      "200000/200000 [==============================] - 2s 11us/step - loss: 0.0018 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 7/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0018 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 8/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0018 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 9/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0018 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 10/99\n",
      "200000/200000 [==============================] - 2s 11us/step - loss: 0.0018 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 11/99\n",
      "200000/200000 [==============================] - 2s 11us/step - loss: 0.0018 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 12/99\n",
      "200000/200000 [==============================] - 2s 8us/step - loss: 0.0018 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 13/99\n",
      "200000/200000 [==============================] - 3s 14us/step - loss: 0.0018 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 14/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0018 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 15/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0018 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 16/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0018 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 17/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 18/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 19/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 20/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 21/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 22/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 23/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 24/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 25/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 26/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 27/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 28/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 29/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 30/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 31/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 32/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 33/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 34/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 35/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 36/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 37/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 38/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 39/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 40/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 41/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 42/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 43/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 44/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 45/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 46/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 47/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 48/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 49/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 50/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 51/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 52/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 53/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 54/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 55/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 56/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 57/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 58/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 59/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 60/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 61/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 62/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 63/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 64/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 65/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 66/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 67/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 68/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 69/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 70/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 71/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 72/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 73/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 74/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 75/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 76/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 77/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 78/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 79/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 80/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 81/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 82/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 83/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 84/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 85/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 86/99\n",
      "200000/200000 [==============================] - 3s 13us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 87/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 88/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 89/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 90/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 91/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 92/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 93/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 94/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 95/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 96/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 97/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 98/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 99/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "50000/50000 [==============================] - 1s 12us/step\n"
     ]
    }
   ],
   "source": [
    "ann_summary, ann_fit, ann_evaluate, ann_prediction_acc, ann_model = ann_pre(X_train, y_train, X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ANN accuracy: 99.824%\n"
     ]
    }
   ],
   "source": [
    "print('ANN accuracy: {}%'.format(ann_prediction_acc * 100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 6. MLP - Multi-layered Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_3 (Dense)              (None, 55)                715       \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 55)                3080      \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 55)                3080      \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 55)                3080      \n",
      "_________________________________________________________________\n",
      "dense_7 (Dense)              (None, 2)                 112       \n",
      "=================================================================\n",
      "Total params: 10,067\n",
      "Trainable params: 10,067\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Train on 200000 samples, validate on 50000 samples\n",
      "Epoch 1/99\n",
      "200000/200000 [==============================] - 3s 14us/step - loss: 0.0033 - acc: 0.9983 - val_loss: 0.0019 - val_acc: 0.9982\n",
      "Epoch 2/99\n",
      "200000/200000 [==============================] - 3s 13us/step - loss: 0.0018 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 3/99\n",
      "200000/200000 [==============================] - 3s 13us/step - loss: 0.0018 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 4/99\n",
      "200000/200000 [==============================] - 3s 14us/step - loss: 0.0018 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 5/99\n",
      "200000/200000 [==============================] - 3s 13us/step - loss: 0.0018 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 6/99\n",
      "200000/200000 [==============================] - 3s 13us/step - loss: 0.0018 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 7/99\n",
      "200000/200000 [==============================] - 3s 13us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 8/99\n",
      "200000/200000 [==============================] - 3s 13us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 9/99\n",
      "200000/200000 [==============================] - 3s 13us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 10/99\n",
      "200000/200000 [==============================] - 3s 13us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 11/99\n",
      "200000/200000 [==============================] - 3s 14us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 12/99\n",
      "200000/200000 [==============================] - 3s 13us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 13/99\n",
      "200000/200000 [==============================] - 3s 14us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 14/99\n",
      "200000/200000 [==============================] - 3s 13us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 15/99\n",
      "200000/200000 [==============================] - 3s 14us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 16/99\n",
      "200000/200000 [==============================] - 3s 14us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 17/99\n",
      "200000/200000 [==============================] - 3s 14us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 18/99\n",
      "200000/200000 [==============================] - 3s 13us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 19/99\n",
      "200000/200000 [==============================] - 3s 14us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 20/99\n",
      "200000/200000 [==============================] - 3s 14us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 21/99\n",
      "200000/200000 [==============================] - 3s 14us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 22/99\n",
      "200000/200000 [==============================] - 3s 14us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 23/99\n",
      "200000/200000 [==============================] - 3s 14us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 24/99\n",
      "200000/200000 [==============================] - 3s 14us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 25/99\n",
      "200000/200000 [==============================] - 3s 14us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 26/99\n",
      "200000/200000 [==============================] - 3s 14us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 27/99\n",
      "200000/200000 [==============================] - 3s 14us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 28/99\n",
      "200000/200000 [==============================] - 3s 14us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 29/99\n",
      "200000/200000 [==============================] - 3s 14us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 30/99\n",
      "200000/200000 [==============================] - 3s 17us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 31/99\n",
      "200000/200000 [==============================] - 4s 19us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 32/99\n",
      "200000/200000 [==============================] - 3s 15us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 33/99\n",
      "200000/200000 [==============================] - 3s 13us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 34/99\n",
      "200000/200000 [==============================] - 3s 15us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 35/99\n",
      "200000/200000 [==============================] - 3s 15us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 36/99\n",
      "200000/200000 [==============================] - 3s 17us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 37/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 38/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 39/99\n",
      "200000/200000 [==============================] - 3s 13us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 40/99\n",
      "200000/200000 [==============================] - 3s 13us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 41/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 42/99\n",
      "200000/200000 [==============================] - 3s 13us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 43/99\n",
      "200000/200000 [==============================] - 3s 15us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 44/99\n",
      "200000/200000 [==============================] - 3s 14us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 45/99\n",
      "200000/200000 [==============================] - 4s 18us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 46/99\n",
      "200000/200000 [==============================] - 4s 20us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 47/99\n",
      "200000/200000 [==============================] - 3s 16us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 48/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 49/99\n",
      "200000/200000 [==============================] - 3s 15us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 50/99\n",
      "200000/200000 [==============================] - 4s 18us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 51/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 52/99\n",
      "200000/200000 [==============================] - 3s 14us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 53/99\n",
      "200000/200000 [==============================] - 3s 17us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 54/99\n",
      "200000/200000 [==============================] - 3s 17us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 55/99\n",
      "200000/200000 [==============================] - 3s 14us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 56/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 57/99\n",
      "200000/200000 [==============================] - 3s 13us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 58/99\n",
      "200000/200000 [==============================] - 3s 14us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 59/99\n",
      "200000/200000 [==============================] - 3s 17us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 60/99\n",
      "200000/200000 [==============================] - 4s 19us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 61/99\n",
      "200000/200000 [==============================] - 3s 15us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 62/99\n",
      "200000/200000 [==============================] - 3s 16us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 63/99\n",
      "200000/200000 [==============================] - 3s 15us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 64/99\n",
      "200000/200000 [==============================] - 3s 16us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 65/99\n",
      "200000/200000 [==============================] - 3s 14us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 66/99\n",
      "200000/200000 [==============================] - 3s 13us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 67/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 68/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 69/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 70/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 71/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 72/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 73/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 74/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 75/99\n",
      "200000/200000 [==============================] - 3s 14us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 76/99\n",
      "200000/200000 [==============================] - 3s 15us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 77/99\n",
      "200000/200000 [==============================] - 3s 15us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 78/99\n",
      "200000/200000 [==============================] - 3s 14us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 79/99\n",
      "200000/200000 [==============================] - 3s 16us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 80/99\n",
      "200000/200000 [==============================] - 3s 13us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 81/99\n",
      "200000/200000 [==============================] - 3s 14us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 82/99\n",
      "200000/200000 [==============================] - 3s 14us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 83/99\n",
      "200000/200000 [==============================] - 3s 15us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 84/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 85/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 86/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 87/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 88/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 89/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 90/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 91/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 92/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 93/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 94/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 95/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 96/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 97/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 98/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 99/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "50000/50000 [==============================] - 1s 14us/step\n"
     ]
    }
   ],
   "source": [
    "mlp_summary, mlp_fit, mlp_evaluate, mlp_prediction_acc, mlp_model = mlp_pre(X_train, y_train, X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MLP accuracy: 99.824%\n"
     ]
    }
   ],
   "source": [
    "print('MLP accuracy: {}%'.format(mlp_prediction_acc * 100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 7. RNN - Recurrent Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      " - 4s - loss: 0.0018\n",
      "Epoch 2/10\n",
      " - 3s - loss: 0.0017\n",
      "Epoch 3/10\n",
      " - 3s - loss: 0.0017\n",
      "Epoch 4/10\n",
      " - 4s - loss: 0.0017\n",
      "Epoch 5/10\n",
      " - 4s - loss: 0.0017\n",
      "Epoch 6/10\n",
      " - 4s - loss: 0.0017\n",
      "Epoch 7/10\n",
      " - 3s - loss: 0.0017\n",
      "Epoch 8/10\n",
      " - 3s - loss: 0.0017\n",
      "Epoch 9/10\n",
      " - 3s - loss: 0.0017\n",
      "Epoch 10/10\n",
      " - 3s - loss: 0.0017\n",
      "Train Score: 0.04 RMSE\n",
      "Test Score: 0.04 RMSE\n",
      "RNN accuracy: 23.906133333333333%\n"
     ]
    }
   ],
   "source": [
    "rnn_acc = rnn_pre(df_train)\n",
    "print('RNN accuracy: {}%'.format(rnn_acc * 100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Conclusion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Random Forest accuracy: 99.884%\n",
      "Gradient Boosting accuracy: 99.57000000000001%\n",
      "Logistic Regression accuracy: 99.824%\n",
      "SVM accuracy: 99.824%\n",
      "ANN accuracy: 99.824%\n",
      "MLP accuracy: 99.824%\n",
      "RNN accuracy: 23.906133333333333%\n"
     ]
    }
   ],
   "source": [
    "print('Random Forest accuracy: {}%'.format(RF_acc * 100))\n",
    "print('Gradient Boosting accuracy: {}%'.format(GB_acc * 100))\n",
    "print('Logistic Regression accuracy: {}%'.format(LG_acc * 100))\n",
    "print('SVM accuracy: {}%'.format(svm_acc * 100))\n",
    "print('ANN accuracy: {}%'.format(ann_prediction_acc * 100))\n",
    "print('MLP accuracy: {}%'.format(mlp_prediction_acc * 100))\n",
    "print('RNN accuracy: {}%'.format(rnn_acc * 100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Random Forest is the best one for our project"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prediction for test.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['ip',\n",
       " 'app',\n",
       " 'device',\n",
       " 'os',\n",
       " 'channel',\n",
       " 'ip_nextClick',\n",
       " 'ip_app_nextClick',\n",
       " 'ip_channel_nextClick',\n",
       " 'ip_os_nextClick',\n",
       " 'ip_app_device_os_channel_nextClick',\n",
       " 'ip_os_device_nextClick',\n",
       " 'ip_os_device_app_nextClick']"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Columns for our current analsis\n",
    "train_cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Read the test data\n",
    "df = pd.read_csv('data/test_small_all_features.csv')[train_cols].astype('float64')\n",
    "df = np.nan_to_num(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Read the output of the test data\n",
    "sample_out = pd.read_csv('data/sample_submission.csv', nrows=1000000)[['is_attributed']].astype('float64')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_predict = rf_model.predict(df)\n",
    "# Compare the prediction with the known values\n",
    "df_acc = sklearn.metrics.accuracy_score(np.array(df_predict)[:], \n",
    "                                         np.array(sample_out)[:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "By using the best algorittm, the accuracy of the prediction: 99.9302%\n"
     ]
    }
   ],
   "source": [
    "print('By using the best algorittm, the accuracy of the prediction: {}%'.format(df_acc * 100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The code in the document is licensed under the MIT License: https://opensource.org/licenses/MIT\n",
    "\n",
    "All writing in the document is licensed bt The Creative Commons Attribution 3.0 https://creativecommons.org/licenses/by/3.0/us/."
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda env:anaconda]",
   "language": "python",
   "name": "conda-env-anaconda-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}