modeling_catboost.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Mobile Ad Anti-Fraud Algorithm Challenge - Modeling Part Ⅱ\n",
    "\n",
    "## 移动广告反欺诈算法挑战赛 数据建模 Ⅱ\n",
    "2019 科大讯飞 A.I.开发者挑战赛"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "-- Date:   02 Sept 2019  \n",
    "-- Author:   Arianna.H"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "___"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5.Modeling - catboost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity = \"all\"  #display multiple outputs in one cell\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import time \n",
    "from sklearn.model_selection import train_test_split \n",
    "from sklearn.metrics import auc\n",
    "#!pip install catboost\n",
    "#!pip install --user --upgrade catboost\n",
    "from catboost import CatBoostClassifier, Pool, cv  #catboost\n",
    "from catboost.utils import get_roc_curve\n",
    "from catboost import MetricVisualizer  #plot metric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'0.16.4'"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import catboost\n",
    "catboost.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('round1_iflyad_anticheat_train_clean_cat.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1000000, 46)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>pkgname</th>\n",
       "      <th>ver</th>\n",
       "      <th>adunitshowid</th>\n",
       "      <th>mediashowid</th>\n",
       "      <th>apptype</th>\n",
       "      <th>ip</th>\n",
       "      <th>city</th>\n",
       "      <th>province</th>\n",
       "      <th>reqrealip</th>\n",
       "      <th>...</th>\n",
       "      <th>hw_ratio</th>\n",
       "      <th>inch</th>\n",
       "      <th>osv_clean</th>\n",
       "      <th>osv_top1</th>\n",
       "      <th>osv_top2</th>\n",
       "      <th>osv_top3</th>\n",
       "      <th>ver_clean</th>\n",
       "      <th>ver_top1</th>\n",
       "      <th>ver_top2</th>\n",
       "      <th>ver_top3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2d2664e827bcbb8b07100c7fbe072e9b</td>\n",
       "      <td>11.11.0</td>\n",
       "      <td>907d0f8c29663840491577a21c7b612a</td>\n",
       "      <td>ca64a500000d84c8fcb8a0587d0e1e0c</td>\n",
       "      <td>280.0</td>\n",
       "      <td>183.197.47.83</td>\n",
       "      <td>石家庄市</td>\n",
       "      <td>河北省</td>\n",
       "      <td>180.76.181.7</td>\n",
       "      <td>...</td>\n",
       "      <td>2.2</td>\n",
       "      <td>859.1</td>\n",
       "      <td>810</td>\n",
       "      <td>8</td>\n",
       "      <td>81</td>\n",
       "      <td>810</td>\n",
       "      <td>11110</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>empty</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10199dc8fea2e173525bc3151bd84312</td>\n",
       "      <td>3f2fc57a6e1f9c6fa4464c25cc1e88a3</td>\n",
       "      <td>319.0</td>\n",
       "      <td>106.34.14.149</td>\n",
       "      <td>开封市</td>\n",
       "      <td>河南省</td>\n",
       "      <td>47.95.111.138</td>\n",
       "      <td>...</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>16b81f93f4b1a35cebbf15f07683f171</td>\n",
       "      <td>3.2.1.0524.1958</td>\n",
       "      <td>83f2ecfe65f936f5f2ed59f8e8ff1d01</td>\n",
       "      <td>eea7280e1a2313e4c2e89290b01d196c</td>\n",
       "      <td>273.0</td>\n",
       "      <td>223.104.16.151</td>\n",
       "      <td>长春市</td>\n",
       "      <td>吉林省</td>\n",
       "      <td>39.96.159.155</td>\n",
       "      <td>...</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>810</td>\n",
       "      <td>8</td>\n",
       "      <td>81</td>\n",
       "      <td>810</td>\n",
       "      <td>32105241958</td>\n",
       "      <td>3</td>\n",
       "      <td>32</td>\n",
       "      <td>321</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 46 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   label                           pkgname              ver  \\\n",
       "0      1  2d2664e827bcbb8b07100c7fbe072e9b          11.11.0   \n",
       "1      1                             empty              NaN   \n",
       "2      0  16b81f93f4b1a35cebbf15f07683f171  3.2.1.0524.1958   \n",
       "\n",
       "                       adunitshowid                       mediashowid  \\\n",
       "0  907d0f8c29663840491577a21c7b612a  ca64a500000d84c8fcb8a0587d0e1e0c   \n",
       "1  10199dc8fea2e173525bc3151bd84312  3f2fc57a6e1f9c6fa4464c25cc1e88a3   \n",
       "2  83f2ecfe65f936f5f2ed59f8e8ff1d01  eea7280e1a2313e4c2e89290b01d196c   \n",
       "\n",
       "   apptype              ip  city province      reqrealip  ... hw_ratio   inch  \\\n",
       "0    280.0   183.197.47.83  石家庄市      河北省   180.76.181.7  ...      2.2  859.1   \n",
       "1    319.0   106.34.14.149   开封市      河南省  47.95.111.138  ...      0.5    0.0   \n",
       "2    273.0  223.104.16.151   长春市      吉林省  39.96.159.155  ...      2.0    0.0   \n",
       "\n",
       "  osv_clean osv_top1  osv_top2 osv_top3    ver_clean  ver_top1  ver_top2  \\\n",
       "0       810        8        81      810        11110         1        11   \n",
       "1         9        9         9        9            0         0         0   \n",
       "2       810        8        81      810  32105241958         3        32   \n",
       "\n",
       "  ver_top3  \n",
       "0      111  \n",
       "1        0  \n",
       "2      321  \n",
       "\n",
       "[3 rows x 46 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape\n",
    "df.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "___"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Fix null value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ver         321824\n",
       "city         15193\n",
       "province     55973\n",
       "osv          12997\n",
       "lan         367527\n",
       "dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "null_value_stats = df.isnull().sum(axis=0)\n",
    "null_value_stats[null_value_stats != 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cols_to_fix():\n",
    "    for c in df.columns:\n",
    "        if df[c].dtype == \"object\":\n",
    "            df[c] = df[c].fillna(\"nan\")\n",
    "        else:\n",
    "            df[c] = df[c].fillna(0.0)\n",
    "\n",
    "cols_to_fix()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "traindata_x = df.drop(['label'],axis=1)\n",
    "traindata_y = df['label']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "One of the special characteristics of CatBoost model is the ability to tackle non-numeric data directly. We do not need to do any convertion of categorical data which is a must for other algorithms like randomforest, logistic regression, etc.  \n",
    "\n",
    "Separate categorical/nominal columns as we need to specify them in the CatBoost model. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "catCol = ['pkgname', 'ver', 'adunitshowid', 'mediashowid', 'apptype', 'ip',\n",
    "       'city', 'province', 'reqrealip', 'adidmd5', 'imeimd5', 'openudidmd5',\n",
    "       'macmd5', 'dvctype', 'model', 'make', 'ntt', 'carrier', 'os', 'osv',\n",
    "       'orientation', 'lan',  'date', 'dayofweek', 'hour',\n",
    "       'minute', 'hourtosecond', 'ip_top', 'reqrealip_top', 'apptype_top1',\n",
    "       'apptype_top2',  'osv_clean', 'osv_top1',\n",
    "       'osv_top2', 'osv_top3', 'ver_clean', 'ver_top1', 'ver_top2',\n",
    "       'ver_top3']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ConvertToCat():\n",
    "    for var in catCol:\n",
    "        traindata_x[var] = traindata_x[var].astype('category')\n",
    "ConvertToCat()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pkgname          category\n",
      "ver              category\n",
      "adunitshowid     category\n",
      "mediashowid      category\n",
      "apptype          category\n",
      "ip               category\n",
      "city             category\n",
      "province         category\n",
      "reqrealip        category\n",
      "adidmd5          category\n",
      "imeimd5          category\n",
      "openudidmd5      category\n",
      "macmd5           category\n",
      "dvctype          category\n",
      "model            category\n",
      "make             category\n",
      "ntt              category\n",
      "carrier          category\n",
      "os               category\n",
      "osv              category\n",
      "orientation      category\n",
      "lan              category\n",
      "h                 float64\n",
      "w                 float64\n",
      "ppi               float64\n",
      "date             category\n",
      "dayofweek        category\n",
      "hour             category\n",
      "minute           category\n",
      "hourtosecond     category\n",
      "ip_top           category\n",
      "reqrealip_top    category\n",
      "apptype_top1     category\n",
      "apptype_top2     category\n",
      "h_w               float64\n",
      "hw_ratio          float64\n",
      "inch              float64\n",
      "osv_clean        category\n",
      "osv_top1         category\n",
      "osv_top2         category\n",
      "osv_top3         category\n",
      "ver_clean        category\n",
      "ver_top1         category\n",
      "ver_top2         category\n",
      "ver_top3         category\n",
      "dtype: object\n"
     ]
    }
   ],
   "source": [
    "print(traindata_x.dtypes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### indicate categorical columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15, 16,\n",
       "       17, 18, 19, 20, 21, 25, 26, 27, 28, 29, 30, 31, 32, 33, 37, 38, 39,\n",
       "       40, 41, 42, 43, 44], dtype=int64)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "categorical_features_indices = np.where(traindata_x.dtypes == 'category')[0]\n",
    "categorical_features_indices"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Split train and validation set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train,x_test,y_train,y_test = train_test_split(traindata_x,traindata_y,train_size=.9,random_state=20,shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(900000, 45) (900000,) (100000, 45) (100000,)\n"
     ]
    }
   ],
   "source": [
    "print(x_train.shape,y_train.shape,x_test.shape,y_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "___"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6.Tune Catboost Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Simple Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = CatBoostClassifier(\n",
    "    loss_function='Logloss',\n",
    "    custom_loss=['Logloss', 'AUC'],\n",
    "    eval_metric='F1',\n",
    "    iterations=150,\n",
    "    learning_rate=0.4,\n",
    "    random_state=20, \n",
    "    verbose=10,\n",
    "    train_dir='it_150_lr_0.4'\n",
    "    #use_best_model=False\n",
    ")\n",
    "print('Model params:')\n",
    "print(model.get_params())\n",
    "print()\n",
    "print('Staring training catboost model...')\n",
    "model.fit(\n",
    "    x_train, y_train,\n",
    "    eval_set=(x_test,y_test),\n",
    "    cat_features=categorical_features_indices\n",
    ")\n",
    "print('Best score is:')\n",
    "print(model.best_score_)\n",
    "print('Best iteration is: '+ str(model.best_iteration_))\n",
    "# print('Tree counts of Catboost with best parameters: ' + str(model.tree_count_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# you can save model information for using next time without re-runing it\n",
    "model.save_model('catboost_model.dump')\n",
    "model = CatBoostClassifier()\n",
    "model.load_model('catboost_model.dump')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training with cv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {\n",
    "    'loss_function':'Logloss',\n",
    "    'custom_metric':['Logloss', 'AUC'],\n",
    "    'iterations': 500,\n",
    "    'learning_rate': 0.1,\n",
    "    'eval_metric': 'F1',\n",
    "    'random_seed': 20,\n",
    "    'use_best_model':True\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start training cv model...\n",
      "\n",
      "0:\tlearn: 0.9019838\ttest: 0.9046348\tbest: 0.9046348 (0)\ttotal: 21.2s\tremaining: 2h 56m 29s\n",
      "10:\tlearn: 0.9184758\ttest: 0.9225637\tbest: 0.9225637 (10)\ttotal: 3m 48s\tremaining: 2h 49m 33s\n",
      "20:\tlearn: 0.9228003\ttest: 0.9261618\tbest: 0.9261618 (20)\ttotal: 7m 43s\tremaining: 2h 56m 6s\n",
      "30:\tlearn: 0.9245322\ttest: 0.9282521\tbest: 0.9282521 (30)\ttotal: 12m 1s\tremaining: 3h 1m 54s\n",
      "40:\tlearn: 0.9252673\ttest: 0.9290518\tbest: 0.9290518 (40)\ttotal: 16m 17s\tremaining: 3h 2m 18s\n",
      "50:\tlearn: 0.9263197\ttest: 0.9299111\tbest: 0.9299111 (50)\ttotal: 20m 22s\tremaining: 2h 59m 23s\n",
      "60:\tlearn: 0.9270366\ttest: 0.9305007\tbest: 0.9305007 (60)\ttotal: 24m 27s\tremaining: 2h 56m 2s\n",
      "70:\tlearn: 0.9275113\ttest: 0.9309147\tbest: 0.9309166 (68)\ttotal: 28m 24s\tremaining: 2h 51m 37s\n",
      "80:\tlearn: 0.9280130\ttest: 0.9314620\tbest: 0.9314620 (80)\ttotal: 32m 27s\tremaining: 2h 47m 53s\n",
      "90:\tlearn: 0.9285783\ttest: 0.9319354\tbest: 0.9319354 (90)\ttotal: 36m 5s\tremaining: 2h 42m 10s\n",
      "100:\tlearn: 0.9291497\ttest: 0.9324818\tbest: 0.9324818 (100)\ttotal: 39m 45s\tremaining: 2h 37m 4s\n",
      "110:\tlearn: 0.9295235\ttest: 0.9327920\tbest: 0.9327920 (110)\ttotal: 43m 33s\tremaining: 2h 32m 39s\n",
      "120:\tlearn: 0.9298206\ttest: 0.9329955\tbest: 0.9329955 (120)\ttotal: 47m 18s\tremaining: 2h 28m 12s\n",
      "130:\tlearn: 0.9301126\ttest: 0.9331496\tbest: 0.9331496 (130)\ttotal: 51m 7s\tremaining: 2h 24m 1s\n",
      "140:\tlearn: 0.9304934\ttest: 0.9333355\tbest: 0.9333355 (140)\ttotal: 54m 55s\tremaining: 2h 19m 50s\n",
      "150:\tlearn: 0.9308079\ttest: 0.9336231\tbest: 0.9336231 (150)\ttotal: 58m 47s\tremaining: 2h 15m 53s\n",
      "160:\tlearn: 0.9311545\ttest: 0.9338466\tbest: 0.9338466 (160)\ttotal: 1h 2m 42s\tremaining: 2h 12m 2s\n",
      "170:\tlearn: 0.9314310\ttest: 0.9340389\tbest: 0.9340389 (170)\ttotal: 1h 6m 31s\tremaining: 2h 7m 59s\n",
      "180:\tlearn: 0.9316862\ttest: 0.9343192\tbest: 0.9343192 (180)\ttotal: 1h 10m 27s\tremaining: 2h 4m 10s\n",
      "190:\tlearn: 0.9319023\ttest: 0.9344243\tbest: 0.9344243 (190)\ttotal: 1h 14m 16s\tremaining: 2h 9s\n",
      "200:\tlearn: 0.9321104\ttest: 0.9345423\tbest: 0.9345423 (200)\ttotal: 1h 18m 2s\tremaining: 1h 56m 5s\n",
      "210:\tlearn: 0.9322017\ttest: 0.9346463\tbest: 0.9346463 (210)\ttotal: 1h 21m 50s\tremaining: 1h 52m 6s\n",
      "220:\tlearn: 0.9324426\ttest: 0.9348285\tbest: 0.9348285 (220)\ttotal: 1h 25m 45s\tremaining: 1h 48m 15s\n",
      "230:\tlearn: 0.9325956\ttest: 0.9349305\tbest: 0.9349420 (229)\ttotal: 1h 29m 55s\tremaining: 1h 44m 42s\n",
      "240:\tlearn: 0.9327245\ttest: 0.9351349\tbest: 0.9351349 (240)\ttotal: 1h 34m 5s\tremaining: 1h 41m 6s\n",
      "250:\tlearn: 0.9329172\ttest: 0.9352216\tbest: 0.9352216 (250)\ttotal: 1h 37m 56s\tremaining: 1h 37m 9s\n",
      "260:\tlearn: 0.9330035\ttest: 0.9352838\tbest: 0.9352838 (260)\ttotal: 1h 41m 41s\tremaining: 1h 33m 7s\n",
      "270:\tlearn: 0.9331581\ttest: 0.9353698\tbest: 0.9353698 (270)\ttotal: 1h 45m 31s\tremaining: 1h 29m 10s\n",
      "280:\tlearn: 0.9332480\ttest: 0.9354543\tbest: 0.9354543 (280)\ttotal: 1h 49m 28s\tremaining: 1h 25m 19s\n",
      "290:\tlearn: 0.9333134\ttest: 0.9355195\tbest: 0.9355195 (290)\ttotal: 1h 53m 42s\tremaining: 1h 21m 40s\n",
      "300:\tlearn: 0.9334762\ttest: 0.9356103\tbest: 0.9356103 (300)\ttotal: 1h 57m 30s\tremaining: 1h 17m 41s\n",
      "310:\tlearn: 0.9335567\ttest: 0.9356972\tbest: 0.9357027 (308)\ttotal: 2h 1m 21s\tremaining: 1h 13m 44s\n",
      "320:\tlearn: 0.9336208\ttest: 0.9357522\tbest: 0.9357522 (320)\ttotal: 2h 5m 18s\tremaining: 1h 9m 52s\n",
      "330:\tlearn: 0.9337015\ttest: 0.9357957\tbest: 0.9358070 (327)\ttotal: 2h 9m 22s\tremaining: 1h 6m 3s\n",
      "340:\tlearn: 0.9337956\ttest: 0.9358817\tbest: 0.9358876 (338)\ttotal: 2h 13m 11s\tremaining: 1h 2m 6s\n",
      "350:\tlearn: 0.9338722\ttest: 0.9359652\tbest: 0.9359652 (350)\ttotal: 2h 17m 7s\tremaining: 58m 12s\n",
      "360:\tlearn: 0.9339211\ttest: 0.9359886\tbest: 0.9360044 (358)\ttotal: 2h 20m 57s\tremaining: 54m 16s\n",
      "370:\tlearn: 0.9339924\ttest: 0.9360384\tbest: 0.9360384 (370)\ttotal: 2h 24m 57s\tremaining: 50m 24s\n",
      "380:\tlearn: 0.9340731\ttest: 0.9360915\tbest: 0.9360920 (379)\ttotal: 2h 28m 40s\tremaining: 46m 26s\n",
      "390:\tlearn: 0.9341597\ttest: 0.9361173\tbest: 0.9361173 (390)\ttotal: 2h 32m 41s\tremaining: 42m 33s\n",
      "400:\tlearn: 0.9342182\ttest: 0.9361117\tbest: 0.9361478 (393)\ttotal: 2h 36m 31s\tremaining: 38m 38s\n",
      "410:\tlearn: 0.9342762\ttest: 0.9361501\tbest: 0.9361501 (410)\ttotal: 2h 40m 23s\tremaining: 34m 44s\n",
      "420:\tlearn: 0.9343446\ttest: 0.9362072\tbest: 0.9362072 (420)\ttotal: 2h 44m 16s\tremaining: 30m 49s\n",
      "430:\tlearn: 0.9344157\ttest: 0.9362373\tbest: 0.9362463 (429)\ttotal: 2h 48m 8s\tremaining: 26m 55s\n",
      "440:\tlearn: 0.9344849\ttest: 0.9363105\tbest: 0.9363105 (440)\ttotal: 2h 52m 16s\tremaining: 23m 2s\n",
      "450:\tlearn: 0.9345747\ttest: 0.9363465\tbest: 0.9363557 (448)\ttotal: 2h 56m 4s\tremaining: 19m 7s\n",
      "460:\tlearn: 0.9346324\ttest: 0.9363964\tbest: 0.9363974 (459)\ttotal: 3h\tremaining: 15m 13s\n",
      "470:\tlearn: 0.9346998\ttest: 0.9364182\tbest: 0.9364190 (469)\ttotal: 3h 3m 55s\tremaining: 11m 19s\n",
      "480:\tlearn: 0.9347243\ttest: 0.9364406\tbest: 0.9364406 (480)\ttotal: 3h 7m 41s\tremaining: 7m 24s\n",
      "490:\tlearn: 0.9347714\ttest: 0.9364925\tbest: 0.9364925 (490)\ttotal: 3h 11m 36s\tremaining: 3m 30s\n",
      "499:\tlearn: 0.9348314\ttest: 0.9365536\tbest: 0.9365536 (499)\ttotal: 3h 15m 9s\tremaining: 0us\n",
      "\n",
      "Time consuming: 203.09min\n"
     ]
    }
   ],
   "source": [
    "start=time.perf_counter()\n",
    "print('Start training cv model...')\n",
    "print()\n",
    "cv_data = cv(\n",
    "    params = params,\n",
    "    pool = Pool(traindata_x, label=traindata_y, cat_features=categorical_features_indices),\n",
    "    fold_count=3,\n",
    "    shuffle=True,\n",
    "    partition_random_seed=20,\n",
    "    stratified=False,\n",
    "    verbose=10\n",
    ")\n",
    "end=time.perf_counter()\n",
    "print()\n",
    "print('Time consuming: {:.2f}min'.format((end-start)/60))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model params:\n",
      "{'loss_function': 'Logloss', 'custom_metric': ['Logloss', 'AUC'], 'iterations': 500, 'learning_rate': 0.1, 'eval_metric': 'F1', 'random_seed': 20, 'use_best_model': True}\n",
      "\n",
      "Best validation F1 score of cv model: 0.936554±0.0002\n",
      "Best validation Logloss of cv model: 0.576101±0.0006\n",
      "Best validation AUC of cv model: 0.985125±0.0001\n",
      "Best iteration: 499\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>iterations</th>\n",
       "      <th>test-F1-mean</th>\n",
       "      <th>test-F1-std</th>\n",
       "      <th>train-F1-mean</th>\n",
       "      <th>train-F1-std</th>\n",
       "      <th>test-Logloss-mean</th>\n",
       "      <th>test-Logloss-std</th>\n",
       "      <th>train-Logloss-mean</th>\n",
       "      <th>train-Logloss-std</th>\n",
       "      <th>test-AUC-mean</th>\n",
       "      <th>test-AUC-std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.904635</td>\n",
       "      <td>0.000390</td>\n",
       "      <td>0.901984</td>\n",
       "      <td>0.000612</td>\n",
       "      <td>0.576101</td>\n",
       "      <td>0.000204</td>\n",
       "      <td>0.577309</td>\n",
       "      <td>0.000655</td>\n",
       "      <td>0.958149</td>\n",
       "      <td>0.000669</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0.906132</td>\n",
       "      <td>0.002957</td>\n",
       "      <td>0.904032</td>\n",
       "      <td>0.001926</td>\n",
       "      <td>0.483626</td>\n",
       "      <td>0.003378</td>\n",
       "      <td>0.486290</td>\n",
       "      <td>0.003456</td>\n",
       "      <td>0.966608</td>\n",
       "      <td>0.001205</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0.909752</td>\n",
       "      <td>0.001971</td>\n",
       "      <td>0.907047</td>\n",
       "      <td>0.001036</td>\n",
       "      <td>0.416538</td>\n",
       "      <td>0.001749</td>\n",
       "      <td>0.419862</td>\n",
       "      <td>0.001501</td>\n",
       "      <td>0.968959</td>\n",
       "      <td>0.000331</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0.911698</td>\n",
       "      <td>0.001893</td>\n",
       "      <td>0.909141</td>\n",
       "      <td>0.000588</td>\n",
       "      <td>0.365799</td>\n",
       "      <td>0.003243</td>\n",
       "      <td>0.370070</td>\n",
       "      <td>0.002723</td>\n",
       "      <td>0.970377</td>\n",
       "      <td>0.000497</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>0.915003</td>\n",
       "      <td>0.001260</td>\n",
       "      <td>0.911628</td>\n",
       "      <td>0.000500</td>\n",
       "      <td>0.324008</td>\n",
       "      <td>0.006229</td>\n",
       "      <td>0.329716</td>\n",
       "      <td>0.005565</td>\n",
       "      <td>0.972545</td>\n",
       "      <td>0.000904</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   iterations  test-F1-mean  test-F1-std  train-F1-mean  train-F1-std  \\\n",
       "0           0      0.904635     0.000390       0.901984      0.000612   \n",
       "1           1      0.906132     0.002957       0.904032      0.001926   \n",
       "2           2      0.909752     0.001971       0.907047      0.001036   \n",
       "3           3      0.911698     0.001893       0.909141      0.000588   \n",
       "4           4      0.915003     0.001260       0.911628      0.000500   \n",
       "\n",
       "   test-Logloss-mean  test-Logloss-std  train-Logloss-mean  train-Logloss-std  \\\n",
       "0           0.576101          0.000204            0.577309           0.000655   \n",
       "1           0.483626          0.003378            0.486290           0.003456   \n",
       "2           0.416538          0.001749            0.419862           0.001501   \n",
       "3           0.365799          0.003243            0.370070           0.002723   \n",
       "4           0.324008          0.006229            0.329716           0.005565   \n",
       "\n",
       "   test-AUC-mean  test-AUC-std  \n",
       "0       0.958149      0.000669  \n",
       "1       0.966608      0.001205  \n",
       "2       0.968959      0.000331  \n",
       "3       0.970377      0.000497  \n",
       "4       0.972545      0.000904  "
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_value = np.max(cv_data['test-F1-mean'])\n",
    "best_iter = np.argmax(cv_data['test-F1-mean'])\n",
    "print('Model params:')\n",
    "print(params) \n",
    "print()\n",
    "print('Best validation F1 score of cv model: {:.6f}±{:.4f}'.format(\n",
    "    best_value,\n",
    "    cv_data['test-F1-std'][best_iter])\n",
    ")\n",
    "print('Best validation Logloss of cv model: {:.6f}±{:.4f}'.format(\n",
    "    np.max(cv_data['test-Logloss-mean']),\n",
    "    cv_data['test-Logloss-std'][best_iter])\n",
    ")\n",
    "print('Best validation AUC of cv model: {:.6f}±{:.4f}'.format(\n",
    "    np.max(cv_data['test-AUC-mean']),\n",
    "    cv_data['test-AUC-std'][best_iter])\n",
    ")\n",
    "print('Best iteration: {}'.format(best_iter))\n",
    "cv_data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Overfitting detector (early stopping)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Staring training catboost model...\n",
      "0:\tlearn: 0.9014496\ttest: 0.9040733\tbest: 0.9040733 (0)\ttotal: 4.17s\tremaining: 34m 42s\n",
      "10:\tlearn: 0.9199248\ttest: 0.9245672\tbest: 0.9245672 (10)\ttotal: 54.7s\tremaining: 40m 29s\n",
      "20:\tlearn: 0.9239455\ttest: 0.9276165\tbest: 0.9276165 (20)\ttotal: 1m 50s\tremaining: 42m 11s\n",
      "30:\tlearn: 0.9259582\ttest: 0.9295252\tbest: 0.9296335 (29)\ttotal: 2m 56s\tremaining: 44m 26s\n",
      "40:\tlearn: 0.9270137\ttest: 0.9304274\tbest: 0.9304274 (40)\ttotal: 3m 55s\tremaining: 43m 53s\n",
      "50:\tlearn: 0.9279284\ttest: 0.9311954\tbest: 0.9311954 (50)\ttotal: 4m 53s\tremaining: 43m 3s\n",
      "60:\tlearn: 0.9288623\ttest: 0.9319872\tbest: 0.9319872 (60)\ttotal: 5m 56s\tremaining: 42m 47s\n",
      "70:\tlearn: 0.9292382\ttest: 0.9326317\tbest: 0.9326749 (68)\ttotal: 6m 51s\tremaining: 41m 25s\n",
      "80:\tlearn: 0.9298903\ttest: 0.9329703\tbest: 0.9329703 (80)\ttotal: 7m 52s\tremaining: 40m 43s\n",
      "90:\tlearn: 0.9303594\ttest: 0.9333166\tbest: 0.9333166 (90)\ttotal: 8m 44s\tremaining: 39m 16s\n",
      "100:\tlearn: 0.9307108\ttest: 0.9335643\tbest: 0.9335643 (100)\ttotal: 9m 41s\tremaining: 38m 18s\n",
      "110:\tlearn: 0.9310157\ttest: 0.9335795\tbest: 0.9336421 (106)\ttotal: 10m 38s\tremaining: 37m 17s\n",
      "120:\tlearn: 0.9312455\ttest: 0.9339607\tbest: 0.9339607 (120)\ttotal: 11m 34s\tremaining: 36m 14s\n",
      "130:\tlearn: 0.9315482\ttest: 0.9339971\tbest: 0.9339971 (130)\ttotal: 12m 33s\tremaining: 35m 22s\n",
      "140:\tlearn: 0.9319045\ttest: 0.9343434\tbest: 0.9343434 (140)\ttotal: 13m 25s\tremaining: 34m 9s\n",
      "150:\tlearn: 0.9321688\ttest: 0.9345603\tbest: 0.9345603 (150)\ttotal: 14m 16s\tremaining: 33m\n",
      "160:\tlearn: 0.9325114\ttest: 0.9346731\tbest: 0.9346969 (155)\ttotal: 15m 7s\tremaining: 31m 50s\n",
      "170:\tlearn: 0.9327933\ttest: 0.9350051\tbest: 0.9350051 (170)\ttotal: 16m 1s\tremaining: 30m 50s\n",
      "180:\tlearn: 0.9329590\ttest: 0.9350828\tbest: 0.9351188 (173)\ttotal: 16m 56s\tremaining: 29m 51s\n",
      "190:\tlearn: 0.9331564\ttest: 0.9352995\tbest: 0.9353036 (189)\ttotal: 17m 45s\tremaining: 28m 44s\n",
      "200:\tlearn: 0.9333124\ttest: 0.9352713\tbest: 0.9353036 (189)\ttotal: 18m 42s\tremaining: 27m 50s\n",
      "210:\tlearn: 0.9335089\ttest: 0.9354499\tbest: 0.9354499 (210)\ttotal: 19m 40s\tremaining: 26m 57s\n",
      "220:\tlearn: 0.9336024\ttest: 0.9355283\tbest: 0.9355599 (215)\ttotal: 20m 33s\tremaining: 25m 57s\n",
      "230:\tlearn: 0.9338498\ttest: 0.9356578\tbest: 0.9356702 (227)\ttotal: 21m 28s\tremaining: 25m\n",
      "240:\tlearn: 0.9339297\ttest: 0.9357381\tbest: 0.9357381 (240)\ttotal: 22m 17s\tremaining: 23m 57s\n",
      "250:\tlearn: 0.9340050\ttest: 0.9357542\tbest: 0.9357542 (250)\ttotal: 23m 9s\tremaining: 22m 58s\n",
      "260:\tlearn: 0.9341554\ttest: 0.9359462\tbest: 0.9359546 (259)\ttotal: 24m 9s\tremaining: 22m 7s\n",
      "270:\tlearn: 0.9343528\ttest: 0.9360797\tbest: 0.9360797 (270)\ttotal: 25m 3s\tremaining: 21m 10s\n",
      "280:\tlearn: 0.9344288\ttest: 0.9362101\tbest: 0.9362421 (277)\ttotal: 25m 53s\tremaining: 20m 10s\n",
      "290:\tlearn: 0.9345294\ttest: 0.9362350\tbest: 0.9363383 (284)\ttotal: 26m 50s\tremaining: 19m 16s\n",
      "300:\tlearn: 0.9345918\ttest: 0.9362642\tbest: 0.9363383 (284)\ttotal: 27m 47s\tremaining: 18m 22s\n",
      "Stopped by overfitting detector  (20 iterations wait)\n",
      "\n",
      "bestTest = 0.9363383336\n",
      "bestIteration = 284\n",
      "\n",
      "Shrink model to first 285 iterations.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<catboost.core.CatBoostClassifier at 0x1fb0f8c1b00>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Time consuming: 35.82min\n",
      "\n",
      "Model params:\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'model' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-15-a789c5799457>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     23\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     24\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'Model params:'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 25\u001b[1;33m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_params\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     26\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     27\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'Best score is:'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'model' is not defined"
     ]
    }
   ],
   "source": [
    "start=time.perf_counter()\n",
    "model_with_early_stop = CatBoostClassifier(\n",
    "    loss_function='Logloss',\n",
    "    custom_loss=['Logloss', 'AUC'],\n",
    "    eval_metric='F1',\n",
    "    iterations=500,\n",
    "    random_seed=20,\n",
    "    learning_rate=0.1,\n",
    "    early_stopping_rounds=20,\n",
    "    train_dir='it_500_lr_0.1'\n",
    ")\n",
    "\n",
    "print('Staring training catboost model...')\n",
    "model_with_early_stop.fit(\n",
    "    x_train, y_train,\n",
    "    cat_features=categorical_features_indices,\n",
    "    eval_set=(x_test, y_test),\n",
    "    verbose=10\n",
    ")\n",
    "end=time.perf_counter()\n",
    "print()\n",
    "print('Time consuming: {:.2f}min'.format((end-start)/60))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model params:\n",
      "{'iterations': 500, 'learning_rate': 0.1, 'loss_function': 'Logloss', 'random_seed': 20, 'train_dir': 'it_500_lr_0.1', 'custom_loss': ['Logloss', 'AUC'], 'eval_metric': 'F1', 'early_stopping_rounds': 20}\n",
      "\n",
      "Best score is:\n",
      "{'learn': {'Logloss': 0.1591147718794998, 'F1': 0.934599361877029}, 'validation': {'Logloss': 0.15473743586425584, 'F1': 0.9363383335591948, 'AUC': 0.9849992111102392}}\n",
      "Best iteration is: 284\n",
      "Tree counts of Catboost with best parameters: 285\n"
     ]
    }
   ],
   "source": [
    "print('Model params:')\n",
    "print(model_with_early_stop.get_params())\n",
    "print()\n",
    "print('Best score is:')\n",
    "print(model_with_early_stop.best_score_)\n",
    "print('Best iteration is: '+ str(model_with_early_stop.best_iteration_))\n",
    "print('Tree counts of Catboost with best parameters: ' + str(model_with_early_stop.tree_count_))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Plot auc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "validate_pool = Pool(x_test, y_test, cat_features=categorical_features_indices)\n",
    "curve = get_roc_curve(model_with_early_stop, validate_pool)\n",
    "(fpr, tpr, thresholds) = curve\n",
    "roc_auc = auc(fpr, tpr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVAAAAEaCAYAAABHKw/TAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXhU5dn48e+dhSQEEnYI+yI7FkERRMGAiBt1ZXEpWvfWttq3b3+ttvattuVt7aK19W2V2mprxQLuuFbQsKhUAQVZZN9lTUhC9mXu3x/PmTCMWSYhM5NJ7s91zTVnzjlzzn3OzNzznPM85zmiqhhjjKm/uGgHYIwxscoSqDHGNJAlUGOMaSBLoMYY00CWQI0xpoEsgRpjTAO1qAQqIg+IiAY8DorIayLylSjHdDRa649FIjJVRL5bzfinRWRVNGKqjoj09b5n06IdS11q2qenuMx6bb+ItPJ+D2ecynIiqUUlUE8ecI73+C4wCHhHRDpEKZ4ngYuitO5YNRX32QX7OfD1yIbSbNS0T0/FAdzvbEWI87cCfgqcETS+vsuJmIRoBxAFFaq60hteKSK7gA+Bi4F5kQ5GVfcB+yKxLhFJUdXiSKyrvhojNlXd3ljxxIIm/nkmq2oJsLLOmeugqqWNsZywUNUW8wAeAI4GjUsBFPhh0PgOwBPAIaAE+AAYGzRPPHAfsAUoxSXCp4PmuQJY5S3jIPBrILG6mIBUoBC4q5rYVwHPBLzuDfwLyAGKgLeBwQHT+3rbdQPwDyAXWFzLvukE/B3I9paXBZwVNM8u4LfAT7xtKQCeBdIbsO8U+B7we+AIsM0bfxnwDnAYyMf9cKYG7S8NejztTXsaWBUw79e96ad7yywEPgeuDopFcKVX/zr/BlzrvbdvHd+pPsBzwFFvv60Drg/6DGZ6+yPP+448CMQFLGOI91nu9ZaxAVcaDJwn01vWRcCr3r7/qzftv4GPveUfAhYBp1UT61XAR0Cx9zm/4cVf4z713ncesNSLLRv4C9C2mv18tve9Kfa+I/7tnxYw7+XAau+zOAb8Bzg/4DsR/Ohb3XK8+W8HPsN9xw4BzxP0XQz3oyUewgfr7T3v9I8QkSRgMXAh8P+AK3E/8sUi0i3gvU/gfgwLgGm4L3JqwHJmAi/ivrSXe/PeAfyyukBUtRB4DZgVOF5E+gNnAvO91x1whzODgW/gfqCpXnwpQYv9LXAcmAH8by374WXcj/P73vrjgPdE5LSg+a4DpuC+vN/DJbwnA2INdd/hTc8AZgN3e+P64RLAbOAaXPJ9U0TO9aY/iTtSOMiJUzE/r2W78OZ/FZdAtgL/EpGeAdO/C/wIeByYjksAv65jmYhIF9zRyxjcfvsq8FegV9Csv8YlvOnAP4H/8Yb9egCbgbuAS3EJ6kHgh9Ws9q/AWtz36a/euJ7AY7g/69txf+zvi0h6QKyzcd/F7bjvy824P/7O1LJPvf2+xJs23dtXlwJPVRPbc7jv76Xe80lEZAAuyb3r7asbvPn8p88me8+/CIjjQDXrQUTux/3+luK+Y9/E/YG0qW7+sIlkto72A6+0hzt1kQAMwJVMPgGSAua7FSgDBgaMS8B9+X4TUGpQ4O4a1iXAbuCpoPG34H6gHQNjCiolVALdA8bdhytptvJe/xxXEugQME973BfoW3py6eelEPbLxd685weMS8UlvicCxu3y4mgTMO4GwAcMDXXfeeMU+KSOuOK8974N/C1g/G+BXdXM/zTVl0BvCRjXEagAvuG9jsf9SP8vaFlvUEcJFPdHWAhk1DDd/xn8I2j8p8C/avneJOAS+o6A8Znesh6pY5/F446qjgM3BuzH/cCLtbyvpn26HHgvaNxkL5YRQfv5nhq2f5r3ejqQXUsMbbz5v17HctrhSsMP1/XdDvejJZZAOwLl3mMbMAp3SFcaMM8U3GHGThFJEBH/ueKlwFne8CTv+eka1jMIV7pd4F+Gt5x3gWRgRA3vexNXWpkRMG4WLhGWBcT3DpAfsNzjXsxncbLXa1hPoLOBI6q61D9CT5SGzwua9x1VLQh4/SLuRz8mILa69l2NsYlITxH5u4jsxyW6clwFx6AQtqMm//YPqGo27lDdXwLtBXTDlVADBb+uzmTgLVWttpRU3fo9GwPWj4gki8iDIrINdyqoHJgD9AvYf37V7bNxIvKOiGTj9lkRLhn599lgoDvVlxprJCKtcaXA4O/wCi/GM+uKLchnQLr3+U4VkdQ65q/JObg/iXptTzi0xASah/uxjwPuxNX8zRORwH3RyZteHvS4mROHZx2BQlXNr2E9nbznN4KW4T9VEHyYB4C6E++v4B3Gi8hgYCTuHFngsmdVE9+kapZ7qIb4AmXUMN8hThxe+R0OircYl/AzAmKra99VG5v3GbwKjMcd5k7CfVZv4v50Gio36HVZwPL8pxWOBM0T/Lo6HanhELMe6wd4CHcKYC7u8HcM7jAWvrzdwfusNy5BC+77fK73/sMB7+3oPYcSa6D2uBLtnzj5sywFEqnnd01VN+NOM/TH/S6Oisg8Eelcz7gauj2NrqXWwvvbCv5HRIpxlSwz8M4x4g5TV+HOqwTzl1SzgVQRSashieZ4z3fgThEE21nNOL/5wCLvxzEL92N+N2jZr1L9ub/jQa9D6a/wANClmvFdObEdfifN551zbcOJL3Mo+66m2E7DHRFcoqpvBa0jXA56z8E/4lB+1Nmc+OM4FTOAP6pq1XlXEbmshnmD99nFQGvgCu+oAa+UGPjHl+091zfWXG99D+ASXrAv6ojtS1T1deB17/zsZbhKxD/iKu1CFbg9UW1D3RITaLB/4k7W/5ATCXQJ7rBxj6oeruF9/oR2I+4EfrDNuPNOfVX1L/WM6d+4GsqZuAT6vKpWBkxf4k3boI3TjOU/wIMiMlFVl0HV4dtlwEtB814oIm0CDuOvxv1w/H9Koey7mvgTZVWiFZE+uFLVuoD5gktwp2IvLolegTvX6nd5CO9dAtwtIl1VNZSSfk1SOHmb4wk9oaTgzkFXBIybycm/bf938SZcBV11vrRPVbVQRFbiWnf8LMR4QqKqebgjv/Nxh+T+GAiOoxof4uoRbsKV3KOmxSdQVVUR+V/gWRG5QFWX4Eqk3wCyROS3wA7cYcPZwEFVfURVN4vIXOB3Xm3sMtzJ7emqeq2q+kTkv4FnRCQNdxhahjt8udKbr6iGmMpF5CVcLXcGrnY20MPA14B3ReSPuB9HV+B8YIWqPlfPffC2iLwPzBeRe3H/8N/H/Th/EzR7Ma4E8Rsvtt/gzs9u9KbXue9qCeVzXDOf34nIT4C2uNro/dXM11VEvg6sx1XC7arPNvupaqW3Lb8RkSPA+7jkebo3i6+Wtz+C+wNdLiJzcMl4KJAaWJoMwTvAt7xzoDnAt4CkEN/7Lu4w+ykR+SswHPfZVZ028L6LP8B9x5/F1ZYr7hzuc94RWU379AfAEhHx4WrQj+PO7V8G/FhVt4S6kSJyJy5ZvoUrvQ7Elb7/4cVZJiI7gZkish7XPGld8HJUNVdEfg7MEZFWuNJxkhfTg6oa/H0Jn2jXYkXyQTXtQPVEzeUW4O2AcenAo7gfRRnuh/0icG7Q+36ESxL+eZ4KWvYluJrMQlwbw09x57cS6ohpCu5Lvp+A9oAB0/2VAodwpZdduNL0cK2m5jKEfdMZ90U+hkuSS4ExQfPsAn7nxXzI26bngHZB84Wy7xT4djVxjOFEW8WtuBrepzm5dj3Z2/bDhNYOtE012/HbgNf+dqBHcAniWdwpCA3etmri7YM7cjmGq7xZC1xb22dQTZxdcSX9fG+//hrXHKkqdk7Uwo+oJoYbca0cinHtZscGb6M339W4Cr4S3J/k60Cf2vapN20sLunle5/5RtyfeHod+/mk7cclz9dxybMEdxrrIU5uATMVlzRLqLsd6J1eLKW4o4gFQFokc4p4gRhTJ++qredVNaqHTZEgIk8CF6pqn2jHYpquFn8Ib4yIjMCda/4Ad8h+Ca7VQHUN2Y2pYgnUGHdYeh7wbdwFBLtxyfN30QzKNH0ROYQXke64RtnDcOdJKgKmjcBdQifAN1X1SyeNjTGmKYpUAk3G1ei+BEwJSqAv4a6D9gF/UtUrwh6QMcY0gogcwqu7uqZERKqb3EFV9wIEdn4QSETuwDVIJzk5+czevXtXN1uz4PP5iItrCheIKYJ6TaP9wz53nOD96UpVu2kFVQQfbgYAH6KKirgx6kN9lUhc/Enr8JPANtgn/akr1X5rmiCfKnHVf8ebhea4fRWVkFcYh6pwIOfwUVWt11VRTeEcaFwNw1VUdS7uMjcGDx6smzdvjkRcUZGVlUVmZmbob1CFimLvUQKVpeCrAF+ZGy4vhvICiEt0ryvLoPQY+CohLgHKvAuX/PP7KlyiDINdu3bRt2/fhr1Z4kHi3HNcvPda3Di858Bh9bntbN3txDSJc+8h4DV446pWFDQuxGkifPbZZ5x++lfqXqYqlOZBSscT0wOXKRL0nuDhoNdledAqvfp1VqfWJFjztNVrVnPmmcHdGYT23lNZb53TG7jcjZvzefmtg5SV+ejcMYlv33fV7jqC+JKmkEB9NQy3PKrE+Uqg8CCUF0LxUUCgogjKi9y4suNwfA8ktvGSXjmhXa1ZH+KSq68cktIhrhXEe4/io9C2l5su8S4xxyV483sXSyWluWla6d6bkAxxCRzKX0nf4eeBJAQlNf/DS5Jx8YD3XDW+6Zd8sve2hQGZ0Q4jbI6nFkG3MXXPGCOObNpFWWIlw8/ozBVXDOHb99V/GU0hgeZ4fTP6cB19NF+VZVCcDWX5LkmikL/bK5HkQtlxeh/cDmtD6Hy7PKBDpPhkSExxz/FJLtHFJXivE70SpUByB296oltnYmv3WuJcQvZPk/CcQihO3g/p/cKybGNCoar4TyVOnNiHLl1SGTKkEzWcXqxTRBKoiCTiLmUcCbwtIj8DzlPVObh7oPwLV9b+ViTiCauKUldKK8mGwkPuuSTHHbJVBvel8WU+SYDk9i75Jbd3Jc70fpDQ2iW8xDbQqg0kpHglw/AlPGOak71783j99a1cf/3ppKUlISIMHVrfjqBOFqlKpHLcpYmBlnrT1vHlPiebPlUoOgxFh1zpMXeHS5ZlBdR4SC3x7vA2Phlad3WJL7WbS4ZtekCrtuwpe5/+ozMjuSXGNGuqykcf7eftt7fj8ynvv7+HSy4Z2CjLbgqH8LHBnzDzd0P+LvdcXlj9vKndIKk9tO7sEmVyB0hq584FxsC5PGOai7KyShYt2sxnn7mOwc45pydTpvRvtOVbAq1NWQEc/gSyN3jnLIMktIY2GZDS2SXL1O7uOc52qzHRlp1dxPz5Gzh8uJBWreK54orBDB9eXbe3DWe/9GCleXBkHeRuc6XMQHGJ0H4gpPWBdqe5kqWVKI1pcoqLy3nyyTUUF1fQqVNrZs0aTufODb2DSM0sgYI7PM/bCXuzXBMhP4l3FThtekCHIe7Q3BKmMU1eSkoi557bmy++OM4VVwwmKSk8qc4S6NENsH/5yYfobXu5kma3sZAQar+2xphoKiws49ixEnr2TAPg3HPdLZsa2kQpFC03geZuh51vQfERQFxNeJfR0OUMdx7TGBMz9u3LZ8GCDVRU+LjzzjNJT08Oa+L0a3kJtOAAbHvJ1aiDa0rU4zzofq6VNo2JMarK6tUHePPNrVRWKr16pREXF7nTbC0ngaq6c5z7lp4Y120M9J5iidOYGFReXslrr21h7Vp3P7+xY3swdeoA4uMjd2FJy0igFaWw4Wko9O682+40GHC5a9RujIk5OTnFzJ+/nkOHCklMjOPyywdz+uldIx5H80+gFaWw7gl3OSXAgCug66joxmSMOSX5+aUcOVJEx44pzJo1gi5dGr+JUiiadwKtLIPN813yTEyFEbdCSodoR2WMaYDAjkD69m3HzJnD6du3HcnJ0UtjzbcXipJcWPcXyNvhrhga/nVLnsbEqKKicp599jO2bcupGjdkSKeoJk9orgnUVwmb/+WaKCW3hxG3WNMkY2LUF18c54knVrFtWw5vvbUNn6/p3Iq9+R3Cq7pmSoUHXa9Hw250SdQYE1NUlTVrDvDGG66JUs+eacycOTyizZTq0vwS6P4VcHS969Bj2GxLnsbEoPLySt54YyuffOKuEBwzpjsXXXQaCQlN66C5eSXQklzYs8QN95oEbXtENx5jTIMsXLiRLVuySUyMY9q0QYwc2S3aIVWreSXQPYvdc6u20H18dGMxxjTYeef1JienmOnTh9GtW5toh1OjplUePhUFX7hDd4BBM63XJGNiiKqyc+exqte9e6dz111jmnTyhOaSQFXh8+fccMZYSOsV3XiMMSErLi5n3rzP+Pvf17JlS3bV+KZUWVST5nEIn73Ru7+5QO8Loh2NMSZEBw4cZ/78DeTmlpCSkkB8fNNPmoGaRwLdv9w9dx/vbulrjGnyPvnkAK+/vpWKCh/du7dl5szhtGuXHO2w6iX2E2h50YnOkLuOjm4sxpg6VVT4ePPNraxe7Tr3OfPMDC65ZGCTa6IUithPoEc/OzGc0jF6cRhjQlJaWsHWrTkkJMRx2WUDGTUqI9ohNVjsJ9D977vnPhdGNw5jTEhSU1sxa5a7oigjo220wzklsZ1AywqgLN8Ndz0zurEYY6qlqixfvoeKCh+TJ/cDoEeP5tEXb2wn0Nzt7jm5IyTE1slnY1qC4uJyXnrpc7ZsyUYERo7sSseOraMdVqOJ7QR6bIt77jYmunEYY77k4MEC5s9fz7FjronS1VcPbVbJE2I9gfpr39P7RjUMY8zJ1q49yKJFW6io8JGR0YaZM4fTvn1KtMNqdLGbQCvLoCQbJB5SrK9PY5qKjz/ez+uvbwVg1KhuXHbZoJhsohSK2E2gx/e555SOEBcf3ViMMVWGD+/CypX7OPfc3oweHbtNlEIRuwk0f7d71qbTO7UxLdW+ffl0796WuDihdetE7rprTERvLxwtsbuF/gTacVh04zCmBXNNlHbz17+uYcmSHVXjW0LyhFgtgfoqIH+XG+5yRlRDMaalKimp4KWXNrF5s+tBKSEh7qQ7Z7YEsZlAi46457gEu2WHMVFw6FAB8+dvICenmORk10Rp0KCWdyl1bCbQ4qPuud3A6MZhTAu0bt0hFi3aTHm5j27dXBOlDh2aXxOlUETsRIWIPCIiy0Xk0aDxM0TkIxH5j4hcEdLCCl0vLrTu0uhxGmNqpqqsW3eI8nIfZ5zRjVtvHdVikydEqAQqIqOBVFWdICJ/FpExqvqxN/m/gExAgbeAV+pcYLF3CJ/aNG80ZUxzJSJcffVQtmzJZuTIri3qfGd1InUIfw7g3fGNxcA4wJ9ANwOp3nB+SEsr9rr9T+nUWPEZY2qwc+cxPvpoP506uSaDrVsncsYZVniByCXQdoDX8wd5wPCAaS8Ca3CnE26u7s0icgdwB0Dnzp3ZuflTRH3sLl2LSmyexq1JQUEBWVlZ0Q4jLJrztkHz2z5VZf36fNasOYYqfOUrKcTHZ0U7rCYlUtknF/D3X5Xmvfb7OScS6hvAv4PfrKpzgbkAQwYP1H59ekNCCn3PnhK+iKMkKyuLzMzMaIcRFs1526B5bV9paQUvv/w52dlCnz7pTJjQG5HdzWb7GkukKpE+BPx3e5sCrAyYVgoUAYVA3Tc0Up97TkpvzPiMMZ7DhwuZO3c1mzYdJSkpnmuvHcEFF/SPibtkRlpESqCqukZESkRkObAW2CMiP1bVOcCfAa9beVfKrI1opRtIahemaI1puQ4fLuTJJ9dQVlZJly6pzJo1vNl1QdeYInYCUVXvCRo1xxv/NPB0qMsRvGvfE+xDNaaxde7cmr5925GcnMC0aYNo1co66qlNDNbA+BNoy217ZkxjOn68FFVIS0tCRJg5czjx8dLimyiFIuau+I/TCjdgt/Aw5pTt3p3LE0+sZsGCDVRWuvqFhIQ4S54hirkSqPpzfmVpdAMxJoapKitX7uOdd3bg8ymdOrWmrKySlJSYK1NFVcgJVETOAWYDGap6lYiMwl1dtCJs0VXLO4RPbnkdFxjTGMrKKnnllc/ZsMFd0Xfuub2slr2BQvq7EZFrgde9l/7mSHHAz8IRVO28BBpfd4snY8zJjh4t4i9/Wc2GDUdISopn1qzhXHjhAEueDRRqCfR+4GJV/UhErvPGfQaMCE9YNRN/D/R2DtSYetu8+ShHjhTRuXNrZs0aQadO1prlVISaQHuo6kfesP8eGhVAFNo4+EugSZFftTExbvz4XsTFCWee2d2aKDWCUM8Y7xCRcUHjxgFbGzmeOoklUGNCVlBQxvz568nLKwFcb0rnnNPLkmcjCbUE+r/AKyLyCJAoIvcA3wO+FbbIauQl0LjEyK/amBiyZ08eCxdu4PjxMnw+5brrTo92SM1OSAlUVV8QkSLgbmA/cCnwDVV9M5zB1RCMe7aG9MZUS1X56KP9vP32dnw+pU+fdKZNGxTtsJqlkJsxecky8gkziKAgcVYLb0w1ysoqWbRoM599dhiAc87pyZQp/VvMXTIjLaQEKiL5qppWzfgcVe3Q+GHVIa4V2JUSxpykstLH3/72CQcPFtCqVTxXXDGY4cPttjfhFGoJtGllKyt9GvMl8fFxnH56FyoqfMyaNZzOnVPrfpM5JbUmUBH5kzfYKmDYbwBRqIUHoKIoKqs1pqnx+ZTs7KKqZDl+fC/GjOlhtewRUteJkbbeQwKG2+LuYfQpcH1Yo6uJVSAZQ2FhGc88s5a//e0TcnNPNFOy5Bk5tZZAVXU2gIhsVNVfRiakELSy3uhNy7ZvXz4LFmwgP7+U1NRECgrKaNfOrs6LtFCbMTWd5AmuFt6YFkhVWbXqC956axuVlUqvXmnMnDmctm3twpJoCLUWvgPwCK4jkc4EVCqpauRrdEpyIr5KY6KtvLyS117bwtq1hwAYO7YHU6cOsCZKURTqnn8M6Ie78qgMmAGsAr4fprhq16Z7VFZrTDQdOlTIZ58dJjExjmuuGcollwy05BlloTZjmgyMVNVDIlKpqq+IyKfAAuAP4QuvBmInyU3L07NnGldcMZiMjLZ06WJNlJqCUP++koDD3nCxiLRW1d3A0PCEVYc4S6Cm+fP5lHff3cmWLdlV40aO7GbJswkJtQS6BRgFrAHWAT8UkXxOJNXIqiiJymqNiZSionJeeGEj27cfo3XrRL773XHWPKkJCjWB/hhoHTC8ANce9I5wBFUnuxLJNGP797smSnl5ronS9OnDLHk2UaE2Y1ocMLwK6B+2iELRqm1UV29MOKgqa9Yc4I03tlJZqfTs6ZoopaVZE6WmqsF35RSR7sADqhr5UqhVIplmaMmSnaxYsQeAs8/uwUUXWROlpq7OT0dErheRh0XkDhGJE5HWIvJrYBtwWvhDrEZczN2N2Zg6DR7ckeTkBK66agiXXmpNlGJBXZ2J/AS4D/gc+AYwHJgE5AGXqmpWuAOsPjArgZrmITu7iI4dXfVCr17pfPe740hOtgJCrKjrL+5G3N04RwNfBb4DzFfVCVFLnmAJ1MQ8n095772dPPbYR3z++dGq8ZY8Y0tdn1YXVV3mDb8LlAMPhTekENi18CaGFRWV8+KLm9i2LQcROHasONohmQaqK4EGXvOuIlKoqhVhjqlu1pDexKgvvjjOggUbyM0toXXrRK65ZigDBkT+pg6mcdSVQFNE5I2A122CXqOqlzZ+WHWwQ3gTg/xNlCoqfPTo0ZaZM4eTnm5d0MWyuhLor4Jerw5XIPViPdKbGFNaWsHSpbuoqPBx1lndufji00hIsFNRsa6uDpV/EqlA6sU6VDYxJikpgZkzh3P4cCGjRmVEOxzTSGLzL9AqkUwM2LYth2XLdle97tEjzZJnMxObbSasEsk0YarKsmW7ycrahSr06ZNOnz7toh2WCYPYTKAxWnA2zV9xcTkvvfQ5W7ZkIwKTJvWld2875dRcRSyBisgjwFnAGlW9J2B8B+BxoBOwRFXnRComYxpTTk4Zc+eu5tixElJSErjmmmGcdpo1UWrOQk6gIhKHS4C9VPUFEUnGNQ8tDeG9o4FUVZ0gIn8WkTGq+rE3+afA/6jq5yFHLVL3PMZE0PbtObz++gF69WpFRkYbZs0aYXfJbAFCvalcP2AR0AfXuP4F4FLgKmB2CIs4B/B3ibcYGAf4E+gI4Eci0gv4kap+WM3678Dre/S0jNas/HgNJUkHQgk95hQUFJCVlRXtMMKiOW9bSUklcXEVJCYe5bTTlE8/XRntkBpdc/78GirUEuhjwIvAA4D//gLvAg+H+P52wHZvOA/XKYnfeGA0kINLzOcFv1lV5wJzAb7Sv5OOO2c8pPUJcdWxJSsri8zMzGiHERbNbduOHy+ldevEgF6TlnDxxRdENaZwam6fX2MItTZmLPAzVfUBCqCquUD7EN+fC6R5w2nea78tqrpJVQ8BvtAWZ5VIJrp27DjGn/+8isWLd1SNS0621iEtTaiZKB84qSpRRDII/Z5IH+LuKQ8wBQg8vtkiIhkikkqoJWJrB2qiRFVZvnw3zzyzlqKico4cKaKyMsT/fdPshJqJXgae9JImItIOd/g+P5Q3q+oaoEREluNKmXtE5Mfe5J8Cz+FOCfwipGisEslEQUlJBf/613qWLNmJKmRm9uX660+3jo9bsFDPgd4PPA3s915n424sF3KTo8CmS5453viNQGaoywGsBGoi7tChAubP30BOTjHJyQlcffVQBg3qGO2wTJSFelO5ImCmiHQD+gK7VTV61eCWQE2ELV26m5ycYrp1a8OsWcNp3z4l2iGZJiDUZkzXAi+q6kHgYHhDCoUlUBNZX/3qINq3TyYzsy+JiVZZZJxQM9FDwEER+T+vUXx02TlQE2b5+aXe7YVdBVFKSiIXXjjAkqc5SajnQPvias+/DqwQkW3A34BnVDW7lveFiSVQEz47dx7j+ec3UlhYTnJyApMn94t2SKaJCvUcqALvAO+ISBpwLXAb8Esg8ieDrARqwkBV+eCDvSxevANV6N+/PWPH9oh2WKYJa0hnIr2AwbjLOqNQ+gQrgZrGVlpawcsvf86mTe4OmRMm9GbSpH7Exdl3zdQs1EqkdsD1wGq1BPAAACAASURBVC24yzBfxR3Ovx22yGoPKCqrNc1TQUEZTz31CdnZxSQlxXP11UMZPLhTtMMyMSDUEuhBYD2uLeg8Vc0JW0QhsQRqGk9qaiKdOrUmISGOWbNG0KGDNVEyoQk1gZ6tquvCGokxEVRZ6aOkpILU1FaICFdfPZS4OLFadlMvoVYiNa3kaYfw5hQcP17KwoUbqaz0cfPNo0hIiCMpKUZvzmCiqsZvjYgcVtUu3nAxXi9MwVS1dZhiq4UlUNMwu3fnsnDhRgoKykhLSyIvr4SOHaPwFTbNQm1/uzMChqeFO5B6sRKoqSdV5cMP97F48Q58PqVfv3ZMnz6M1NRW0Q7NxLAaE6iqLg14uVlV9wXPIyI9wxJVnSyBmtCVllbwyiub2bjxCADnntuLCy7ob02UzCkL9cTPRk50iBxoHRCFu2bZF9+Ebv36w2zceISkpHiuvHIIQ4d2jnZIppkINYF+KWOJSAI1nBcNOzuEN/UwenQGOTnFjBqVQadOdr7TNJ5aE6iIvOENJgUM+/UBPg1LVHWyBGpq5vMpWVm7GD06g3btkhERLrxwQLTDMs1QXSXQ1d7zBQHD4HqVX0SIPdIbEykFBWUsXLiB3bvz2LHjGLfeOgqxIxYTJrUmUFX9CYCIbFLVeZEJKQT2gzDV2LMnj4ULN3D8eBlt27Zi6tQBljxNWIXakL7pJE/ADuFNIFXlP//Zz7//vR2fT+nTJ50ZM4bTpo01UTLhFZsN6a1UYTyqyssvf87atYcAGD++Fxdc0M9u9GYiIjYb0hvjERG6dWvDpk1HufLKIQwbZk2UTOSE1JBeVZdEJpxQWQm0pSssLKu6imjcuJ4MG9aZ9PTkKEdlWpqQjnNE5AoRGewN9xORxSLylohE514HdgjfYvl8yjvvbOexxz7i2LFiwJVCLXmaaKjPTeWKAoYPA/uAP4YjKGOqU1hYxjPPrOX99/dSWlrJ3r350Q7JtHChXonUTVX3ikg87uZy/YASYH/YIquVlUBbmn378lmwYAP5+aW0adOKGTOG0adPu2iHZVq4UBNomYik427nsUNV87xLOa2diAkrVeXjj7/g7be3UVmp9O6dzowZw2jbNinaoRkTcgJ9FXdXzra423oAjAD2hiGmutk50BYjJ6eYt97ahs+njBvXkwsv7G9NlEyTEWoCvQu4FSjjRALtCMwJQ0y1ik7vJSZaOnZszaWXDiQ5OYERI7pEOxxjThLqlUhlwJ+DxjWxpk2mudi82d1a2H9nzLPO6h7NcIypUajNmERE7hGRz0Qk13u+R+xCY9OIfD5lyZIdPPfcel58cRP5+aXRDsmYWoV6CH8vcAfwG2A7MAD4PtAa+GV4QjMtSVFROc8/v5EdO44hAhMn9qFtW6ujNE1bqAn0FuBSVd3kHyEi7wGvYQnUnKL9+10Tpby8UlJTE5k+fRj9+rWPdljG1CnUBNoR2Bo0bhtRuZ2HaU7WrTvEK698TmWl0rNnGjNnDictzZoomdgQanuQT4H/Dhr3XWBt44ZjWpoOHVIAOPvsHtx88xmWPE1MCbUE+j3g3yJyJ7AT6ItrEzo1THHVwuqtYl1JSQXJye6r17NnGt/61tlVidSYWBJSCVRVPwUGAT8FlgAPAIO88SERkUdEZLmIPFrNtBQROSgiU0JdnolNW7Zk8+ijK9m06UjVOEueJlbVWQIVkVSgP7BNVZ9pyEpEZDSQqqoTROTPIjJGVT8OmOUOYH1Dlm1ig8+nfPLJMbKyPgNg06ajdnthE/NqLYGKyARchyFrgX0icnYD13MOsNgbXgyMC1hHK2AssKKByzZNXFFROfPmfcbatXmIwAUX9OOqq4ZEOyxjTlldJdD/Bf4J/AW4E/g5cFED1tMO134UIA/XKYnfzcAzuCRaLRG5A1dKZUC3VLKyshoQQmwoKChoVtt39GgpWVlHKCioQKSCgQOLqazcydKlO6MdWqNrbp9dsOa+fQ1RVwIdBkxR1VIR+X98uSlTqHKBNG84zXuN16PTRap6tYjUmEBVdS4wF+Ar/TtpZmZmA8No+rKysmgu2+fzKf/3fx/RqVMSI0e2JSPjGNOmNd/T3M3ps6tOc9++hqirEilRVUsBVLUQaGi33x/i7i0Prj/Rld5wV6CXiLwFfA34pYhYC+pmIi5OuOaaYV4TpVG0aRNqow9jYkNd3+hEEZnBibZDwa9R1QV1rURV14hIiYgsx51P3SMiP1bVOcAYABF5AFihqsdqXZY1Y2rScnNL2Lz5KGPH9gSge/e2dO/eNspRGRMedSXQbODhgNe5Qa8VqDOBAqjqPUGj5gRNfyCU5Zima9u2HF54YSPFxRWkpyczZEinaIdkTFjVmkBVtWekAjGxS1VZtmw3WVm7UIVBgzrSp096tMMyJuzspJQ5JcXF5bz44ia2bs1BBCZP7seECb2xng5NS2AJ1DTY0aNFPPvsOo4dKyElJYFrrhnGaadZ/zKm5bAEahosNTURcBVFM2cOp107uze7aVksgZp6qajwIQLx8XGkpCQye/ZI0tKSSEiwG72Zlse+9SZkeXklPPXUJ/z739urxnXokGLJ07RYod4TKV5E7hWRTSKS7Y2bKiK3hzc801Rs357DE0+sZv/+42zenE1JSUW0QzIm6kItOjwITMd1ZxfvjdsGfDMcQZmmQ1VZvnw3//znOoqKyjnttA7ccceZVf15GtOShfor+BowXlW/EJHHvXH+jpVNM1VSUsFLL21i8+ZsRCAzsy8TJ/YhLs6aKBkDoSfQVOBg0LhWQFnjhmOaknff3cnmzdkkJydwzTVDGTiwY7RDMqZJCTWBfgLcBDwVMG4W8FGjR2SajMmT+1FQUMaFF/anfXvrNd6YYKEm0O8DWSIyC2gtIi/hOkmeHLbITMRVVvpYuXIfY8f2JCEhjuTkBGbOHF73G41poUJKoKq6TkSG4Uqhu4DdwF2qeiCMsZkIys8vZcGCDezbl09eXimXXjow2iEZ0+SFXJWqqgeBh8IYi4mSnTuP8fzzGyksLCc9PYmRI7tGOyRjYkJICVREflDTNFX9deOFYyJJVXn//b0sWbIDVRgwoD3XXDOM1q0Tox2aMTEh1BLoV4NeZwC9cT3LRziBWhOaxlBeXsmLL25i06ajAEyc2IfMzL7WRMmYegj1HOiE4HEi8l3AOn2MUf7LL5OTE7jqqiEMHmydHxtTX6dyOcljwD7cVUomRpSXV5KYGI+IcOWVQygsLKdDB2uiZExDnEovEMM5cVmnaeIqK328+eZWnnrqUyoqfAAkJSVY8jTmFIRaifQm7v5Hfqm4m8H9IRxBmcZ1/LhrorR3bz7x8cLevXn062c3PzXmVIV6CL8q6HUB8DNVXdLI8ZhGtmtXLs8/v5GCgjLS0pKYOXM4PXumRTssY5qFOhOoiCTgLuV8Q1VLwh9SHaySOCSqyocf7mPx4h34fEq/fu2YPn0Yqamtoh2aMc1GnQlUVStE5O+qajf3jiFbtmRXdXx83nm9mTy5nzVRMqaRhXoIv1pERqjq+rBGYxrNoEEdGT06g4EDOzB0aOdoh2NMsxRqAn0HWOT1Bbob8PknqOqCcARm6m/jxiNkZLShffsURITLLx8c7ZCMadZCTaDf8J6/HTReAUugUVZZ6WPx4h18+OE+unVrw223jbb7FBkTAaFeidQr3IGYhikoKGPhwg3s3p1HXJwwalQ34uPtXKcxkVBrAhWRfFW1Ni9N1J49eSxcuIHjx8to27YVM2cOp1cvu7rWmEipqwRqRZkm6qOP9vPWW9vw+ZS+fV0TpTZtrImSMZFUVwLVOqabKFFVfD5l/PheTJnS35ooGRMFdSXQZBH5W20zqOotjRiPqUVlpY/4eFc5dPbZPejRI82uKjImikKpqq2s42EiYOPGI/zhD/8hJ6cYABGx5GlMlNVVAi1R1dsjEompls+nLFmyg/ff3wvAJ58c4IIL+kc5KmMMnFp/oCbMCgrKeP75jezalUtcnHDhhf0ZN65ntMMyxnisFr6J2rs3jwULXBOlNm1aMWPGMPr0aRftsIwxAWpNoNaBSHQUFZXzzDPrKCurpHfvdGbMGEbbtknRDssYEyRih/Ai8ghwFrBGVe8JGP8EMALXZOouVV0XqZiaqtatE5k6dQDZ2UVMmdK/qubdGNO0RCSBishoIFVVJ4jIn0VkjKp+7E3+laruFJGBwK+AayIRU1OTnV1Ebu6J7lbPOqt7FKMxxoQiUiXQc4DF3vBiYBzwMYCq7vTGl1NDsygRuQO4A2BAtzZkZWWFM9aI27OniOXL3e2FJ01Ka3bb51dQUNBstw1s+1qiSCXQdsB2bzgPd0O6YL+khnssqepcYC7A6QO6aGZmZhhCjDyfT3n33Z3s2LGHHj16M3RoJ9q3P0xz2b5gWVlZzXbbwLavJYpUAs0F/K2+07zXVbx7zG9U1RURiifqCgvLeOGFTezYcQwRmDKlP+PH92Lp0qPRDs0YE6JIJdAPgTtxfYdOAZ72TxCRqcB4YFaEYom6/fvzmT9/A/n5paSmJjJ9+jC7S6YxMSgi1buqugYoEZHluN7s94jIj73JfwT6Ae95NfLNnohQWFhGr15p3HnnWZY8jYlREWvGFNh0yTPHG98i7jvh82lVj0ndu7flppvOoEePttZEyZgYZr/eCMjJKWbu3NVs2HC4alzv3umWPI2JcfYLDrMtW7KZO3c1Bw8WsGLFHlSti1VjmgvrTCRMfD4lK2sXy5btBmDIkE5ceeUQRKx7AWOaC0ugYVBUVM4LL2xk+3bXROmCC/pz7rm9LHka08xYAg2DhQs3sHNnLq1buyZK/ftbLbsxzZEl0DCYOnUAb7+9nauuGkJ6enK0wzHGhIlVIjWC8vJK1q8/UcOekdGWm24aacnTmGbOSqCn6NixYhYs2MCBAwWoKqef3hXAznca0wJYAj0FW7dm8+KLmygurqB9+2Q6d06NdkjGmAiyBNoAqsrSpbtZunQXqjBoUEeuumoIKSmJ0Q7NGBNBlkDrqbi4nBdf3MTWrTmIwOTJ/ZgwobcdshvTAlkCrScRITu7mNatE7nmmqEMGNAh2iEZY6LEEmiIVBURITk5geuuG0FiYjzt2lktuzEtmSXQOlRU+Hjzza3ExQmXXTYIwCqLjDFATCbQyJ1rzM0tYcGCDXzxxXESEuIYP74X7dunRGz9xpimLQYTaGRs357DCy9soqionHbtkpk1a7glT2PMSSyBBlFVli/fw3vv7UQVBg7swNVXD7UmSsaYL7EEGuSDD/by7rs7EYHMzL6cf34fa6JkjKmWJdAgZ53Vnc8/P8rEiX0YOLBjtMMxAXJzczlw4EC0w6hReno6mzZtinYYYdOcti85OZmePXuSmHhqR5aWQIHPPz/Kaad1ICEhjqSkBG65ZZSVOpugo0eP0rdvX1JSmua56OPHj9O2bdtohxE2zWX7VJXs7Gz27dtHv379TmlZLbo3pooKH6+/voV//Ws9b765tWq8Jc+mqby8nORka3trTo2I0LFjR0pKSk55WS22BJqfX8qCBRvYty+f+Hihe/fY/2dtCezPzTSGxvoetcgS6M6dx3jiiVXs25dPenoSt946mjPP7B7tsEwTkJWVRZ8+fZg0aRIXXngh2dnZABQVFXHzzTeTmZnJpEmTWL16ddV7nnvuOSZOnMhFF13E1772NcrKyiIe98KFC3n55Zcjvt5Azz77LOPHj2fatGnk5+efNG3Pnj1MnjyZiRMnsmDBAgAOHjxIZmYm559/PrfccstJ899999187WtfAyAvL49p06aRmZnJ73//ewDWr1/P+PHjmTBhAjfffDOqWu24tWvX8pvf/CZs29yiEqiqsmLFHv7xj7UUFpYzYEB77rzzLCt9mpPMnj2b9957j5tuuonnnnsOgAcffJCpU6eSlZXF/PnzufvuuykrK2PDhg3MmzePxYsX8/bbb3P33XdTWVnZ4HX7fL4GvW/evHl89atfDdvy61JeXs7jjz/OsmXLmD17Nk888cRJ0x966CHmzJnDe++9x5NPPklFRQXz5s3jlltuYenSpcTHx7N27VoADh06xK5du6reO3fuXG644QaysrJYtmwZR48eZfDgwXzwwQcsX74cgFWrVlU7buTIkXzwwQdh2+4WlUABvvjiOKowcWIfbrjhK7Rube07TfVyc3Orht9//32uu+46ALp06cLkyZNZuXIlzz//PPfccw+tWrUC4Oyzzz6pksvn83Hbbbdx/vnnc8kllwBw3nnnVU3PzMysev7BD37AjTfeyLRp08jLywPge9/7Hh9//DHbtm1j6tSpnH/++fziF784Kc6cnBx8Ph/x8fGUl5dzwQUXMHHiRK655hoqKyvZtWsXkyZNYvr06Tz99NP85z//ITMzk3PPPZennnoKgF/+8pecf/75jB07lk8++aTe+2rLli2cfvrpJCQkMGXKFFauXHnS9B07dvCVr3yF+Ph4unbtyrZt2xgyZAjHjx8HXAVVu3btAHjkkUf4zne+86X3AgwfPpxVq1adVHuelJREr169qh0HMHDgQD799NN6b1MoWsQ5UH9HICLCFVcM5swzM6wXpVj2wQONs5zx1S/nmWeeYdGiRfh8PlasWAF8+ZxZz549OXDgAAcOHCAjI6PGVbzyyit06dKFJ598ss5S0FVXXcU555zDE088wauvvsrs2bNZs2YNDz/8MLNmzeKvf/0rvXr14rrrrmPfvn307NkTgG3bttG7d28AEhISeO2110hJSeH+++/n3XffZeDAgRw+fJjFixcTHx/PRRddxKuvvkrbtm258MILueGGG7jnnnu477772LZtGz/96U959tlnq+Las2cPN954I5WVlcTHxwPQo0ePk+bJzc0lLS0NcM2djh07dtK2DR48mKVLl5KZmcnKlSs5duwYZ511Fj/5yU947LHHGDNmDH369CEnJ4cjR44wcODAL713yJAhLFu2jGHDhgHw6quv8qMf/YhBgwbRsWPHGsf179+fzz//nNGjR9e6/xsi5kqgWs/51607xFNPfUp5uTusSkpKsORpauVPXGeddRZ79uwB3J9woH379pGRkUFGRgZffPFFjcvasmUL48ePByAu7uSfW/AyzzzzTACuvvpqXn75ZVavXl01bvPmzcyePZvMzEw2bdrE/v37q11OYWEht956K+effz7PP/98VWwjR46sSn5r167l8ssvZ9KkSRw8eJAjR47wzDPPMHHiRG677bYvbU/v3r3JysrijTfeICsri6ysrJOSJ0C7du2qznvm5+dXlSb97rvvPubOncuMGTMYMmQIXbt25Xe/+x333nsvmzZtIj09nWXLlvHoo4/y7W9/+6T33n777XzwwQdccskldO/ena5d3W1zLr/8ctavX0+PHj147bXXahznL0CFQ7MtgVZW+nj77e189JH7on322WFGj665pGBiSA0lx8YUHx/PvffeywMPPMBzzz3HOeecw3PPPcd1113H4cOHWbJkCffffz8dO3bk3nvvrToUX7VqFcOHD686jB88eDArV65k2rRp+Hw+4uLiUFVKS0vZvHnzSev0J9jOnTtTXFzM008/zfXXX1+1nN///vdkZGRQWVl5UkIYOHAgu3fvBuDtt99m0KBBzJs3jx//+MdVyTUweY8aNYrnn3+e1NRUysvLSUxM5E9/+hOffPIJ27dv5/bbbz8prlBKoIMGDWL9+vVUVlayePFixo0bd9Iyunbtyssvv0xxcTGzZ8+mX79+qCodOrjCTMeOHcnLy2Pnzp3cd999FBcXs3XrVhYsWMDMmTP55z//SWVlJddffz3jxo2jtLSUpKQkANLS0khJSal2HMDOnTu59tprQ//w66FZJtD8/FIWLtzA3r2uidIllwxk1Khu0Q7LxJjBgwdz5MgRDh48yAMPPMBdd93F448/jojw6KOPkpSUxPDhw7nuuuu44IILqKiooH///lXnFcGViBYtWsTEiRNp06YNb7zxBl//+tc577zzuPjii2tc97Rp03jooYf4wx/+AMCcOXO45ZZbKC0tJTExkRdeeIE2bdoA0KFDB+Li4qisrGTs2LHMmTOHVatWkZ6eftKhsN+DDz7I5Zdfjs/no0OHDrzwwgucffbZTJw4kYkTJ35pfn8JtLaG9ImJidx+++1MmDCB9u3bM2/ePAC+853v8Mc//pHXX3+d3/72t8THx/PQQw8hItx1113cdNNN/OxnP6Njx4786Ec/qqoI27VrF/fffz8zZ85k9erVfP/730dE+MEPfkDr1q155ZVXePjhhwH3BzJ16lQWLVr0pXHgSu9nnHFG7R92A0nwYURTN2JAV12//VCN03ftymXhwg0UFpaTlpbEzJnD6dkzLYIRnpqsrKyq0kxzc6rbtmnTJoYOHdp4ATWyaF6ps3DhQhISErjqqqvCto5YvBJp7dq1vPXWW/zwhz/80rTg75OIrFbVs+qz/GZVAj1w4Dj/+MdafD6lX792TJ8+jNTUVtEOy5iwmzFjRrRDaJJGjhzJyJEjw7b8ZpVAu3Vrw/DhnUlPT2by5H7ExdlVK8aY8In5BHrkSCHx8XF06JCCiHD11UPtcr9mLJw1qqblaKxTlzHXjCnQ+vWH+ctf1jB//vqqZkr242q+EhMTG6UDCNOy+XtjaoyOaWKyBFpZ6eOdd3awcuU+ALp0SSXG6sJMA3Tq1OmkS/yampKSkmbdW1Rz2j5/f6CnKmIJVEQeAc4C1qjqPQHjRwCP4+4W901VXVfbcnw++Pvf17JnTx5xccLFF5/GmDHdreTZArRr1+5LDbSbkqysLEaNGhXtMMKmuW9fQ0TkEF5ERgOpqjoBaCUiYwIm/xy4DpjpDdcqJ1/YsyePtm1bcfPNZ3D22T0seRpjoiJSJdBzgMXe8GJgHPCx97qDqu4FEJH0uhbkU6FvX9dEqU0ba6JkjImeSCXQdsB2bzgPGB4wLa6G4Soicgdwh/ey9OabR62/+eZGj7Gp6AQcjXYQYdKctw1s+2Ld4Pq+IVIJNBfwXw6U5r3289UwXEVV5wJzAURkVX2vFoglzXn7mvO2gW1frBORVfV9T6SaMX0IXOANTwECOwvMEZGeItIdVzo1xpiYEJEEqqprgBIRWY4rZe4RkR97k38K/AtY6A0bY0xMiFgzpsCmS5453vh1wHlffkeN5jZaUE1Tc96+5rxtYNsX6+q9fTHXG5MxxjQVMX0ppzHGRJMlUGOMaaAmnUBF5BERWS4ijwaNHyEiK0TkfRH5SrTiO1W1bN8T3rataI7b501LEZGDIjIlGrGdqlo+uw4iskBE3g2oKI05tWzfDBH5SET+IyJXRCu+UyEi3UVkjYiUiEhC0LR65ZYmm0Ab8/LPpqiO7fuVqp4L3EyMtkyoY/vAXRixPvKRnbo6tu2nwP+o6mRVnROdCE9NHdv3X0Cm9/he5KNrFDm4ZpUrq5lWr9zSZBMo1V/+6ddBVfeq6n6gzss/m6gat09Vd3qD5UBlhONqLDVun4i0AsYCK6IQV2Oo7bs5AviRiLwnIudEPLLGUdv2bQZSgTZAfoTjahSqWqKqx2qYXK/c0pQTaDtOfEB5QPuAaXVe/hkDats+v18Cf4hYRI2rtu27GXgm4hE1ntq2bTzuc7sW+E2E42ostW3fi8Aa4FPgjxGOKxLqlVuacvI5pcs/Y0Bt24eIfBfYqKqxWkqrdvu8c04Xqeqb0QqsEdT22W1R1U2qeojm+d38Oa4vi6HA/0Q4rkioV25pygm0uV/+WeP2ichUXEnmF1GIq7HUtH1dgV4i8hbwNeCXIlJd6bspq+27uUVEMkQklRjtsJzat68UKAIKgebYHVq9ckuTTaDN/fLPOrbvj0A/4D0ReSJaMZ6KmrZPVfer6hhVvRj4J3BfLeejmqQQvpvPAe8So3+AdWzfn4H3gQ+I0SuTRCRRRBYDI4G3ReT8huYWuxLJGGMaqMmWQI0xpqmzBGqMMQ1kCdQYYxrIEqgxxjSQJVBjjGkgS6AmJCLyC6/tZswQkf4iUiAiXU5lHmNqYgm0BRGRLBEp9RKG//FkE4hrRUBceSKyWkSuOtXlquoOVW2jqoe99dwmIp/XNk9j89bpC9jfe0Tk915/AKEuI+b+vFoKS6Atz8+9hOF/3BbtgDw/VdU2uFvnPg8sEJEBUY6psWzx72/gMuB64AdRjsk0AkugBgARGSUiy0QkW0RyROR1EelXy/z/JSK7ROS4iOwTkZ8HTOsrIi96/X1+ISJ/9i5trJOqluOudknA9WyEiPQTkUUictQrwT0sIsneNBGRX3nrOS4iO0Xkm96000RERaSbiEwAHgMGBZQGJwTN01lEykRkRNC2vi8iP/KGE0XkfhHZIiK54vrMHB3qflbVz3BX8lTdHri2fS8iN+CS7ZSAuHt70873YjsmItu8/hNMBFkCNX4K/ATIAPrjrnn+R3UzishQXKcSl6hqW+B04DVvWmvgPWAt7nLUEd7zI6EEISJJwLeAMmCdiCQCbwB7gd7Aubi+KB/y3nIJrkR3thfLOKrp51FVlwPfJqA06I0LnOeItx1fD4hnIK7rPf++mIMrRV4EdMRdjvqWiNTZ9ZmX7EcBE3DdwlWtmhr2vao+C/waWBwQ9x4R8e/zX+FK7V8F/ktErq8rDtOIVNUeLeQBZAHFuN51/I9xNcx7Bu466GTv9S+At7zhgbgOJaYDbYLedy0uSQWOG+utV2pY14qAuA7jSmiXedMmetNSAua/DCjwhqd475kKJAUt9zRccurmvb4N+LyOeb4KHAQSvNf/C7zpDcd52z0+aBmbgGtr2LbbcH265gIl3roW4josrulzqnHfB8zzODA3aNwPg+ezR3gfVgJteeaoaruAx0pwJS0ReUlE9otIPrAMEFwp6ySquhW4CfgG8IV3GOu/NUc/oJ93eJsrIrnAv71l1VbT/aAXTxdVPVdVX/fG9wIOKXPcKQAAAkRJREFUqmpxwLzbgVQR6aCqi3Hdqv0UOCIib9XnkLoa/m72LhWROOBG4G/euK5ACvBm0Pb1AXrWssytqtoO1xHxLbgOi9v5J9Zn3wfoB8wOiuN+XCnWRIglUOM3FzgGnK6qabiSH7gf8peo6kJVnYI7fHwJeMU7L7kb149pYJJOV9VkdX1k1tdeoJuIpASM6w8UqmqOF8vj6m6BkgFsxFVCVafO/h1VtQLX2fPXgQtxCfNVb/JhXGk4M2j7Wqvqb0NYdqWqPgUsBX4fMKmufV9d3LtxJdDAONqq6si64jCNxxKo8UsDCoA8EekMPFjTjCIyVESmeuc7y3H9Jiruh/4q0EZEfigibbzzfj1F5MoGxvUhLln8WtyN6HoCP8MrFYrIWBE51zt3WgIcBypqWNZBXDJuU8c6nwKmAd8HnlXVUnAJENfV4O/8LQS8bbxYRLrVY5seAK6UE/caqmvfHwT6eOeD/f4PuEFELvMqthJEZLiITMREjCVQ4/ddYDLuVg5ZwKJa5k3C/cgP4s7tfRO4WlXLVLUAmITra3EzLrm+g6toqjd1tfKX4Q5Z9+EqiFbgzveBSz6PAUe9xyRcpVJ1FuMquHZ7h73n1bDOjcAnuPOrfwua/GPcYf5r3uH2FtwN8kL+LXmnQJ7F3foD6t7383H7+pAXd29VXQtciUvyB4AjXqydQo3DnDrrD9QYYxrISqDGGNNAlkCNMaaBLIEaY0wDWQI1xpgGsgRqjDENZAnUGGMayBKoMcY0kCVQY4xpoP8PzK+yo0LLD9EAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 360x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "_=plt.figure(figsize=(5, 4))\n",
    "lw = 2\n",
    "_=plt.plot(fpr, tpr, color='darkorange',\n",
    "         lw=lw, label='ROC curve (area = %0.6f)' % roc_auc, alpha=0.5)\n",
    "_=plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--', alpha=0.5)\n",
    "_=plt.xlim([0.0, 1.0])\n",
    "_=plt.ylim([0.0, 1.0])\n",
    "_=plt.xticks(fontsize=8)\n",
    "_=plt.yticks(fontsize=8)\n",
    "_=plt.grid(True)\n",
    "_=plt.xlabel('False Positive Rate', fontsize=13)\n",
    "_=plt.ylabel('True Positive Rate', fontsize=13)\n",
    "_=plt.title('Receiver operating characteristic', fontsize=15)\n",
    "_=plt.legend(loc=\"lower right\", fontsize=8)\n",
    "_=plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Graphing Metrics (F1, LogLoss, AUC) and comparing between models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "MetricVisualizer(['it_200_lr_0.7','it_300_lr_0.3','it_200_lr_0.5','it_500_lr_0.1']).start()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img style=\"float: left;\" src=\"Catboost-F1-plot.png\" width=70%> "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Best scores on different parameters\n",
    "1)  \n",
    "'it_200_lr_0.7'  \n",
    "Model params:  \n",
    "{'iterations': 200, 'learning_rate': 0.7}  \n",
    "Best score is:  \n",
    "{'validation': {'Logloss': 0.16177110710959713, 'F1': 0.9328283669052772, 'AUC': 0.9836359552616086}}  \n",
    "\n",
    "2)  \n",
    "'it_300_lr_0.3'   \n",
    "Model params:  \n",
    "{'iterations': 300, 'learning_rate': 0.3}  \n",
    "Best score is:  \n",
    "{'validation': {'Logloss': 0.15979496946276414, 'F1': 0.9335764902042518, 'AUC': 0.9840637573668797}}  \n",
    "\n",
    "3)  \n",
    "'it_200_lr_0.5_'  \n",
    "Model params:  \n",
    "{'iterations': 200, 'learning_rate': 0.5}  \n",
    "Best score is:  \n",
    "{'validation': {'Logloss': 0.16158135498462647, 'F1': 0.933329170959115, 'AUC': 0.983678392330355}}  \n",
    "Best iteration is: 196  \n",
    "Tree counts of Catboost with best parameters: 197  \n",
    "\n",
    "4)  \n",
    "'it_500_lr_0.1'\n",
    "Model params:  \n",
    "{'iterations': 500, 'learning_rate': 0.1,early_stopping_rounds=20}  \n",
    "Best score is:  \n",
    "{'validation': {'Logloss': 0.15473743586425584, 'F1': 0.9363383335591948, 'AUC': 0.9849992111102392}}  \n",
    "Best iteration is: 284  \n",
    "Tree counts of Catboost with best parameters: 285  \n",
    "Time consuming: 35.82min  \n",
    "\n",
    "5)  \n",
    "**==> BEST  \n",
    "Model params:     \n",
    "{'iterations': 500, 'learning_rate': 0.1, 'cv': 3}  \n",
    "Best validation F1 score of cv model: 0.936554±0.0002  \n",
    "Best validation Logloss of cv model: 0.576101±0.0006  \n",
    "Best validation AUC of cv model: 0.985125±0.0001  \n",
    "Best iteration: 499  \n",
    "Time consuming: 203.09min**  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "____"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 7.Compare best performance between models\n",
    "\n",
    "Best F1 scores on validation set:\n",
    "\n",
    "lgb = 0.936973 > CatBoost = 0.936554 > rf = 0.933681"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}