repeated_buyers.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import unicodedata\n",
    "import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd  \n",
    "train = pd.read_csv(\"data_format2/train_format2.csv\", sep=',')  \n",
    "\n",
    "# train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(7030723, 6)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>age_range</th>\n",
       "      <th>gender</th>\n",
       "      <th>merchant_id</th>\n",
       "      <th>label</th>\n",
       "      <th>activity_log</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>34176</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>944</td>\n",
       "      <td>-1</td>\n",
       "      <td>408895:1505:7370:1107:0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>34176</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>412</td>\n",
       "      <td>-1</td>\n",
       "      <td>17235:1604:4396:0818:0#954723:1604:4396:0818:0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>34176</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1945</td>\n",
       "      <td>-1</td>\n",
       "      <td>231901:662:2758:0818:0#231901:662:2758:0818:0#...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>34176</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4752</td>\n",
       "      <td>-1</td>\n",
       "      <td>174142:821:6938:1027:0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>34176</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>643</td>\n",
       "      <td>-1</td>\n",
       "      <td>716371:1505:968:1024:3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  age_range  gender  merchant_id  label  \\\n",
       "0    34176        6.0     0.0          944     -1   \n",
       "1    34176        6.0     0.0          412     -1   \n",
       "2    34176        6.0     0.0         1945     -1   \n",
       "3    34176        6.0     0.0         4752     -1   \n",
       "4    34176        6.0     0.0          643     -1   \n",
       "\n",
       "                                        activity_log  \n",
       "0                            408895:1505:7370:1107:0  \n",
       "1  17235:1604:4396:0818:0#954723:1604:4396:0818:0...  \n",
       "2  231901:662:2758:0818:0#231901:662:2758:0818:0#...  \n",
       "3                             174142:821:6938:1027:0  \n",
       "4                             716371:1505:968:1024:3  "
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(train.shape)\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# user_log = pd.read_csv(\"data_format1/user_log_format1.csv\", sep=',')\n",
    "# user_log.rename(columns={'seller_id':'merchant_id'}, inplace = True)\n",
    "# user_log.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "# user_log.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import pandas as pd  \n",
    "# user_info = pd.read_csv(\"data_format1/user_info_format1.csv\", sep=',')\n",
    "\n",
    "# user_info.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_format1 = pd.read_csv(\"data_format1/train_format1.csv\", sep=',')\n",
    "\n",
    "# train_format1.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>age_range</th>\n",
       "      <th>gender</th>\n",
       "      <th>merchant_id</th>\n",
       "      <th>label</th>\n",
       "      <th>activity_log</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>163968</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4378</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>101206:812:6968:0614:0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>163968</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2300</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>588758:844:3833:0618:0#71782:844:3833:1111:2#7...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>163968</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1551</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>312747:243:1954:0627:0#312747:243:1954:0627:0#...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>163968</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4343</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>932390:1612:3201:0628:0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>163968</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4911</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>957657:662:3089:0612:0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  age_range  gender  merchant_id  label  \\\n",
       "0   163968        0.0     0.0         4378   -1.0   \n",
       "1   163968        0.0     0.0         2300   -1.0   \n",
       "2   163968        0.0     0.0         1551   -1.0   \n",
       "3   163968        0.0     0.0         4343   -1.0   \n",
       "4   163968        0.0     0.0         4911   -1.0   \n",
       "\n",
       "                                        activity_log  \n",
       "0                             101206:812:6968:0614:0  \n",
       "1  588758:844:3833:0618:0#71782:844:3833:1111:2#7...  \n",
       "2  312747:243:1954:0627:0#312747:243:1954:0627:0#...  \n",
       "3                            932390:1612:3201:0628:0  \n",
       "4                             957657:662:3089:0612:0  "
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test = pd.read_csv(\"data_format2/test_format2.csv\", sep=',')  \n",
    "\n",
    "test.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(7027943, 6)\n"
     ]
    }
   ],
   "source": [
    "print(test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(261477, 6)"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_for_output = test[test.label != -1.0]\n",
    "test_for_output.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# This cell is for various test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>age_range</th>\n",
       "      <th>gender</th>\n",
       "      <th>merchant_id</th>\n",
       "      <th>label</th>\n",
       "      <th>activity_log</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5872630</th>\n",
       "      <td>2</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2194</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>192767:276:5103:1020:0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5872631</th>\n",
       "      <td>2</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3552</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>293244:1401:2276:1010:2#917794:1401:2276:1010:2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5872632</th>\n",
       "      <td>2</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1816</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>215563:276:2044:1020:0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5872633</th>\n",
       "      <td>2</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4924</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>77292:1142:856:1111:2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5872634</th>\n",
       "      <td>2</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1179</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>916474:500:8120:1111:2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5872635</th>\n",
       "      <td>2</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1974</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>431502:737:8321:0817:0#864941:737:8321:0817:0#...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5872636</th>\n",
       "      <td>2</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1544</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>669860:737:654:1002:0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5872637</th>\n",
       "      <td>2</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1784</td>\n",
       "      <td>NaN</td>\n",
       "      <td>274076:420:7936:1111:2#274076:420:7936:1111:2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5872638</th>\n",
       "      <td>2</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1679</td>\n",
       "      <td>NaN</td>\n",
       "      <td>286376:1130:7931:1111:2#655497:1130:7931:1111:...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5872639</th>\n",
       "      <td>2</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3716</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>899591:898:2946:1111:3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5872640</th>\n",
       "      <td>2</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2412</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>281455:389:4801:1108:3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5872641</th>\n",
       "      <td>2</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2223</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>348983:177:3273:0527:0#749563:177:3273:0527:0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5872642</th>\n",
       "      <td>2</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2076</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>1068315:703:1283:0721:2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5872643</th>\n",
       "      <td>2</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>420</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>239288:602:4953:0626:0#751744:602:4953:0626:0#...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         user_id  age_range  gender  merchant_id  label  \\\n",
       "5872630        2        3.0     0.0         2194   -1.0   \n",
       "5872631        2        3.0     0.0         3552   -1.0   \n",
       "5872632        2        3.0     0.0         1816   -1.0   \n",
       "5872633        2        3.0     0.0         4924   -1.0   \n",
       "5872634        2        3.0     0.0         1179   -1.0   \n",
       "5872635        2        3.0     0.0         1974   -1.0   \n",
       "5872636        2        3.0     0.0         1544   -1.0   \n",
       "5872637        2        3.0     0.0         1784    NaN   \n",
       "5872638        2        3.0     0.0         1679    NaN   \n",
       "5872639        2        3.0     0.0         3716   -1.0   \n",
       "5872640        2        3.0     0.0         2412   -1.0   \n",
       "5872641        2        3.0     0.0         2223   -1.0   \n",
       "5872642        2        3.0     0.0         2076   -1.0   \n",
       "5872643        2        3.0     0.0          420   -1.0   \n",
       "\n",
       "                                              activity_log  \n",
       "5872630                             192767:276:5103:1020:0  \n",
       "5872631    293244:1401:2276:1010:2#917794:1401:2276:1010:2  \n",
       "5872632                             215563:276:2044:1020:0  \n",
       "5872633                              77292:1142:856:1111:2  \n",
       "5872634                             916474:500:8120:1111:2  \n",
       "5872635  431502:737:8321:0817:0#864941:737:8321:0817:0#...  \n",
       "5872636                              669860:737:654:1002:0  \n",
       "5872637      274076:420:7936:1111:2#274076:420:7936:1111:2  \n",
       "5872638  286376:1130:7931:1111:2#655497:1130:7931:1111:...  \n",
       "5872639                             899591:898:2946:1111:3  \n",
       "5872640                             281455:389:4801:1108:3  \n",
       "5872641      348983:177:3273:0527:0#749563:177:3273:0527:0  \n",
       "5872642                            1068315:703:1283:0721:2  \n",
       "5872643  239288:602:4953:0626:0#751744:602:4953:0626:0#...  "
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# test.loc[test['user_id'] == 328862]\n",
    "test.loc[test['user_id'] == 2]\n",
    "\n",
    "# train['merchant_id'].value_counts()\n",
    "# user_log.item_id.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(261477, 3)"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_sub = pd.read_csv(\"sample_submission.csv\", sep=',')  \n",
    "\n",
    "sample_sub.shape\n",
    "# no need to hand in all 7027943 rows, only rows with label = NaN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Merge train and test\n",
    "### then we can clean the data in these two data sets together\n",
    "\n",
    "### We don't care about label = -1 in train and test\n",
    "### We should predict the probability only for those rows with label = NaN in test "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(14058666, 6)"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all = pd.concat([train,test])\n",
    "df_all.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### An important thing I notice is that:\n",
    "in train, no NaN in cloumn 'label'\n",
    "so we can split 'df_all' according to this observation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "user_id             0\n",
       "age_range       19380\n",
       "gender          61712\n",
       "merchant_id         0\n",
       "label               0\n",
       "activity_log     2975\n",
       "dtype: int64"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "user_id              0\n",
       "age_range        19420\n",
       "gender           63250\n",
       "merchant_id          0\n",
       "label           261477\n",
       "activity_log      3006\n",
       "dtype: int64"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "user_id              0\n",
       "age_range        38800\n",
       "gender          124962\n",
       "merchant_id          0\n",
       "label           261477\n",
       "activity_log      5981\n",
       "dtype: int64"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all.isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### for column age_range 0 means unknown\n",
    "### for column gender 2 means unknown."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "user_id              0\n",
       "age_range            0\n",
       "gender               0\n",
       "merchant_id          0\n",
       "label           261477\n",
       "activity_log      5981\n",
       "dtype: int64"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all['age_range'] = df_all['age_range'].fillna(0)\n",
    "df_all['gender'] = df_all['gender'].fillna(2)\n",
    "\n",
    "df_all.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-1.0    13536325\n",
       " 0.0      244912\n",
       " 1.0       15952\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all['label'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Drop rows with label = -1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 522341 entries, 6 to 7027926\n",
      "Data columns (total 6 columns):\n",
      "user_id         522341 non-null int64\n",
      "age_range       522341 non-null float64\n",
      "gender          522341 non-null float64\n",
      "merchant_id     522341 non-null int64\n",
      "label           260864 non-null float64\n",
      "activity_log    522224 non-null object\n",
      "dtypes: float64(3), int64(2), object(1)\n",
      "memory usage: 27.9+ MB\n"
     ]
    }
   ],
   "source": [
    "# -1 represents that it is not a new customer of the given merchant\n",
    "# thus out of our prediction.\n",
    "df_all = df_all[df_all.label != -1]\n",
    "df_all.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Whether label imbalanced? \n",
    "# We only focus on label = 0 and 1\n",
    "# Maybe SMOTE needed?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(15952, 6)\n"
     ]
    }
   ],
   "source": [
    "# df_all.loc[((df_all.label==1)==True).index]\n",
    "new_test = (df_all.loc[df_all['label'] ==1])\n",
    "print(new_test.shape)\n",
    "# new_test['user_id'].value_counts()\n",
    "# df_all.loc[df_all['user_id'] ==34176]\n",
    "# af.groupby(['name','course'])['score'].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# print('1 denotes  it is a repeat buyer for that merchant_id , while 0 is the opposite. -1 represents that it is not a new customer of the given merchant, thus out of our prediction. ')\n",
    "\n",
    "# df_all.groupby('label').count()[['age_range']].plot.bar()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### The next thing is to handle Activity_log\n",
    "### item_id : category_id : brand_id : time_stamp : action_type\n",
    "### '#' is used to separate two neighbouring elements"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_all['activity_log'].isnull().value_counts()\n",
    "# user_log['time_stamp'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [],
   "source": [
    "# there are some NaN in activity log\n",
    "# which can be troublesome when I split the cell\n",
    "# so I decide to change NaN to ''\n",
    "df_all['activity_log'] = df_all['activity_log'].fillna('-2:-2:-2:-2:-2')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# df_all.head(8830)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# df_all.loc[156, :].activity_log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "user_id                 373146\n",
       "age_range                    3\n",
       "gender                       0\n",
       "merchant_id               3376\n",
       "label                        0\n",
       "activity_log    -2:-2:-2:-2:-2\n",
       "Name: 234278, dtype: object"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_tmp = df_all.loc[234278,:]\n",
    "df_tmp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "# counter_a = 0\n",
    "# counter_b = 0\n",
    "# log_test  = df_all\n",
    "# tmp_arr = []\n",
    "# for j in list(log_test['activity_log']):\n",
    "#     counter_a += 1\n",
    "#     for i in j.split('#'):\n",
    "#         counter_b += 1\n",
    "#         print(counter_a, counter_b)\n",
    "#         tmp_arr.append(i.split(':')[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [],
   "source": [
    "# extend activity_log into several columns\n",
    "# make a copy\n",
    "log_test  = df_all\n",
    "# item_id:category_id:brand_id:time_stamp:action_type\n",
    "\n",
    "log_test['item_list'] = log_test['activity_log'].apply(lambda x: [i.split(':')[0] for i in x.split('#')])\n",
    "log_test['cat_list'] = log_test['activity_log'].apply(lambda x: [i.split(':')[1] for i in x.split('#')])\n",
    "log_test['brand_list'] = log_test['activity_log'].apply(lambda x: [i.split(':')[2] for i in x.split('#')])\n",
    "log_test['time_list'] = log_test['activity_log'].apply(lambda x: [i.split(':')[3] for i in x.split('#')])\n",
    "#log_test['act_list'] = log_test['activity_log'].apply(lambda x: [i.split(':')[4] for i in x.split('#')])\n",
    "log_test['act_times'] = log_test['item_list'].apply(len)\n",
    "log_test['click'] = log_test['activity_log'].apply(lambda x: [i.split(':').count('0') for i in x.split('#')].count(1))\n",
    "log_test['add_cart'] = log_test['activity_log'].apply(lambda x: [i.split(':').count('1') for i in x.split('#')].count(1))\n",
    "log_test['purchase'] = log_test['activity_log'].apply(lambda x: [i.split(':').count('2') for i in x.split('#')].count(1))\n",
    "log_test['add_fav'] = log_test['activity_log'].apply(lambda x: [i.split(':').count('3') for i in x.split('#')].count(1))\n",
    "log_test.drop(['activity_log'],axis=1,inplace=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 522341 entries, 6 to 7027926\n",
      "Data columns (total 14 columns):\n",
      "user_id        522341 non-null int64\n",
      "age_range      522341 non-null float64\n",
      "gender         522341 non-null float64\n",
      "merchant_id    522341 non-null int64\n",
      "label          260864 non-null float64\n",
      "item_list      522341 non-null object\n",
      "cat_list       522341 non-null object\n",
      "brand_list     522341 non-null object\n",
      "time_list      522341 non-null object\n",
      "act_times      522341 non-null int64\n",
      "click          522341 non-null int64\n",
      "add_cart       522341 non-null int64\n",
      "purchase       522341 non-null int64\n",
      "add_fav        522341 non-null int64\n",
      "dtypes: float64(3), int64(7), object(4)\n",
      "memory usage: 79.8+ MB\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>age_range</th>\n",
       "      <th>gender</th>\n",
       "      <th>merchant_id</th>\n",
       "      <th>label</th>\n",
       "      <th>item_list</th>\n",
       "      <th>cat_list</th>\n",
       "      <th>brand_list</th>\n",
       "      <th>time_list</th>\n",
       "      <th>act_times</th>\n",
       "      <th>click</th>\n",
       "      <th>add_cart</th>\n",
       "      <th>purchase</th>\n",
       "      <th>add_fav</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>34176</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3906</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[757713, 757713, 757713, 718096, 757713, 61369...</td>\n",
       "      <td>[821, 821, 821, 1142, 821, 821, 821, 821, 1397...</td>\n",
       "      <td>[6268, 6268, 6268, 6268, 6268, 6268, 6268, 626...</td>\n",
       "      <td>[1110, 1110, 1110, 1031, 1031, 1021, 1108, 102...</td>\n",
       "      <td>39</td>\n",
       "      <td>36</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>34176</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>121</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[800752, 800752, 800752, 800752, 800752, 80075...</td>\n",
       "      <td>[1028, 1028, 1028, 1028, 1028, 1028, 1028, 102...</td>\n",
       "      <td>[2337, 2337, 2337, 2337, 2337, 2337, 2337, 233...</td>\n",
       "      <td>[1110, 1110, 1110, 1110, 1108, 1108, 1108, 110...</td>\n",
       "      <td>14</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>34176</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4356</td>\n",
       "      <td>1.0</td>\n",
       "      <td>[709909, 709909, 709909, 709909, 709909, 70990...</td>\n",
       "      <td>[1208, 1208, 1208, 1208, 1208, 1208, 1208, 120...</td>\n",
       "      <td>[2276, 2276, 2276, 2276, 2276, 2276, 2276, 227...</td>\n",
       "      <td>[1110, 1110, 1110, 1110, 1110, 1110, 1111, 111...</td>\n",
       "      <td>18</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>34176</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2217</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[399879, 399879]</td>\n",
       "      <td>[656, 656]</td>\n",
       "      <td>[3012, 3012]</td>\n",
       "      <td>[1111, 1111]</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117</th>\n",
       "      <td>230784</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4818</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[155695, 155695, 155695, 155695, 155695, 15569...</td>\n",
       "      <td>[2, 2, 2, 2, 2, 2, 2, 2]</td>\n",
       "      <td>[7606, 7606, 7606, 7606, 7606, 7606, 7606, 7606]</td>\n",
       "      <td>[1111, 1111, 1110, 1110, 1108, 1108, 1108, 1108]</td>\n",
       "      <td>8</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     user_id  age_range  gender  merchant_id  label  \\\n",
       "6      34176        6.0     0.0         3906    0.0   \n",
       "30     34176        6.0     0.0          121    0.0   \n",
       "40     34176        6.0     0.0         4356    1.0   \n",
       "63     34176        6.0     0.0         2217    0.0   \n",
       "117   230784        0.0     0.0         4818    0.0   \n",
       "\n",
       "                                             item_list  \\\n",
       "6    [757713, 757713, 757713, 718096, 757713, 61369...   \n",
       "30   [800752, 800752, 800752, 800752, 800752, 80075...   \n",
       "40   [709909, 709909, 709909, 709909, 709909, 70990...   \n",
       "63                                    [399879, 399879]   \n",
       "117  [155695, 155695, 155695, 155695, 155695, 15569...   \n",
       "\n",
       "                                              cat_list  \\\n",
       "6    [821, 821, 821, 1142, 821, 821, 821, 821, 1397...   \n",
       "30   [1028, 1028, 1028, 1028, 1028, 1028, 1028, 102...   \n",
       "40   [1208, 1208, 1208, 1208, 1208, 1208, 1208, 120...   \n",
       "63                                          [656, 656]   \n",
       "117                           [2, 2, 2, 2, 2, 2, 2, 2]   \n",
       "\n",
       "                                            brand_list  \\\n",
       "6    [6268, 6268, 6268, 6268, 6268, 6268, 6268, 626...   \n",
       "30   [2337, 2337, 2337, 2337, 2337, 2337, 2337, 233...   \n",
       "40   [2276, 2276, 2276, 2276, 2276, 2276, 2276, 227...   \n",
       "63                                        [3012, 3012]   \n",
       "117   [7606, 7606, 7606, 7606, 7606, 7606, 7606, 7606]   \n",
       "\n",
       "                                             time_list  act_times  click  \\\n",
       "6    [1110, 1110, 1110, 1031, 1031, 1021, 1108, 102...         39     36   \n",
       "30   [1110, 1110, 1110, 1110, 1108, 1108, 1108, 110...         14     13   \n",
       "40   [1110, 1110, 1110, 1110, 1110, 1110, 1111, 111...         18     12   \n",
       "63                                        [1111, 1111]          2      1   \n",
       "117   [1111, 1111, 1110, 1110, 1108, 1108, 1108, 1108]          8      7   \n",
       "\n",
       "     add_cart  purchase  add_fav  \n",
       "6           0         1        2  \n",
       "30          0         1        0  \n",
       "40          0         6        0  \n",
       "63          0         1        0  \n",
       "117         0         7        0  "
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log_test.info()\n",
    "log_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [],
   "source": [
    "def div_ten(c):  \n",
    "    return int(int(c)/100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [],
   "source": [
    "log_test['time_5'] = [ list(map(div_ten,it)).count(5) for it in log_test['time_list'].tolist()]\n",
    "log_test['time_6'] = [ list(map(div_ten,it)).count(6) for it in log_test['time_list'].tolist()]\n",
    "log_test['time_7'] = [ list(map(div_ten,it)).count(7) for it in log_test['time_list'].tolist()]\n",
    "log_test['time_8'] = [ list(map(div_ten,it)).count(8) for it in log_test['time_list'].tolist()]\n",
    "log_test['time_9'] = [ list(map(div_ten,it)).count(9) for it in log_test['time_list'].tolist()]\n",
    "log_test['time_10'] = [ list(map(div_ten,it)).count(10) for it in log_test['time_list'].tolist()]\n",
    "log_test['time_11'] = [ list(map(div_ten,it)).count(11) for it in log_test['time_list'].tolist()]\n",
    "\n",
    "idx = log_test.columns.tolist().index('time_5')\n",
    "\n",
    "\n",
    "log_test['time_cov'] = np.sqrt(log_test.iloc[:, idx:idx+7].var(axis=1))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [],
   "source": [
    "# log_test.drop(['time_list'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>age_range</th>\n",
       "      <th>gender</th>\n",
       "      <th>merchant_id</th>\n",
       "      <th>label</th>\n",
       "      <th>item_list</th>\n",
       "      <th>cat_list</th>\n",
       "      <th>brand_list</th>\n",
       "      <th>time_list</th>\n",
       "      <th>act_times</th>\n",
       "      <th>...</th>\n",
       "      <th>purchase</th>\n",
       "      <th>add_fav</th>\n",
       "      <th>time_5</th>\n",
       "      <th>time_6</th>\n",
       "      <th>time_7</th>\n",
       "      <th>time_8</th>\n",
       "      <th>time_9</th>\n",
       "      <th>time_10</th>\n",
       "      <th>time_11</th>\n",
       "      <th>time_cov</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>34176</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3906</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[757713, 757713, 757713, 718096, 757713, 61369...</td>\n",
       "      <td>[821, 821, 821, 1142, 821, 821, 821, 821, 1397...</td>\n",
       "      <td>[6268, 6268, 6268, 6268, 6268, 6268, 6268, 626...</td>\n",
       "      <td>[1110, 1110, 1110, 1031, 1031, 1021, 1108, 102...</td>\n",
       "      <td>39</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>29</td>\n",
       "      <td>10</td>\n",
       "      <td>10.982670</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>34176</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>121</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[800752, 800752, 800752, 800752, 800752, 80075...</td>\n",
       "      <td>[1028, 1028, 1028, 1028, 1028, 1028, 1028, 102...</td>\n",
       "      <td>[2337, 2337, 2337, 2337, 2337, 2337, 2337, 233...</td>\n",
       "      <td>[1110, 1110, 1110, 1110, 1108, 1108, 1108, 110...</td>\n",
       "      <td>14</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>14</td>\n",
       "      <td>5.291503</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>34176</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4356</td>\n",
       "      <td>1.0</td>\n",
       "      <td>[709909, 709909, 709909, 709909, 709909, 70990...</td>\n",
       "      <td>[1208, 1208, 1208, 1208, 1208, 1208, 1208, 120...</td>\n",
       "      <td>[2276, 2276, 2276, 2276, 2276, 2276, 2276, 227...</td>\n",
       "      <td>[1110, 1110, 1110, 1110, 1110, 1110, 1111, 111...</td>\n",
       "      <td>18</td>\n",
       "      <td>...</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>18</td>\n",
       "      <td>6.803361</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>34176</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2217</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[399879, 399879]</td>\n",
       "      <td>[656, 656]</td>\n",
       "      <td>[3012, 3012]</td>\n",
       "      <td>[1111, 1111]</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0.755929</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117</th>\n",
       "      <td>230784</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4818</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[155695, 155695, 155695, 155695, 155695, 15569...</td>\n",
       "      <td>[2, 2, 2, 2, 2, 2, 2, 2]</td>\n",
       "      <td>[7606, 7606, 7606, 7606, 7606, 7606, 7606, 7606]</td>\n",
       "      <td>[1111, 1111, 1110, 1110, 1108, 1108, 1108, 1108]</td>\n",
       "      <td>8</td>\n",
       "      <td>...</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>3.023716</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     user_id  age_range  gender  merchant_id  label  \\\n",
       "6      34176        6.0     0.0         3906    0.0   \n",
       "30     34176        6.0     0.0          121    0.0   \n",
       "40     34176        6.0     0.0         4356    1.0   \n",
       "63     34176        6.0     0.0         2217    0.0   \n",
       "117   230784        0.0     0.0         4818    0.0   \n",
       "\n",
       "                                             item_list  \\\n",
       "6    [757713, 757713, 757713, 718096, 757713, 61369...   \n",
       "30   [800752, 800752, 800752, 800752, 800752, 80075...   \n",
       "40   [709909, 709909, 709909, 709909, 709909, 70990...   \n",
       "63                                    [399879, 399879]   \n",
       "117  [155695, 155695, 155695, 155695, 155695, 15569...   \n",
       "\n",
       "                                              cat_list  \\\n",
       "6    [821, 821, 821, 1142, 821, 821, 821, 821, 1397...   \n",
       "30   [1028, 1028, 1028, 1028, 1028, 1028, 1028, 102...   \n",
       "40   [1208, 1208, 1208, 1208, 1208, 1208, 1208, 120...   \n",
       "63                                          [656, 656]   \n",
       "117                           [2, 2, 2, 2, 2, 2, 2, 2]   \n",
       "\n",
       "                                            brand_list  \\\n",
       "6    [6268, 6268, 6268, 6268, 6268, 6268, 6268, 626...   \n",
       "30   [2337, 2337, 2337, 2337, 2337, 2337, 2337, 233...   \n",
       "40   [2276, 2276, 2276, 2276, 2276, 2276, 2276, 227...   \n",
       "63                                        [3012, 3012]   \n",
       "117   [7606, 7606, 7606, 7606, 7606, 7606, 7606, 7606]   \n",
       "\n",
       "                                             time_list  act_times    ...      \\\n",
       "6    [1110, 1110, 1110, 1031, 1031, 1021, 1108, 102...         39    ...       \n",
       "30   [1110, 1110, 1110, 1110, 1108, 1108, 1108, 110...         14    ...       \n",
       "40   [1110, 1110, 1110, 1110, 1110, 1110, 1111, 111...         18    ...       \n",
       "63                                        [1111, 1111]          2    ...       \n",
       "117   [1111, 1111, 1110, 1110, 1108, 1108, 1108, 1108]          8    ...       \n",
       "\n",
       "     purchase  add_fav  time_5  time_6  time_7  time_8  time_9  time_10  \\\n",
       "6           1        2       0       0       0       0       0       29   \n",
       "30          1        0       0       0       0       0       0        0   \n",
       "40          6        0       0       0       0       0       0        0   \n",
       "63          1        0       0       0       0       0       0        0   \n",
       "117         7        0       0       0       0       0       0        0   \n",
       "\n",
       "     time_11   time_cov  \n",
       "6         10  10.982670  \n",
       "30        14   5.291503  \n",
       "40        18   6.803361  \n",
       "63         2   0.755929  \n",
       "117        8   3.023716  \n",
       "\n",
       "[5 rows x 22 columns]"
      ]
     },
     "execution_count": 142,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False    522224\n",
       "True        117\n",
       "Name: time_cov, dtype: int64"
      ]
     },
     "execution_count": 143,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(log_test['time_cov']==0).value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Handle Unknown Values in age_range and gender\n",
    "## age_range=0 and gender=2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.0    138524\n",
       "0.0    114245\n",
       "4.0    102598\n",
       "2.0     62385\n",
       "5.0     51232\n",
       "6.0     43382\n",
       "7.0      8430\n",
       "8.0      1517\n",
       "1.0        28\n",
       "Name: age_range, dtype: int64"
      ]
     },
     "execution_count": 144,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log_test['age_range'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Replace age 8 with 7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "log_test.age_range.replace(8, 7, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "# make a copy \n",
    "for_test = log_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [],
   "source": [
    "# qq = for_test[for_test['age_range']==0].index\n",
    "# len(list(qq))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [],
   "source": [
    "# change age_range=0  -->  age_range=3 or 4 according to binomial distribution\n",
    "# for j in list(qq):\n",
    "#     for_test.loc[j,'age_range'] = 3 + np.random.binomial(1,0.5)\n",
    "#     print(j)\n",
    "\n",
    "# for_test.loc[for_test['age_range']==0,'age_range'] = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.0    138524\n",
       "0.0    114245\n",
       "4.0    102598\n",
       "2.0     62385\n",
       "5.0     51232\n",
       "6.0     43382\n",
       "7.0      9947\n",
       "1.0        28\n",
       "Name: age_range, dtype: int64"
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for_test['age_range'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0    352691\n",
       "1.0    148094\n",
       "2.0     21556\n",
       "Name: gender, dtype: int64"
      ]
     },
     "execution_count": 150,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for_test['gender'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "# qq = for_test[log_test['gender']==2].index\n",
    "# len(list(qq))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [],
   "source": [
    "# change gender=2  -->  gender=0 or 1 according to binomial distribution\n",
    "# for j in list(qq):\n",
    "#     for_test.loc[j,'gender'] = np.random.binomial(1,0.3)\n",
    "for_test.loc[for_test['gender']==2,'gender'] = 0\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0    374247\n",
       "1.0    148094\n",
       "Name: gender, dtype: int64"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for_test['gender'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "log_test = for_test\n",
    "# for_test.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for_test = log_test\n",
    "# for_test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dummy Encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [],
   "source": [
    "for_test = pd.get_dummies(log_test, columns=['age_range'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>gender</th>\n",
       "      <th>merchant_id</th>\n",
       "      <th>label</th>\n",
       "      <th>item_list</th>\n",
       "      <th>cat_list</th>\n",
       "      <th>brand_list</th>\n",
       "      <th>time_list</th>\n",
       "      <th>act_times</th>\n",
       "      <th>click</th>\n",
       "      <th>...</th>\n",
       "      <th>time_11</th>\n",
       "      <th>time_cov</th>\n",
       "      <th>age_range_0.0</th>\n",
       "      <th>age_range_1.0</th>\n",
       "      <th>age_range_2.0</th>\n",
       "      <th>age_range_3.0</th>\n",
       "      <th>age_range_4.0</th>\n",
       "      <th>age_range_5.0</th>\n",
       "      <th>age_range_6.0</th>\n",
       "      <th>age_range_7.0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>34176</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3906</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[757713, 757713, 757713, 718096, 757713, 61369...</td>\n",
       "      <td>[821, 821, 821, 1142, 821, 821, 821, 821, 1397...</td>\n",
       "      <td>[6268, 6268, 6268, 6268, 6268, 6268, 6268, 626...</td>\n",
       "      <td>[1110, 1110, 1110, 1031, 1031, 1021, 1108, 102...</td>\n",
       "      <td>39</td>\n",
       "      <td>36</td>\n",
       "      <td>...</td>\n",
       "      <td>10</td>\n",
       "      <td>10.982670</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>34176</td>\n",
       "      <td>0.0</td>\n",
       "      <td>121</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[800752, 800752, 800752, 800752, 800752, 80075...</td>\n",
       "      <td>[1028, 1028, 1028, 1028, 1028, 1028, 1028, 102...</td>\n",
       "      <td>[2337, 2337, 2337, 2337, 2337, 2337, 2337, 233...</td>\n",
       "      <td>[1110, 1110, 1110, 1110, 1108, 1108, 1108, 110...</td>\n",
       "      <td>14</td>\n",
       "      <td>13</td>\n",
       "      <td>...</td>\n",
       "      <td>14</td>\n",
       "      <td>5.291503</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>34176</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4356</td>\n",
       "      <td>1.0</td>\n",
       "      <td>[709909, 709909, 709909, 709909, 709909, 70990...</td>\n",
       "      <td>[1208, 1208, 1208, 1208, 1208, 1208, 1208, 120...</td>\n",
       "      <td>[2276, 2276, 2276, 2276, 2276, 2276, 2276, 227...</td>\n",
       "      <td>[1110, 1110, 1110, 1110, 1110, 1110, 1111, 111...</td>\n",
       "      <td>18</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>18</td>\n",
       "      <td>6.803361</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>34176</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2217</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[399879, 399879]</td>\n",
       "      <td>[656, 656]</td>\n",
       "      <td>[3012, 3012]</td>\n",
       "      <td>[1111, 1111]</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>0.755929</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117</th>\n",
       "      <td>230784</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4818</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[155695, 155695, 155695, 155695, 155695, 15569...</td>\n",
       "      <td>[2, 2, 2, 2, 2, 2, 2, 2]</td>\n",
       "      <td>[7606, 7606, 7606, 7606, 7606, 7606, 7606, 7606]</td>\n",
       "      <td>[1111, 1111, 1110, 1110, 1108, 1108, 1108, 1108]</td>\n",
       "      <td>8</td>\n",
       "      <td>7</td>\n",
       "      <td>...</td>\n",
       "      <td>8</td>\n",
       "      <td>3.023716</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 29 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     user_id  gender  merchant_id  label  \\\n",
       "6      34176     0.0         3906    0.0   \n",
       "30     34176     0.0          121    0.0   \n",
       "40     34176     0.0         4356    1.0   \n",
       "63     34176     0.0         2217    0.0   \n",
       "117   230784     0.0         4818    0.0   \n",
       "\n",
       "                                             item_list  \\\n",
       "6    [757713, 757713, 757713, 718096, 757713, 61369...   \n",
       "30   [800752, 800752, 800752, 800752, 800752, 80075...   \n",
       "40   [709909, 709909, 709909, 709909, 709909, 70990...   \n",
       "63                                    [399879, 399879]   \n",
       "117  [155695, 155695, 155695, 155695, 155695, 15569...   \n",
       "\n",
       "                                              cat_list  \\\n",
       "6    [821, 821, 821, 1142, 821, 821, 821, 821, 1397...   \n",
       "30   [1028, 1028, 1028, 1028, 1028, 1028, 1028, 102...   \n",
       "40   [1208, 1208, 1208, 1208, 1208, 1208, 1208, 120...   \n",
       "63                                          [656, 656]   \n",
       "117                           [2, 2, 2, 2, 2, 2, 2, 2]   \n",
       "\n",
       "                                            brand_list  \\\n",
       "6    [6268, 6268, 6268, 6268, 6268, 6268, 6268, 626...   \n",
       "30   [2337, 2337, 2337, 2337, 2337, 2337, 2337, 233...   \n",
       "40   [2276, 2276, 2276, 2276, 2276, 2276, 2276, 227...   \n",
       "63                                        [3012, 3012]   \n",
       "117   [7606, 7606, 7606, 7606, 7606, 7606, 7606, 7606]   \n",
       "\n",
       "                                             time_list  act_times  click  \\\n",
       "6    [1110, 1110, 1110, 1031, 1031, 1021, 1108, 102...         39     36   \n",
       "30   [1110, 1110, 1110, 1110, 1108, 1108, 1108, 110...         14     13   \n",
       "40   [1110, 1110, 1110, 1110, 1110, 1110, 1111, 111...         18     12   \n",
       "63                                        [1111, 1111]          2      1   \n",
       "117   [1111, 1111, 1110, 1110, 1108, 1108, 1108, 1108]          8      7   \n",
       "\n",
       "         ...        time_11   time_cov  age_range_0.0  age_range_1.0  \\\n",
       "6        ...             10  10.982670              0              0   \n",
       "30       ...             14   5.291503              0              0   \n",
       "40       ...             18   6.803361              0              0   \n",
       "63       ...              2   0.755929              0              0   \n",
       "117      ...              8   3.023716              1              0   \n",
       "\n",
       "     age_range_2.0  age_range_3.0  age_range_4.0  age_range_5.0  \\\n",
       "6                0              0              0              0   \n",
       "30               0              0              0              0   \n",
       "40               0              0              0              0   \n",
       "63               0              0              0              0   \n",
       "117              0              0              0              0   \n",
       "\n",
       "     age_range_6.0  age_range_7.0  \n",
       "6                1              0  \n",
       "30               1              0  \n",
       "40               1              0  \n",
       "63               1              0  \n",
       "117              0              0  \n",
       "\n",
       "[5 rows x 29 columns]"
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for_test.head()\n",
    "# print(for_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "for_test_backup = for_test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create new features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Split out train set, calculate some features for mid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(260864, 29)"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_after_split_for_add_new_feature = for_test[for_test.label.notnull()]\n",
    "train_after_split_for_add_new_feature.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Add features for mid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [],
   "source": [
    "mid_act_type_ori = ['label', 'click', 'add_cart', 'purchase', 'add_fav', 'gender', 'age_range_1.0', 'age_range_2.0', 'age_range_3.0', 'age_range_4.0', 'age_range_5.0', 'age_range_6.0', 'age_range_7.0', 'age_range_0.0']\n",
    "mid_act_type = ['mid_label_ratio', 'mid_mean_click_times', 'mid_mean_add_cart', 'mid_mean_purchase', 'mid_mean_add_fav', 'mid_mean_gender', 'mid_mean_age_range_1.0', 'mid_mean_age_range_2.0', 'mid_mean_age_range_3.0', 'mid_mean_age_range_4.0', 'mid_mean_age_range_5.0', 'mid_mean_age_range_6.0', 'mid_mean_age_range_7.0', 'mid_mean_age_range_0.0']\n",
    "\n",
    "# mid_act_type_ori = ['label', 'click', 'add_cart', 'purchase', 'add_fav', 'gender']\n",
    "# mid_act_type = ['mid_label_ratio', 'mid_mean_click_times', 'mid_mean_add_cart', 'mid_mean_purchase', 'mid_mean_add_fav', 'mid_mean_gender']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for_test_backup = for_test\n",
    "# for_test = for_test_backup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(522341, 29)\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 522341 entries, 6 to 7027926\n",
      "Data columns (total 29 columns):\n",
      "user_id          522341 non-null int64\n",
      "gender           522341 non-null float64\n",
      "merchant_id      522341 non-null int64\n",
      "label            260864 non-null float64\n",
      "item_list        522341 non-null object\n",
      "cat_list         522341 non-null object\n",
      "brand_list       522341 non-null object\n",
      "time_list        522341 non-null object\n",
      "act_times        522341 non-null int64\n",
      "click            522341 non-null int64\n",
      "add_cart         522341 non-null int64\n",
      "purchase         522341 non-null int64\n",
      "add_fav          522341 non-null int64\n",
      "time_5           522341 non-null int64\n",
      "time_6           522341 non-null int64\n",
      "time_7           522341 non-null int64\n",
      "time_8           522341 non-null int64\n",
      "time_9           522341 non-null int64\n",
      "time_10          522341 non-null int64\n",
      "time_11          522341 non-null int64\n",
      "time_cov         522341 non-null float64\n",
      "age_range_0.0    522341 non-null uint8\n",
      "age_range_1.0    522341 non-null uint8\n",
      "age_range_2.0    522341 non-null uint8\n",
      "age_range_3.0    522341 non-null uint8\n",
      "age_range_4.0    522341 non-null uint8\n",
      "age_range_5.0    522341 non-null uint8\n",
      "age_range_6.0    522341 non-null uint8\n",
      "age_range_7.0    522341 non-null uint8\n",
      "dtypes: float64(3), int64(14), object(4), uint8(8)\n",
      "memory usage: 111.7+ MB\n"
     ]
    }
   ],
   "source": [
    "print(for_test.shape)\n",
    "for_test.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(522341, 43)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# create_feature_test = train_after_split_for_add_new_feature\n",
    "\n",
    "# for i in range(len(mid_act_type_ori)):\n",
    "#     mid_mean_all_grouped = create_feature_test[mid_act_type_ori[i]].groupby(create_feature_test['merchant_id'])\n",
    "#     mid_mean_all = mid_mean_all_grouped.mean()\n",
    "\n",
    "#     mid_mean_all = mid_mean_all.to_frame(name = mid_act_type[i])\n",
    "#     mid_mean_all.reset_index(inplace = True)\n",
    "\n",
    "#     mid_mean_all.loc[mid_mean_all.shape[0]+1] = {'merchant_id':4807, mid_act_type[i]: mid_mean_all.mean()[1] }\n",
    "\n",
    "#     for_test = pd.merge(for_test, mid_mean_all, on='merchant_id')\n",
    "# for_test.shape\n",
    "\n",
    "\n",
    "create_feature_test = for_test\n",
    "\n",
    "for i in range(len(mid_act_type_ori)):\n",
    "# for i in range(1):\n",
    "    \n",
    "    mid_mean_all_grouped = create_feature_test[mid_act_type_ori[i]].groupby(create_feature_test['merchant_id'])\n",
    "    mid_mean_all = mid_mean_all_grouped.mean()\n",
    "\n",
    "    mid_mean_all = mid_mean_all.to_frame(name = mid_act_type[i])\n",
    "    \n",
    "#     print(mid_mean_all.loc[4807, :])\n",
    "    mid_mean_all.reset_index(inplace = True)\n",
    "    \n",
    "#     print(int(mid_mean_all.loc[mid_mean_all['merchant_id']==4807].index))\n",
    "    if i == 0:\n",
    "        mid_mean_all.loc[mid_mean_all.loc[mid_mean_all['merchant_id']==4807].index, 'mid_label_ratio'] = mid_mean_all.mean()[1]\n",
    "#     print(mid_mean_all.loc[mid_mean_all['merchant_id']==4807])\n",
    "\n",
    "    for_test = pd.merge(for_test, mid_mean_all, on='merchant_id')\n",
    "print(for_test.shape)\n",
    "# for_test.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>gender</th>\n",
       "      <th>merchant_id</th>\n",
       "      <th>label</th>\n",
       "      <th>item_list</th>\n",
       "      <th>cat_list</th>\n",
       "      <th>brand_list</th>\n",
       "      <th>time_list</th>\n",
       "      <th>act_times</th>\n",
       "      <th>click</th>\n",
       "      <th>...</th>\n",
       "      <th>mid_mean_add_fav</th>\n",
       "      <th>mid_mean_gender</th>\n",
       "      <th>mid_mean_age_range_1.0</th>\n",
       "      <th>mid_mean_age_range_2.0</th>\n",
       "      <th>mid_mean_age_range_3.0</th>\n",
       "      <th>mid_mean_age_range_4.0</th>\n",
       "      <th>mid_mean_age_range_5.0</th>\n",
       "      <th>mid_mean_age_range_6.0</th>\n",
       "      <th>mid_mean_age_range_7.0</th>\n",
       "      <th>mid_mean_age_range_0.0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>325008</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3906</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[523545, 523545, 523545, 523545, 523545]</td>\n",
       "      <td>[662, 662, 662, 662, 662]</td>\n",
       "      <td>[6268, 6268, 6268, 6268, 6268]</td>\n",
       "      <td>[1014, 1014, 1110, 1110, 1111]</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>0.65035</td>\n",
       "      <td>0.13986</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.048951</td>\n",
       "      <td>0.202797</td>\n",
       "      <td>0.13986</td>\n",
       "      <td>0.167832</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0.027972</td>\n",
       "      <td>0.230769</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74</th>\n",
       "      <td>282768</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3906</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[670847, 874935, 384944, 684569, 670847, 89304...</td>\n",
       "      <td>[1142, 1142, 1577, 1142, 1142, 154, 748, 1142,...</td>\n",
       "      <td>[6268, 6268, 6268, 6268, 6268, 6268, 6268, 626...</td>\n",
       "      <td>[1111, 1111, 1111, 1111, 1111, 1111, 1111, 111...</td>\n",
       "      <td>9</td>\n",
       "      <td>8</td>\n",
       "      <td>...</td>\n",
       "      <td>0.65035</td>\n",
       "      <td>0.13986</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.048951</td>\n",
       "      <td>0.202797</td>\n",
       "      <td>0.13986</td>\n",
       "      <td>0.167832</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0.027972</td>\n",
       "      <td>0.230769</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>187542</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3906</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[561246, 118930, 118930, 561246, 67356, 67356,...</td>\n",
       "      <td>[1095, 662, 662, 1095, 1095, 1095, 1095, 662, ...</td>\n",
       "      <td>[6268, 6268, 6268, 6268, 6268, 6268, 6268, 626...</td>\n",
       "      <td>[1111, 1111, 1111, 1111, 1111, 1111, 1111, 111...</td>\n",
       "      <td>27</td>\n",
       "      <td>25</td>\n",
       "      <td>...</td>\n",
       "      <td>0.65035</td>\n",
       "      <td>0.13986</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.048951</td>\n",
       "      <td>0.202797</td>\n",
       "      <td>0.13986</td>\n",
       "      <td>0.167832</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0.027972</td>\n",
       "      <td>0.230769</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76</th>\n",
       "      <td>97206</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3906</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[832131, 371457, 832131, 212935, 832131, 38494...</td>\n",
       "      <td>[662, 662, 662, 662, 662, 1577, 662, 1577, 662...</td>\n",
       "      <td>[6268, 6268, 6268, 6268, 6268, 6268, 6268, 626...</td>\n",
       "      <td>[0704, 0704, 0704, 0704, 0704, 1111, 0704, 111...</td>\n",
       "      <td>27</td>\n",
       "      <td>26</td>\n",
       "      <td>...</td>\n",
       "      <td>0.65035</td>\n",
       "      <td>0.13986</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.048951</td>\n",
       "      <td>0.202797</td>\n",
       "      <td>0.13986</td>\n",
       "      <td>0.167832</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0.027972</td>\n",
       "      <td>0.230769</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>153795</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3906</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[315990, 315990, 315990, 315990, 315990, 31599...</td>\n",
       "      <td>[1577, 1577, 1577, 1577, 1577, 1577, 1577, 157...</td>\n",
       "      <td>[6268, 6268, 6268, 6268, 6268, 6268, 6268, 626...</td>\n",
       "      <td>[1107, 1111, 1111, 1111, 1107, 1107, 1111, 110...</td>\n",
       "      <td>18</td>\n",
       "      <td>16</td>\n",
       "      <td>...</td>\n",
       "      <td>0.65035</td>\n",
       "      <td>0.13986</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.048951</td>\n",
       "      <td>0.202797</td>\n",
       "      <td>0.13986</td>\n",
       "      <td>0.167832</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0.027972</td>\n",
       "      <td>0.230769</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 43 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    user_id  gender  merchant_id  label  \\\n",
       "73   325008     0.0         3906    NaN   \n",
       "74   282768     0.0         3906    NaN   \n",
       "75   187542     0.0         3906    NaN   \n",
       "76    97206     0.0         3906    NaN   \n",
       "77   153795     0.0         3906    NaN   \n",
       "\n",
       "                                            item_list  \\\n",
       "73           [523545, 523545, 523545, 523545, 523545]   \n",
       "74  [670847, 874935, 384944, 684569, 670847, 89304...   \n",
       "75  [561246, 118930, 118930, 561246, 67356, 67356,...   \n",
       "76  [832131, 371457, 832131, 212935, 832131, 38494...   \n",
       "77  [315990, 315990, 315990, 315990, 315990, 31599...   \n",
       "\n",
       "                                             cat_list  \\\n",
       "73                          [662, 662, 662, 662, 662]   \n",
       "74  [1142, 1142, 1577, 1142, 1142, 154, 748, 1142,...   \n",
       "75  [1095, 662, 662, 1095, 1095, 1095, 1095, 662, ...   \n",
       "76  [662, 662, 662, 662, 662, 1577, 662, 1577, 662...   \n",
       "77  [1577, 1577, 1577, 1577, 1577, 1577, 1577, 157...   \n",
       "\n",
       "                                           brand_list  \\\n",
       "73                     [6268, 6268, 6268, 6268, 6268]   \n",
       "74  [6268, 6268, 6268, 6268, 6268, 6268, 6268, 626...   \n",
       "75  [6268, 6268, 6268, 6268, 6268, 6268, 6268, 626...   \n",
       "76  [6268, 6268, 6268, 6268, 6268, 6268, 6268, 626...   \n",
       "77  [6268, 6268, 6268, 6268, 6268, 6268, 6268, 626...   \n",
       "\n",
       "                                            time_list  act_times  click  \\\n",
       "73                     [1014, 1014, 1110, 1110, 1111]          5      2   \n",
       "74  [1111, 1111, 1111, 1111, 1111, 1111, 1111, 111...          9      8   \n",
       "75  [1111, 1111, 1111, 1111, 1111, 1111, 1111, 111...         27     25   \n",
       "76  [0704, 0704, 0704, 0704, 0704, 1111, 0704, 111...         27     26   \n",
       "77  [1107, 1111, 1111, 1111, 1107, 1107, 1111, 110...         18     16   \n",
       "\n",
       "             ...            mid_mean_add_fav  mid_mean_gender  \\\n",
       "73           ...                     0.65035          0.13986   \n",
       "74           ...                     0.65035          0.13986   \n",
       "75           ...                     0.65035          0.13986   \n",
       "76           ...                     0.65035          0.13986   \n",
       "77           ...                     0.65035          0.13986   \n",
       "\n",
       "    mid_mean_age_range_1.0  mid_mean_age_range_2.0  mid_mean_age_range_3.0  \\\n",
       "73                     0.0                0.048951                0.202797   \n",
       "74                     0.0                0.048951                0.202797   \n",
       "75                     0.0                0.048951                0.202797   \n",
       "76                     0.0                0.048951                0.202797   \n",
       "77                     0.0                0.048951                0.202797   \n",
       "\n",
       "    mid_mean_age_range_4.0  mid_mean_age_range_5.0  mid_mean_age_range_6.0  \\\n",
       "73                 0.13986                0.167832                0.181818   \n",
       "74                 0.13986                0.167832                0.181818   \n",
       "75                 0.13986                0.167832                0.181818   \n",
       "76                 0.13986                0.167832                0.181818   \n",
       "77                 0.13986                0.167832                0.181818   \n",
       "\n",
       "    mid_mean_age_range_7.0  mid_mean_age_range_0.0  \n",
       "73                0.027972                0.230769  \n",
       "74                0.027972                0.230769  \n",
       "75                0.027972                0.230769  \n",
       "76                0.027972                0.230769  \n",
       "77                0.027972                0.230769  \n",
       "\n",
       "[5 rows x 43 columns]"
      ]
     },
     "execution_count": 164,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for_test[for_test.label.isnull()].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [],
   "source": [
    "mid_month_ori = ['time_5', 'time_6', 'time_7', 'time_8', 'time_9', 'time_10', 'time_11']\n",
    "mid_month = ['mid_time_5', 'mid_time_6', 'mid_time_7', 'mid_time_8', 'mid_time_9', 'mid_time_10', 'mid_time_11']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(522341, 50)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "create_feature_test = for_test\n",
    "\n",
    "for i in range(len(mid_month_ori)):\n",
    "# for i in range(1):\n",
    "    \n",
    "    mid_mean_all_grouped = create_feature_test[mid_month_ori[i]].groupby(create_feature_test['merchant_id'])\n",
    "    mid_mean_all = mid_mean_all_grouped.mean()\n",
    "\n",
    "    mid_mean_all = mid_mean_all.to_frame(name = mid_month[i])\n",
    "    \n",
    "    mid_mean_all.reset_index(inplace = True)\n",
    "    \n",
    "#     print(int(mid_mean_all.loc[mid_mean_all['merchant_id']==4807].index))\n",
    "#     if i == 0:\n",
    "#         mid_mean_all.loc[mid_mean_all.loc[mid_mean_all['merchant_id']==4807].index, 'mid_label_ratio'] = mid_mean_all.mean()[1]\n",
    "#     print(mid_mean_all.loc[mid_mean_all['merchant_id']==4807])\n",
    "\n",
    "    for_test = pd.merge(for_test, mid_mean_all, on = 'merchant_id')\n",
    "print(for_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Add mid item/cat/brand features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [],
   "source": [
    "# mid_item_cat_brand_ori = ['item_list', 'cat_list', 'brand_list']\n",
    "# mid_all_count_item_cat_brand = ['mid_item_count', 'mid_cat_count', 'mid_brand_count']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for i in range(len(mid_all_count_item_cat_brand)):\n",
    "\n",
    "\n",
    "#     create_feature_test = for_test.copy()\n",
    "    \n",
    "#     mid_mean_all_grouped = create_feature_test[mid_item_cat_brand_ori[i]].groupby(create_feature_test['merchant_id'])\n",
    "\n",
    "#     mid_mean_all = mid_mean_all_grouped.sum()\n",
    "\n",
    "#     mid_mean_all = mid_mean_all.to_frame(name = 'tmp_number')\n",
    "#     mid_mean_all.reset_index(inplace = True)\n",
    "    \n",
    "#     mid_mean_all[mid_all_count_item_cat_brand[i]] = mid_mean_all['tmp_number'].apply(lambda x: len(set(x)))\n",
    "\n",
    "#     mid_mean_all.drop('tmp_number', inplace = True, axis = 1)\n",
    "\n",
    "#     for_test = pd.merge(for_test, mid_mean_all, on = 'merchant_id')\n",
    "\n",
    "# #     for_test[uid_mid_count_item_cat_brand_time[i]] = for_test[uid_item_cat_brand_time_ori[i]].apply(lambda x: len(set(x)))\n",
    "    \n",
    "# #     for_test[uid_mid_ratio_item_cat_brand_time[i]] = for_test[uid_mid_count_item_cat_brand_time[i]] / for_test[uid_all_count_item_cat_brand_time[i]]\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for_test.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Add features for uid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [],
   "source": [
    "uid_act_type_ori = ['click', 'add_cart', 'purchase', 'add_fav']\n",
    "uid_act_type = ['uid_mean_click_times', 'uid_mean_add_cart', 'uid_mean_purchase', 'uid_mean_add_fav']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(522341, 54)"
      ]
     },
     "execution_count": 172,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "create_feature_test = for_test\n",
    "\n",
    "for i in range(len(uid_act_type_ori)):\n",
    "    uid_mean_all_grouped = create_feature_test[uid_act_type_ori[i]].groupby(create_feature_test['user_id'])\n",
    "    uid_mean_all = uid_mean_all_grouped.mean()\n",
    "\n",
    "    uid_mean_all = uid_mean_all.to_frame(name = uid_act_type[i])\n",
    "    uid_mean_all.reset_index(inplace = True)\n",
    "    # uid_mean_all.head()\n",
    "    for_test = pd.merge(for_test, uid_mean_all, on='user_id')\n",
    "for_test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Add uid item/cat/brand/time features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [],
   "source": [
    "uid_item_cat_brand_time_ori = ['item_list', 'cat_list', 'brand_list', 'time_list']\n",
    "uid_all_count_item_cat_brand_time = ['uid_item_count', 'uid_cat_count', 'uid_brand_count', 'uid_time_count']\n",
    "uid_mid_count_item_cat_brand_time = ['uid_mid_item_count', 'uid_mid_cat_count', 'uid_mid_brand_count', 'uid_mid_time_count']\n",
    "uid_mid_ratio_item_cat_brand_time = ['uid_mid_item_ratio', 'uid_mid_cat_ratio', 'uid_mid_brand_ratio', 'uid_mid_time_ratio']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(uid_mid_ratio_item_cat_brand_time)):\n",
    "\n",
    "\n",
    "    create_feature_test = for_test.copy()\n",
    "    \n",
    "    uid_mean_all_grouped = create_feature_test[uid_item_cat_brand_time_ori[i]].groupby(create_feature_test['user_id'])\n",
    "\n",
    "    uid_mean_all = uid_mean_all_grouped.sum()\n",
    "\n",
    "    uid_mean_all = uid_mean_all.to_frame(name = 'tmp_number')\n",
    "    uid_mean_all.reset_index(inplace = True)\n",
    "    \n",
    "    uid_mean_all[uid_all_count_item_cat_brand_time[i]] = uid_mean_all['tmp_number'].apply(lambda x: len(set(x)))\n",
    "\n",
    "    uid_mean_all.drop('tmp_number', inplace = True, axis = 1)\n",
    "\n",
    "    for_test = pd.merge(for_test, uid_mean_all, on = 'user_id')\n",
    "\n",
    "    for_test[uid_mid_count_item_cat_brand_time[i]] = for_test[uid_item_cat_brand_time_ori[i]].apply(lambda x: len(set(x)))\n",
    "    \n",
    "    for_test[uid_mid_ratio_item_cat_brand_time[i]] = for_test[uid_mid_count_item_cat_brand_time[i]] / for_test[uid_all_count_item_cat_brand_time[i]]\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>gender</th>\n",
       "      <th>merchant_id</th>\n",
       "      <th>label</th>\n",
       "      <th>item_list</th>\n",
       "      <th>cat_list</th>\n",
       "      <th>brand_list</th>\n",
       "      <th>time_list</th>\n",
       "      <th>act_times</th>\n",
       "      <th>click</th>\n",
       "      <th>...</th>\n",
       "      <th>uid_mid_item_ratio</th>\n",
       "      <th>uid_cat_count</th>\n",
       "      <th>uid_mid_cat_count</th>\n",
       "      <th>uid_mid_cat_ratio</th>\n",
       "      <th>uid_brand_count</th>\n",
       "      <th>uid_mid_brand_count</th>\n",
       "      <th>uid_mid_brand_ratio</th>\n",
       "      <th>uid_time_count</th>\n",
       "      <th>uid_mid_time_count</th>\n",
       "      <th>uid_mid_time_ratio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>34176</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3906</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[757713, 757713, 757713, 718096, 757713, 61369...</td>\n",
       "      <td>[821, 821, 821, 1142, 821, 821, 821, 821, 1397...</td>\n",
       "      <td>[6268, 6268, 6268, 6268, 6268, 6268, 6268, 626...</td>\n",
       "      <td>[1110, 1110, 1110, 1031, 1031, 1021, 1108, 102...</td>\n",
       "      <td>39</td>\n",
       "      <td>36</td>\n",
       "      <td>...</td>\n",
       "      <td>0.833333</td>\n",
       "      <td>9</td>\n",
       "      <td>6</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>0.25</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>34176</td>\n",
       "      <td>0.0</td>\n",
       "      <td>121</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[800752, 800752, 800752, 800752, 800752, 80075...</td>\n",
       "      <td>[1028, 1028, 1028, 1028, 1028, 1028, 1028, 102...</td>\n",
       "      <td>[2337, 2337, 2337, 2337, 2337, 2337, 2337, 233...</td>\n",
       "      <td>[1110, 1110, 1110, 1110, 1108, 1108, 1108, 110...</td>\n",
       "      <td>14</td>\n",
       "      <td>13</td>\n",
       "      <td>...</td>\n",
       "      <td>0.041667</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>0.111111</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>0.25</td>\n",
       "      <td>9</td>\n",
       "      <td>3</td>\n",
       "      <td>0.333333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>34176</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4356</td>\n",
       "      <td>1.0</td>\n",
       "      <td>[709909, 709909, 709909, 709909, 709909, 70990...</td>\n",
       "      <td>[1208, 1208, 1208, 1208, 1208, 1208, 1208, 120...</td>\n",
       "      <td>[2276, 2276, 2276, 2276, 2276, 2276, 2276, 227...</td>\n",
       "      <td>[1110, 1110, 1110, 1110, 1110, 1110, 1111, 111...</td>\n",
       "      <td>18</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>0.111111</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>0.25</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>0.222222</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>34176</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2217</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[399879, 399879]</td>\n",
       "      <td>[656, 656]</td>\n",
       "      <td>[3012, 3012]</td>\n",
       "      <td>[1111, 1111]</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0.041667</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>0.111111</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>0.25</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>0.111111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>81036</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3906</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[684569, 684569, 488509, 684569, 684569, 67084...</td>\n",
       "      <td>[1142, 1142, 1577, 1142, 1142, 1142, 1142]</td>\n",
       "      <td>[6268, 6268, 6268, 6268, 6268, 6268, 6268]</td>\n",
       "      <td>[1111, 1111, 1111, 1111, 1111, 1104, 1104]</td>\n",
       "      <td>7</td>\n",
       "      <td>5</td>\n",
       "      <td>...</td>\n",
       "      <td>0.375000</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>0.400000</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0.50</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>0.666667</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 66 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  gender  merchant_id  label  \\\n",
       "0    34176     0.0         3906    0.0   \n",
       "1    34176     0.0          121    0.0   \n",
       "2    34176     0.0         4356    1.0   \n",
       "3    34176     0.0         2217    0.0   \n",
       "4    81036     0.0         3906    0.0   \n",
       "\n",
       "                                           item_list  \\\n",
       "0  [757713, 757713, 757713, 718096, 757713, 61369...   \n",
       "1  [800752, 800752, 800752, 800752, 800752, 80075...   \n",
       "2  [709909, 709909, 709909, 709909, 709909, 70990...   \n",
       "3                                   [399879, 399879]   \n",
       "4  [684569, 684569, 488509, 684569, 684569, 67084...   \n",
       "\n",
       "                                            cat_list  \\\n",
       "0  [821, 821, 821, 1142, 821, 821, 821, 821, 1397...   \n",
       "1  [1028, 1028, 1028, 1028, 1028, 1028, 1028, 102...   \n",
       "2  [1208, 1208, 1208, 1208, 1208, 1208, 1208, 120...   \n",
       "3                                         [656, 656]   \n",
       "4         [1142, 1142, 1577, 1142, 1142, 1142, 1142]   \n",
       "\n",
       "                                          brand_list  \\\n",
       "0  [6268, 6268, 6268, 6268, 6268, 6268, 6268, 626...   \n",
       "1  [2337, 2337, 2337, 2337, 2337, 2337, 2337, 233...   \n",
       "2  [2276, 2276, 2276, 2276, 2276, 2276, 2276, 227...   \n",
       "3                                       [3012, 3012]   \n",
       "4         [6268, 6268, 6268, 6268, 6268, 6268, 6268]   \n",
       "\n",
       "                                           time_list  act_times  click  \\\n",
       "0  [1110, 1110, 1110, 1031, 1031, 1021, 1108, 102...         39     36   \n",
       "1  [1110, 1110, 1110, 1110, 1108, 1108, 1108, 110...         14     13   \n",
       "2  [1110, 1110, 1110, 1110, 1110, 1110, 1111, 111...         18     12   \n",
       "3                                       [1111, 1111]          2      1   \n",
       "4         [1111, 1111, 1111, 1111, 1111, 1104, 1104]          7      5   \n",
       "\n",
       "          ...          uid_mid_item_ratio  uid_cat_count  uid_mid_cat_count  \\\n",
       "0         ...                    0.833333              9                  6   \n",
       "1         ...                    0.041667              9                  1   \n",
       "2         ...                    0.083333              9                  1   \n",
       "3         ...                    0.041667              9                  1   \n",
       "4         ...                    0.375000              5                  2   \n",
       "\n",
       "   uid_mid_cat_ratio  uid_brand_count  uid_mid_brand_count  \\\n",
       "0           0.666667                4                    1   \n",
       "1           0.111111                4                    1   \n",
       "2           0.111111                4                    1   \n",
       "3           0.111111                4                    1   \n",
       "4           0.400000                2                    1   \n",
       "\n",
       "   uid_mid_brand_ratio  uid_time_count  uid_mid_time_count  uid_mid_time_ratio  \n",
       "0                 0.25               9                   9            1.000000  \n",
       "1                 0.25               9                   3            0.333333  \n",
       "2                 0.25               9                   2            0.222222  \n",
       "3                 0.25               9                   1            0.111111  \n",
       "4                 0.50               3                   2            0.666667  \n",
       "\n",
       "[5 rows x 66 columns]"
      ]
     },
     "execution_count": 175,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Drop columns with list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [],
   "source": [
    "for_test.drop('item_list', axis=1, inplace=True)\n",
    "for_test.drop('brand_list', axis=1, inplace=True)\n",
    "for_test.drop('cat_list', axis=1, inplace=True)\n",
    "for_test.drop('time_list', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Overview"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 522341 entries, 0 to 522340\n",
      "Data columns (total 62 columns):\n",
      "user_id                   522341 non-null int64\n",
      "gender                    522341 non-null float64\n",
      "merchant_id               522341 non-null int64\n",
      "label                     260864 non-null float64\n",
      "act_times                 522341 non-null int64\n",
      "click                     522341 non-null int64\n",
      "add_cart                  522341 non-null int64\n",
      "purchase                  522341 non-null int64\n",
      "add_fav                   522341 non-null int64\n",
      "time_5                    522341 non-null int64\n",
      "time_6                    522341 non-null int64\n",
      "time_7                    522341 non-null int64\n",
      "time_8                    522341 non-null int64\n",
      "time_9                    522341 non-null int64\n",
      "time_10                   522341 non-null int64\n",
      "time_11                   522341 non-null int64\n",
      "time_cov                  522341 non-null float64\n",
      "age_range_0.0             522341 non-null uint8\n",
      "age_range_1.0             522341 non-null uint8\n",
      "age_range_2.0             522341 non-null uint8\n",
      "age_range_3.0             522341 non-null uint8\n",
      "age_range_4.0             522341 non-null uint8\n",
      "age_range_5.0             522341 non-null uint8\n",
      "age_range_6.0             522341 non-null uint8\n",
      "age_range_7.0             522341 non-null uint8\n",
      "mid_label_ratio           522341 non-null float64\n",
      "mid_mean_click_times      522341 non-null float64\n",
      "mid_mean_add_cart         522341 non-null float64\n",
      "mid_mean_purchase         522341 non-null float64\n",
      "mid_mean_add_fav          522341 non-null float64\n",
      "mid_mean_gender           522341 non-null float64\n",
      "mid_mean_age_range_1.0    522341 non-null float64\n",
      "mid_mean_age_range_2.0    522341 non-null float64\n",
      "mid_mean_age_range_3.0    522341 non-null float64\n",
      "mid_mean_age_range_4.0    522341 non-null float64\n",
      "mid_mean_age_range_5.0    522341 non-null float64\n",
      "mid_mean_age_range_6.0    522341 non-null float64\n",
      "mid_mean_age_range_7.0    522341 non-null float64\n",
      "mid_mean_age_range_0.0    522341 non-null float64\n",
      "mid_time_5                522341 non-null float64\n",
      "mid_time_6                522341 non-null float64\n",
      "mid_time_7                522341 non-null float64\n",
      "mid_time_8                522341 non-null float64\n",
      "mid_time_9                522341 non-null float64\n",
      "mid_time_10               522341 non-null float64\n",
      "mid_time_11               522341 non-null float64\n",
      "uid_mean_click_times      522341 non-null float64\n",
      "uid_mean_add_cart         522341 non-null float64\n",
      "uid_mean_purchase         522341 non-null float64\n",
      "uid_mean_add_fav          522341 non-null float64\n",
      "uid_item_count            522341 non-null int64\n",
      "uid_mid_item_count        522341 non-null int64\n",
      "uid_mid_item_ratio        522341 non-null float64\n",
      "uid_cat_count             522341 non-null int64\n",
      "uid_mid_cat_count         522341 non-null int64\n",
      "uid_mid_cat_ratio         522341 non-null float64\n",
      "uid_brand_count           522341 non-null int64\n",
      "uid_mid_brand_count       522341 non-null int64\n",
      "uid_mid_brand_ratio       522341 non-null float64\n",
      "uid_time_count            522341 non-null int64\n",
      "uid_mid_time_count        522341 non-null int64\n",
      "uid_mid_time_ratio        522341 non-null float64\n",
      "dtypes: float64(32), int64(22), uint8(8)\n",
      "memory usage: 223.2 MB\n"
     ]
    }
   ],
   "source": [
    "for_test.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>gender</th>\n",
       "      <th>merchant_id</th>\n",
       "      <th>label</th>\n",
       "      <th>act_times</th>\n",
       "      <th>click</th>\n",
       "      <th>add_cart</th>\n",
       "      <th>purchase</th>\n",
       "      <th>add_fav</th>\n",
       "      <th>time_5</th>\n",
       "      <th>...</th>\n",
       "      <th>uid_mid_item_ratio</th>\n",
       "      <th>uid_cat_count</th>\n",
       "      <th>uid_mid_cat_count</th>\n",
       "      <th>uid_mid_cat_ratio</th>\n",
       "      <th>uid_brand_count</th>\n",
       "      <th>uid_mid_brand_count</th>\n",
       "      <th>uid_mid_brand_ratio</th>\n",
       "      <th>uid_time_count</th>\n",
       "      <th>uid_mid_time_count</th>\n",
       "      <th>uid_mid_time_ratio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>111</th>\n",
       "      <td>325008</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3906</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0.500</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>112</th>\n",
       "      <td>325008</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4687</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0.500</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>0.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113</th>\n",
       "      <td>282768</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3906</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.583333</td>\n",
       "      <td>8</td>\n",
       "      <td>4</td>\n",
       "      <td>0.500</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>114</th>\n",
       "      <td>282768</td>\n",
       "      <td>0.0</td>\n",
       "      <td>643</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>0.125</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>115</th>\n",
       "      <td>282768</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4568</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>8</td>\n",
       "      <td>3</td>\n",
       "      <td>0.375</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 62 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     user_id  gender  merchant_id  label  act_times  click  add_cart  \\\n",
       "111   325008     0.0         3906    NaN          5      2         0   \n",
       "112   325008     0.0         4687    NaN          5      3         0   \n",
       "113   282768     0.0         3906    NaN          9      8         0   \n",
       "114   282768     0.0          643    NaN          3      2         0   \n",
       "115   282768     0.0         4568    NaN          5      4         0   \n",
       "\n",
       "     purchase  add_fav  time_5         ...          uid_mid_item_ratio  \\\n",
       "111         1        2       0         ...                    0.500000   \n",
       "112         1        1       0         ...                    0.500000   \n",
       "113         1        0       0         ...                    0.583333   \n",
       "114         1        0       0         ...                    0.083333   \n",
       "115         1        0       0         ...                    0.333333   \n",
       "\n",
       "     uid_cat_count  uid_mid_cat_count  uid_mid_cat_ratio  uid_brand_count  \\\n",
       "111              2                  1              0.500                2   \n",
       "112              2                  1              0.500                2   \n",
       "113              8                  4              0.500                3   \n",
       "114              8                  1              0.125                3   \n",
       "115              8                  3              0.375                3   \n",
       "\n",
       "     uid_mid_brand_count  uid_mid_brand_ratio  uid_time_count  \\\n",
       "111                    1             0.500000               3   \n",
       "112                    1             0.500000               3   \n",
       "113                    1             0.333333               1   \n",
       "114                    1             0.333333               1   \n",
       "115                    1             0.333333               1   \n",
       "\n",
       "     uid_mid_time_count  uid_mid_time_ratio  \n",
       "111                   3            1.000000  \n",
       "112                   2            0.666667  \n",
       "113                   1            1.000000  \n",
       "114                   1            1.000000  \n",
       "115                   1            1.000000  \n",
       "\n",
       "[5 rows x 62 columns]"
      ]
     },
     "execution_count": 178,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for_test[for_test.label.isnull()].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Reorder --> submission order"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [],
   "source": [
    "for_test_label_null = for_test[for_test.label.isnull()]\n",
    "submission_order = pd.merge(sample_sub, for_test_label_null, on=['user_id','merchant_id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(261477, 62)\n"
     ]
    }
   ],
   "source": [
    "submission_order.drop('prob', axis=1, inplace=True)\n",
    "submission_order.head()\n",
    "print(submission_order.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(260864, 62)"
      ]
     },
     "execution_count": 181,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for_test = for_test.drop(for_test[for_test.label.isnull()].index.tolist())\n",
    "for_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(522341, 62)"
      ]
     },
     "execution_count": 182,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for_test = pd.concat([for_test, submission_order], axis=0)\n",
    "for_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    4605\n",
       "1    1581\n",
       "2    1964\n",
       "3    3645\n",
       "4    3361\n",
       "Name: merchant_id, dtype: int64"
      ]
     },
     "execution_count": 183,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check\n",
    "for_test[for_test.label.isnull()].merchant_id.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Drop user_id and merchant_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(522341, 60)"
      ]
     },
     "execution_count": 184,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for_test.drop(['user_id', 'merchant_id'], axis=1, inplace=True)\n",
    "for_test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Move around columns to split x and y easily"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>act_times</th>\n",
       "      <th>add_cart</th>\n",
       "      <th>add_fav</th>\n",
       "      <th>age_range_0.0</th>\n",
       "      <th>age_range_1.0</th>\n",
       "      <th>age_range_2.0</th>\n",
       "      <th>age_range_3.0</th>\n",
       "      <th>age_range_4.0</th>\n",
       "      <th>age_range_5.0</th>\n",
       "      <th>...</th>\n",
       "      <th>uid_mean_purchase</th>\n",
       "      <th>uid_mid_brand_count</th>\n",
       "      <th>uid_mid_brand_ratio</th>\n",
       "      <th>uid_mid_cat_count</th>\n",
       "      <th>uid_mid_cat_ratio</th>\n",
       "      <th>uid_mid_item_count</th>\n",
       "      <th>uid_mid_item_ratio</th>\n",
       "      <th>uid_mid_time_count</th>\n",
       "      <th>uid_mid_time_ratio</th>\n",
       "      <th>uid_time_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>39</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>2.25</td>\n",
       "      <td>1</td>\n",
       "      <td>0.25</td>\n",
       "      <td>6</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>20</td>\n",
       "      <td>0.833333</td>\n",
       "      <td>9</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>14</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>2.25</td>\n",
       "      <td>1</td>\n",
       "      <td>0.25</td>\n",
       "      <td>1</td>\n",
       "      <td>0.111111</td>\n",
       "      <td>1</td>\n",
       "      <td>0.041667</td>\n",
       "      <td>3</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>2.25</td>\n",
       "      <td>1</td>\n",
       "      <td>0.25</td>\n",
       "      <td>1</td>\n",
       "      <td>0.111111</td>\n",
       "      <td>2</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>2</td>\n",
       "      <td>0.222222</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>2.25</td>\n",
       "      <td>1</td>\n",
       "      <td>0.25</td>\n",
       "      <td>1</td>\n",
       "      <td>0.111111</td>\n",
       "      <td>1</td>\n",
       "      <td>0.041667</td>\n",
       "      <td>1</td>\n",
       "      <td>0.111111</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.50</td>\n",
       "      <td>1</td>\n",
       "      <td>0.50</td>\n",
       "      <td>2</td>\n",
       "      <td>0.400000</td>\n",
       "      <td>3</td>\n",
       "      <td>0.375000</td>\n",
       "      <td>2</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 60 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   label  act_times  add_cart  add_fav  age_range_0.0  age_range_1.0  \\\n",
       "0    0.0         39         0        2              0              0   \n",
       "1    0.0         14         0        0              0              0   \n",
       "2    1.0         18         0        0              0              0   \n",
       "3    0.0          2         0        0              0              0   \n",
       "4    0.0          7         0        1              1              0   \n",
       "\n",
       "   age_range_2.0  age_range_3.0  age_range_4.0  age_range_5.0       ...        \\\n",
       "0              0              0              0              0       ...         \n",
       "1              0              0              0              0       ...         \n",
       "2              0              0              0              0       ...         \n",
       "3              0              0              0              0       ...         \n",
       "4              0              0              0              0       ...         \n",
       "\n",
       "   uid_mean_purchase  uid_mid_brand_count  uid_mid_brand_ratio  \\\n",
       "0               2.25                    1                 0.25   \n",
       "1               2.25                    1                 0.25   \n",
       "2               2.25                    1                 0.25   \n",
       "3               2.25                    1                 0.25   \n",
       "4               1.50                    1                 0.50   \n",
       "\n",
       "   uid_mid_cat_count  uid_mid_cat_ratio  uid_mid_item_count  \\\n",
       "0                  6           0.666667                  20   \n",
       "1                  1           0.111111                   1   \n",
       "2                  1           0.111111                   2   \n",
       "3                  1           0.111111                   1   \n",
       "4                  2           0.400000                   3   \n",
       "\n",
       "   uid_mid_item_ratio  uid_mid_time_count  uid_mid_time_ratio  uid_time_count  \n",
       "0            0.833333                   9            1.000000               9  \n",
       "1            0.041667                   3            0.333333               9  \n",
       "2            0.083333                   2            0.222222               9  \n",
       "3            0.041667                   1            0.111111               9  \n",
       "4            0.375000                   2            0.666667               3  \n",
       "\n",
       "[5 rows x 60 columns]"
      ]
     },
     "execution_count": 185,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label_move_around = for_test['label']\n",
    "for_test.drop('label', axis=1, inplace=True)\n",
    "for_test.insert(0,'label',label_move_around)\n",
    "for_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 522341 entries, 0 to 261476\n",
      "Data columns (total 60 columns):\n",
      "label                     260864 non-null float64\n",
      "act_times                 522341 non-null int64\n",
      "add_cart                  522341 non-null int64\n",
      "add_fav                   522341 non-null int64\n",
      "age_range_0.0             522341 non-null uint8\n",
      "age_range_1.0             522341 non-null uint8\n",
      "age_range_2.0             522341 non-null uint8\n",
      "age_range_3.0             522341 non-null uint8\n",
      "age_range_4.0             522341 non-null uint8\n",
      "age_range_5.0             522341 non-null uint8\n",
      "age_range_6.0             522341 non-null uint8\n",
      "age_range_7.0             522341 non-null uint8\n",
      "click                     522341 non-null int64\n",
      "gender                    522341 non-null float64\n",
      "mid_label_ratio           522341 non-null float64\n",
      "mid_mean_add_cart         522341 non-null float64\n",
      "mid_mean_add_fav          522341 non-null float64\n",
      "mid_mean_age_range_0.0    522341 non-null float64\n",
      "mid_mean_age_range_1.0    522341 non-null float64\n",
      "mid_mean_age_range_2.0    522341 non-null float64\n",
      "mid_mean_age_range_3.0    522341 non-null float64\n",
      "mid_mean_age_range_4.0    522341 non-null float64\n",
      "mid_mean_age_range_5.0    522341 non-null float64\n",
      "mid_mean_age_range_6.0    522341 non-null float64\n",
      "mid_mean_age_range_7.0    522341 non-null float64\n",
      "mid_mean_click_times      522341 non-null float64\n",
      "mid_mean_gender           522341 non-null float64\n",
      "mid_mean_purchase         522341 non-null float64\n",
      "mid_time_10               522341 non-null float64\n",
      "mid_time_11               522341 non-null float64\n",
      "mid_time_5                522341 non-null float64\n",
      "mid_time_6                522341 non-null float64\n",
      "mid_time_7                522341 non-null float64\n",
      "mid_time_8                522341 non-null float64\n",
      "mid_time_9                522341 non-null float64\n",
      "purchase                  522341 non-null int64\n",
      "time_10                   522341 non-null int64\n",
      "time_11                   522341 non-null int64\n",
      "time_5                    522341 non-null int64\n",
      "time_6                    522341 non-null int64\n",
      "time_7                    522341 non-null int64\n",
      "time_8                    522341 non-null int64\n",
      "time_9                    522341 non-null int64\n",
      "time_cov                  522341 non-null float64\n",
      "uid_brand_count           522341 non-null int64\n",
      "uid_cat_count             522341 non-null int64\n",
      "uid_item_count            522341 non-null int64\n",
      "uid_mean_add_cart         522341 non-null float64\n",
      "uid_mean_add_fav          522341 non-null float64\n",
      "uid_mean_click_times      522341 non-null float64\n",
      "uid_mean_purchase         522341 non-null float64\n",
      "uid_mid_brand_count       522341 non-null int64\n",
      "uid_mid_brand_ratio       522341 non-null float64\n",
      "uid_mid_cat_count         522341 non-null int64\n",
      "uid_mid_cat_ratio         522341 non-null float64\n",
      "uid_mid_item_count        522341 non-null int64\n",
      "uid_mid_item_ratio        522341 non-null float64\n",
      "uid_mid_time_count        522341 non-null int64\n",
      "uid_mid_time_ratio        522341 non-null float64\n",
      "uid_time_count            522341 non-null int64\n",
      "dtypes: float64(32), int64(20), uint8(8)\n",
      "memory usage: 215.2 MB\n"
     ]
    }
   ],
   "source": [
    "for_test.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Split out x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(522341, 59)"
      ]
     },
     "execution_count": 187,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_for_PCA = for_test.iloc[:, 1:]\n",
    "y_for_PCA = for_test.iloc[:, 0]\n",
    "x_for_PCA.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [],
   "source": [
    "# x_for_PCA.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Remove Highly Correlated Feature (to be tuned)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['click', 'mid_time_11', 'time_cov']\n",
      "3\n"
     ]
    }
   ],
   "source": [
    "corr_matrix = x_for_PCA.corr().abs()\n",
    "upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1).astype(np.bool))#upper tri matrix\n",
    "to_drop = [column for column in upper.columns if any(upper[column] > 0.95)]\n",
    "# to_drop = [column for column in upper.columns if any(upper[column] > 0.92)]\n",
    "\n",
    "# to_drop.remove('time_11')\n",
    "# to_drop.remove('time_cov')\n",
    "\n",
    "print(to_drop)\n",
    "print(len(to_drop))\n",
    "\n",
    "x_for_PCA.drop(to_drop, axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>act_times</th>\n",
       "      <th>add_cart</th>\n",
       "      <th>add_fav</th>\n",
       "      <th>age_range_0.0</th>\n",
       "      <th>age_range_1.0</th>\n",
       "      <th>age_range_2.0</th>\n",
       "      <th>age_range_3.0</th>\n",
       "      <th>age_range_4.0</th>\n",
       "      <th>age_range_5.0</th>\n",
       "      <th>age_range_6.0</th>\n",
       "      <th>...</th>\n",
       "      <th>uid_mean_purchase</th>\n",
       "      <th>uid_mid_brand_count</th>\n",
       "      <th>uid_mid_brand_ratio</th>\n",
       "      <th>uid_mid_cat_count</th>\n",
       "      <th>uid_mid_cat_ratio</th>\n",
       "      <th>uid_mid_item_count</th>\n",
       "      <th>uid_mid_item_ratio</th>\n",
       "      <th>uid_mid_time_count</th>\n",
       "      <th>uid_mid_time_ratio</th>\n",
       "      <th>uid_time_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>39</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>2.25</td>\n",
       "      <td>1</td>\n",
       "      <td>0.25</td>\n",
       "      <td>6</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>20</td>\n",
       "      <td>0.833333</td>\n",
       "      <td>9</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>14</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>2.25</td>\n",
       "      <td>1</td>\n",
       "      <td>0.25</td>\n",
       "      <td>1</td>\n",
       "      <td>0.111111</td>\n",
       "      <td>1</td>\n",
       "      <td>0.041667</td>\n",
       "      <td>3</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>2.25</td>\n",
       "      <td>1</td>\n",
       "      <td>0.25</td>\n",
       "      <td>1</td>\n",
       "      <td>0.111111</td>\n",
       "      <td>2</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>2</td>\n",
       "      <td>0.222222</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>2.25</td>\n",
       "      <td>1</td>\n",
       "      <td>0.25</td>\n",
       "      <td>1</td>\n",
       "      <td>0.111111</td>\n",
       "      <td>1</td>\n",
       "      <td>0.041667</td>\n",
       "      <td>1</td>\n",
       "      <td>0.111111</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.50</td>\n",
       "      <td>1</td>\n",
       "      <td>0.50</td>\n",
       "      <td>2</td>\n",
       "      <td>0.400000</td>\n",
       "      <td>3</td>\n",
       "      <td>0.375000</td>\n",
       "      <td>2</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 56 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   act_times  add_cart  add_fav  age_range_0.0  age_range_1.0  age_range_2.0  \\\n",
       "0         39         0        2              0              0              0   \n",
       "1         14         0        0              0              0              0   \n",
       "2         18         0        0              0              0              0   \n",
       "3          2         0        0              0              0              0   \n",
       "4          7         0        1              1              0              0   \n",
       "\n",
       "   age_range_3.0  age_range_4.0  age_range_5.0  age_range_6.0       ...        \\\n",
       "0              0              0              0              1       ...         \n",
       "1              0              0              0              1       ...         \n",
       "2              0              0              0              1       ...         \n",
       "3              0              0              0              1       ...         \n",
       "4              0              0              0              0       ...         \n",
       "\n",
       "   uid_mean_purchase  uid_mid_brand_count  uid_mid_brand_ratio  \\\n",
       "0               2.25                    1                 0.25   \n",
       "1               2.25                    1                 0.25   \n",
       "2               2.25                    1                 0.25   \n",
       "3               2.25                    1                 0.25   \n",
       "4               1.50                    1                 0.50   \n",
       "\n",
       "   uid_mid_cat_count  uid_mid_cat_ratio  uid_mid_item_count  \\\n",
       "0                  6           0.666667                  20   \n",
       "1                  1           0.111111                   1   \n",
       "2                  1           0.111111                   2   \n",
       "3                  1           0.111111                   1   \n",
       "4                  2           0.400000                   3   \n",
       "\n",
       "   uid_mid_item_ratio  uid_mid_time_count  uid_mid_time_ratio  uid_time_count  \n",
       "0            0.833333                   9            1.000000               9  \n",
       "1            0.041667                   3            0.333333               9  \n",
       "2            0.083333                   2            0.222222               9  \n",
       "3            0.041667                   1            0.111111               9  \n",
       "4            0.375000                   2            0.666667               3  \n",
       "\n",
       "[5 rows x 56 columns]"
      ]
     },
     "execution_count": 190,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# x_for_PCA_backup = x_for_PCA\n",
    "# y_for_PCA_backup = y_for_PCA\n",
    "# for_test_backup = for_test\n",
    "# x_for_PCA = x_for_PCA_backup\n",
    "# y_for_PCA = y_for_PCA_backup\n",
    "\n",
    "x_for_PCA.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_for_PCA = y_for_PCA.to_frame(name = 'label')\n",
    "y_for_PCA.reset_index(drop = True, inplace = True)\n",
    "# x_for_PCA.reset_index(drop = True, inplace = True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PCA (n_components to be tuned)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.decomposition import PCA  \n",
    "pca = PCA(n_components=50)  \n",
    "new_x_for_PCA = pd.DataFrame(pca.fit_transform(x_for_PCA))  \n",
    "# new_x_for_PCA = x_for_PCA\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 522341 entries, 0 to 522340\n",
      "Data columns (total 50 columns):\n",
      "0     522341 non-null float64\n",
      "1     522341 non-null float64\n",
      "2     522341 non-null float64\n",
      "3     522341 non-null float64\n",
      "4     522341 non-null float64\n",
      "5     522341 non-null float64\n",
      "6     522341 non-null float64\n",
      "7     522341 non-null float64\n",
      "8     522341 non-null float64\n",
      "9     522341 non-null float64\n",
      "10    522341 non-null float64\n",
      "11    522341 non-null float64\n",
      "12    522341 non-null float64\n",
      "13    522341 non-null float64\n",
      "14    522341 non-null float64\n",
      "15    522341 non-null float64\n",
      "16    522341 non-null float64\n",
      "17    522341 non-null float64\n",
      "18    522341 non-null float64\n",
      "19    522341 non-null float64\n",
      "20    522341 non-null float64\n",
      "21    522341 non-null float64\n",
      "22    522341 non-null float64\n",
      "23    522341 non-null float64\n",
      "24    522341 non-null float64\n",
      "25    522341 non-null float64\n",
      "26    522341 non-null float64\n",
      "27    522341 non-null float64\n",
      "28    522341 non-null float64\n",
      "29    522341 non-null float64\n",
      "30    522341 non-null float64\n",
      "31    522341 non-null float64\n",
      "32    522341 non-null float64\n",
      "33    522341 non-null float64\n",
      "34    522341 non-null float64\n",
      "35    522341 non-null float64\n",
      "36    522341 non-null float64\n",
      "37    522341 non-null float64\n",
      "38    522341 non-null float64\n",
      "39    522341 non-null float64\n",
      "40    522341 non-null float64\n",
      "41    522341 non-null float64\n",
      "42    522341 non-null float64\n",
      "43    522341 non-null float64\n",
      "44    522341 non-null float64\n",
      "45    522341 non-null float64\n",
      "46    522341 non-null float64\n",
      "47    522341 non-null float64\n",
      "48    522341 non-null float64\n",
      "49    522341 non-null float64\n",
      "dtypes: float64(50)\n",
      "memory usage: 199.3 MB\n"
     ]
    }
   ],
   "source": [
    "new_x_for_PCA.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## concat new_x_for_PCA and y_for_PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for_test = new_x_for_PCA.insert(0, 'label', y_for_PCA)\n",
    "for_test = pd.concat([y_for_PCA, new_x_for_PCA], axis=1)\n",
    "# for_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 522341 entries, 0 to 522340\n",
      "Data columns (total 51 columns):\n",
      "label    260864 non-null float64\n",
      "0        522341 non-null float64\n",
      "1        522341 non-null float64\n",
      "2        522341 non-null float64\n",
      "3        522341 non-null float64\n",
      "4        522341 non-null float64\n",
      "5        522341 non-null float64\n",
      "6        522341 non-null float64\n",
      "7        522341 non-null float64\n",
      "8        522341 non-null float64\n",
      "9        522341 non-null float64\n",
      "10       522341 non-null float64\n",
      "11       522341 non-null float64\n",
      "12       522341 non-null float64\n",
      "13       522341 non-null float64\n",
      "14       522341 non-null float64\n",
      "15       522341 non-null float64\n",
      "16       522341 non-null float64\n",
      "17       522341 non-null float64\n",
      "18       522341 non-null float64\n",
      "19       522341 non-null float64\n",
      "20       522341 non-null float64\n",
      "21       522341 non-null float64\n",
      "22       522341 non-null float64\n",
      "23       522341 non-null float64\n",
      "24       522341 non-null float64\n",
      "25       522341 non-null float64\n",
      "26       522341 non-null float64\n",
      "27       522341 non-null float64\n",
      "28       522341 non-null float64\n",
      "29       522341 non-null float64\n",
      "30       522341 non-null float64\n",
      "31       522341 non-null float64\n",
      "32       522341 non-null float64\n",
      "33       522341 non-null float64\n",
      "34       522341 non-null float64\n",
      "35       522341 non-null float64\n",
      "36       522341 non-null float64\n",
      "37       522341 non-null float64\n",
      "38       522341 non-null float64\n",
      "39       522341 non-null float64\n",
      "40       522341 non-null float64\n",
      "41       522341 non-null float64\n",
      "42       522341 non-null float64\n",
      "43       522341 non-null float64\n",
      "44       522341 non-null float64\n",
      "45       522341 non-null float64\n",
      "46       522341 non-null float64\n",
      "47       522341 non-null float64\n",
      "48       522341 non-null float64\n",
      "49       522341 non-null float64\n",
      "dtypes: float64(51)\n",
      "memory usage: 203.2 MB\n"
     ]
    }
   ],
   "source": [
    "for_test.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Split train and test "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(260864, 51) (261477, 51)\n"
     ]
    }
   ],
   "source": [
    "train_after_split = for_test[for_test.label.notnull()]\n",
    "test_after_split = for_test[for_test.label.isnull()]\n",
    "\n",
    "print(train_after_split.shape, test_after_split.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare variables for model training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(260864,) (260864, 50) (261477,) (261477, 50)\n"
     ]
    }
   ],
   "source": [
    "y_train = train_after_split.iloc[:, 0]\n",
    "x_train = train_after_split.iloc[:, 1:]\n",
    "y_test = test_after_split.iloc[:, 0]\n",
    "x_test = test_after_split.iloc[:, 1:]\n",
    "\n",
    "print(y_train.shape, x_train.shape, y_test.shape, x_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Leave out some local test set (train_test_split)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\install\\anacoda3\\lib\\site-packages\\sklearn\\cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.cross_validation import train_test_split\n",
    "# leave_out_x_train, leave_out_x_test, leave_out_y_train, leave_out_y_test = train_test_split(x_train, y_train, test_size=0.2, random_state=18, stratify = y_train)   \n",
    "# x_train = leave_out_x_train\n",
    "# y_train = leave_out_y_train\n",
    "\n",
    "# print(x_train.shape, y_train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0    244912\n",
       "1.0     15952\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 199,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [],
   "source": [
    "# leave_out_y_test.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# # SMOTE\n",
    "\n",
    "# #oversampling, use SMOTE algorithm\n",
    "# from imblearn.over_sampling import SMOTE\n",
    "# from sklearn.ensemble import RandomForestClassifier\n",
    "# from sklearn.metrics import confusion_matrix\n",
    "# from sklearn.model_selection import train_test_split\n",
    "\n",
    "# oversampler = SMOTE(random_state=0)\n",
    "# over_sample_features,over_sample_labels = oversampler.fit_sample(x_train, y_train)\n",
    "\n",
    "# # over_sample_features.shape (14620, 31).\n",
    "# # Note it's an array\n",
    "# # over_sample_features\n",
    "# # total_rows = over_sample_features.shape[0]\n",
    "# # 14620 rows in total after oversampling\n",
    "\n",
    "# x_train = pd.DataFrame(over_sample_features)\n",
    "# y_train = pd.DataFrame(over_sample_labels)\n",
    "\n",
    "# y_train[0].value_counts().plot.bar()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {},
   "outputs": [],
   "source": [
    "ttttt=y_train\n",
    "ttttt=ttttt.values.squeeze()\n",
    "y_train=ttttt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Start training Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(260864, 50)"
      ]
     },
     "execution_count": 203,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Random Forest Analysis\n",
      "Training Accuracy: 0.776\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:   55.7s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of CV: 0.7058 (+/- 0.0066)\n",
      "TNR: 0.73, FPR:0.27, FNR:0.34, TPR:0.66\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn import  metrics  \n",
    "\n",
    "rf2 = RandomForestClassifier(n_estimators=50, \n",
    "                             random_state = 18, \n",
    "                             class_weight='balanced',\n",
    "                             n_jobs=-1,\n",
    "                             max_features='sqrt',\n",
    "                            min_samples_leaf = 40,\n",
    "                            max_depth=10)\n",
    "\n",
    "# rf2 = RandomForestClassifier(\n",
    "#     n_estimators = 150, \n",
    "#     max_depth = 22,\n",
    "#     min_samples_split = 15,\n",
    "#     min_samples_leaf = 4,\n",
    "#     max_features = 10,\n",
    "#     oob_score = True, random_state = 10, class_weight='balanced')  \n",
    "\n",
    "skf_rf = StratifiedKFold(n_splits=5,shuffle=True)\n",
    "\n",
    "print(\"Random Forest Analysis\")\n",
    "rf2.fit(x_train, y_train)\n",
    "\n",
    "pre = rf2.predict(x_train)\n",
    "predict_label_train = rf2.predict_proba(x_train)[:,1] \n",
    "\n",
    "train_auc = metrics.roc_auc_score(y_train,predict_label_train)#\n",
    "print(\"Training Accuracy: %.3f\"%train_auc)\n",
    "\n",
    "\n",
    "scores_rf = cross_val_score(rf2, x_train, y_train, cv=skf_rf,verbose=1,scoring='roc_auc')\n",
    "print(\"Accuracy of CV: %0.4f (+/- %0.4f)\" % (scores_rf.mean(), scores_rf.std()))\n",
    "# print(scores_rf)\n",
    "\n",
    "\n",
    "tn, fp, fn, tp = metrics.confusion_matrix(y_train, pre, labels=None, sample_weight=None).ravel()\n",
    "[tn, fp, fn, tp]\n",
    "print(\"TNR: %0.2f, FPR:%0.2f, FNR:%0.2f, TPR:%0.2f\"%(tn/(tn+fp),fp/(tn+fp),fn/(fn+tp),tp/(fn+tp)))\n",
    "\n",
    "\n",
    "# print(\"On local leave-out test:\")\n",
    "# predict_on_leave_out_test = rf2.predict_proba(leave_out_x_test)[:,1] \n",
    "# leave_out_auc = metrics.roc_auc_score(leave_out_y_test, predict_on_leave_out_test)#\n",
    "# print(\"AUC on leave-out test: %.3f\"%leave_out_auc)\n",
    "\n",
    "\n",
    "\n",
    "predict_label_test =  rf2.predict_proba(x_test)[:,1] \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {},
   "outputs": [],
   "source": [
    "# write back to CSV"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# xgboost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "lgb\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\install\\anacoda3\\lib\\site-packages\\sklearn\\preprocessing\\label.py:151: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if diff:\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy: 0.775\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:   11.2s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC of CV: 0.7101 (+/- 0.0023)\n",
      "[0.71447795 0.70751696 0.70900414 0.71018748 0.70949157]\n",
      "TNR: 0.69, FPR:0.31, FNR:0.29, TPR:0.71\n"
     ]
    }
   ],
   "source": [
    "import lightgbm as lgb\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn import  metrics \n",
    "\n",
    "params = {\n",
    "    'boosting_type': 'gbdt',\n",
    "    'objective': 'binary',\n",
    "    'metric': {'binary_logloss', 'auc'},\n",
    "    'num_leaves': 5,\n",
    "    'max_depth': 6,\n",
    "    'min_data_in_leaf': 450,\n",
    "    'learning_rate': 0.1,\n",
    "    'feature_fraction': 0.9,\n",
    "    'bagging_fraction': 0.95,\n",
    "    'bagging_freq': 5,\n",
    "    'lambda_l1': 1,  \n",
    "    'lambda_l2': 0.001,  # 越小: L2正则程度越高\n",
    "    'min_gain_to_split': 0.2,\n",
    "    'verbose': 5,\n",
    "    'is_unbalance': True\n",
    "}\n",
    "\n",
    "\n",
    "\n",
    "lgbm = lgb.sklearn.LGBMClassifier(\n",
    "                             class_weight='balanced',\n",
    "                             n_jobs=-1,\n",
    "                            )\n",
    "\n",
    "skf_lgbm = StratifiedKFold(n_splits=5,shuffle=True)\n",
    "\n",
    "print(\"lgb\")\n",
    "lgbm.fit(x_train, y_train)\n",
    "\n",
    "pre = lgbm.predict(x_train)\n",
    "predict_label_train = lgbm.predict_proba(x_train)[:,1] \n",
    "\n",
    "train_auc = metrics.roc_auc_score(y_train, predict_label_train)#\n",
    "print(\"Training Accuracy: %.3f\"%train_auc)\n",
    "\n",
    "scores_lgbm = cross_val_score(lgbm, x_train, y_train, cv=skf_lgbm,verbose=1,scoring='roc_auc')\n",
    "print(\"AUC of CV: %0.4f (+/- %0.4f)\" % (scores_lgbm.mean(), scores_lgbm.std()))\n",
    "print(scores_lgbm)\n",
    "\n",
    "\n",
    "tn, fp, fn, tp = metrics.confusion_matrix(y_train, pre, labels=None, sample_weight=None).ravel()\n",
    "[tn, fp, fn, tp]\n",
    "print(\"TNR: %0.2f, FPR:%0.2f, FNR:%0.2f, TPR:%0.2f\"%(tn/(tn+fp),fp/(tn+fp),fn/(fn+tp),tp/(fn+tp)))\n",
    "\n",
    "\n",
    "# print(\"On local leave-out test:\")\n",
    "# predict_on_leave_out_test = lgbm.predict_proba(leave_out_x_test)[:,1] \n",
    "# leave_out_auc = metrics.roc_auc_score(leave_out_y_test, predict_on_leave_out_test)#\n",
    "# print(\"AUC on leave-out test: %.3f\"%leave_out_auc)\n",
    "\n",
    "\n",
    "predict_label_test =  lgbm.predict_proba(x_test)[:,1] \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {},
   "outputs": [],
   "source": [
    "# result = pd.DataFrame({'user_id':test_for_output['user_id'].as_matrix(), \n",
    "#                        'merchant_id':test_for_output['merchant_id'].as_matrix(), \n",
    "#                        'label':predict_label_test.astype(np.float32)},\n",
    "#                      columns=['user_id','merchant_id','label'])\n",
    "# result.to_csv(\"./u_it_cat_b_time_age87_0608_ALL_train_711_LGB_with_PCA46.csv\", index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from sklearn.ensemble import AdaBoostClassifier\n",
    "# from sklearn.tree import DecisionTreeClassifier\n",
    "\n",
    "# ada = AdaBoostClassifier(base_estimator=DecisionTreeClassifier(class_weight='balanced'))\n",
    "\n",
    "# skf_ada = StratifiedKFold(n_splits=5,shuffle=True)\n",
    "\n",
    "# print(\"ada Analysis\")\n",
    "# ada.fit(x_train, y_train)\n",
    "\n",
    "# pre = ada.predict(x_train)\n",
    "# predict_label_train = ada.predict_proba(x_train)[:,1] \n",
    "\n",
    "# train_auc = metrics.roc_auc_score(y_train,predict_label_train)#\n",
    "# print(\"Training Accuracy: %.3f\"%train_auc)\n",
    "\n",
    "\n",
    "# scores_ada = cross_val_score(rf2, x_train, y_train, cv=skf_ada,verbose=1,scoring='roc_auc')\n",
    "# print(\"Accuracy of CV: %0.2f (+/- %0.2f)\" % (scores_rf.mean(), scores_rf.std()))\n",
    "# # print(scores_rf)\n",
    "\n",
    "\n",
    "# tn, fp, fn, tp = metrics.confusion_matrix(y_train, pre, labels=None, sample_weight=None).ravel()\n",
    "# [tn, fp, fn, tp]\n",
    "# print(\"TNR: %0.2f, FPR:%0.2f, FNR:%0.2f, TPR:%0.2f\"%(tn/(tn+fp),fp/(tn+fp),fn/(fn+tp),tp/(fn+tp)))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Result of LR\n",
      "Training Accuracy: 0.718\n",
      "Accuracy of CV: 0.7116 (+/- 0.0101)\n",
      "TNR: 0.69, FPR:0.31, FNR:0.29, TPR:0.71\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:  3.0min finished\n"
     ]
    }
   ],
   "source": [
    "# rescale the data (center and normalize)\n",
    "\n",
    "from sklearn import preprocessing\n",
    "normer = preprocessing.StandardScaler()\n",
    "# x_train_norm = normer.fit_transform(x_train)\n",
    "# x_test_norm = normer.transform(x_test)\n",
    "x_train_norm =x_train\n",
    "x_test_norm = x_test\n",
    "from sklearn.linear_model import LogisticRegression  \n",
    "lr = LogisticRegression(penalty = 'l2',class_weight = 'balanced')  \n",
    "\n",
    "lr.fit(x_train_norm, y_train)  \n",
    "# lr.score(x_train_norm, y_train)  \n",
    "# pre = lr.predict(x_train_norm)\n",
    "skf_lr = StratifiedKFold(n_splits = 5, shuffle = True)\n",
    "\n",
    "print(\"Result of LR\")\n",
    "\n",
    "predict_label_train = lr.predict_proba(x_train)[:,1] \n",
    "\n",
    "train_auc = metrics.roc_auc_score(y_train,predict_label_train)#\n",
    "print(\"Training Accuracy: %.3f\"%train_auc)\n",
    "\n",
    "scores_lr = cross_val_score(lr, x_train_norm, y_train, cv = skf_lr, verbose = 1,scoring='roc_auc')\n",
    "print(\"Accuracy of CV: %0.4f (+/- %0.4f)\" % (scores_lr.mean(), scores_lr.std()))\n",
    "\n",
    "tn, fp, fn, tp = metrics.confusion_matrix(y_train, pre, labels = None, sample_weight = None).ravel()\n",
    "[tn, fp, fn, tp]\n",
    "print(\"TNR: %0.2f, FPR:%0.2f, FNR:%0.2f, TPR:%0.2f\"%(tn/(tn+fp),fp/(tn+fp),fn/(fn+tp),tp/(fn+tp)))\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "predict_label_test = lr.predict_proba(x_test_norm)[:,1] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {},
   "outputs": [],
   "source": [
    "# result = pd.DataFrame({'user_id':test_for_output['user_id'].as_matrix(), \n",
    "#                        'merchant_id':test_for_output['merchant_id'].as_matrix(), \n",
    "#                        'label':predict_label_test.astype(np.float32)},\n",
    "#                      columns=['user_id','merchant_id','label'])\n",
    "# result.to_csv(\"./LR_0617_cv_0.72.csv\", index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=6)]: Done   2 out of   6 | elapsed:  1.8min remaining:  3.6min\n",
      "[Parallel(n_jobs=6)]: Done   6 out of   6 | elapsed:  1.9min finished\n",
      "[Parallel(n_jobs=6)]: Done   2 out of   6 | elapsed:   18.1s remaining:   36.3s\n",
      "[Parallel(n_jobs=6)]: Done   6 out of   6 | elapsed:   21.4s finished\n",
      "[Parallel(n_jobs=6)]: Done   2 out of   6 | elapsed:   15.3s remaining:   30.7s\n",
      "[Parallel(n_jobs=6)]: Done   6 out of   6 | elapsed:   21.8s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Result of bb\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=6)]: Done   2 out of   6 | elapsed:   15.1s remaining:   30.3s\n",
      "[Parallel(n_jobs=6)]: Done   6 out of   6 | elapsed:   21.3s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy: 0.782\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=6)]: Done   2 out of   6 | elapsed:  1.4min remaining:  2.7min\n",
      "[Parallel(n_jobs=6)]: Done   6 out of   6 | elapsed:  1.4min finished\n",
      "[Parallel(n_jobs=6)]: Done   2 out of   6 | elapsed:    4.3s remaining:    8.7s\n",
      "[Parallel(n_jobs=6)]: Done   6 out of   6 | elapsed:    8.4s finished\n",
      "[Parallel(n_jobs=6)]: Done   2 out of   6 | elapsed:  1.3min remaining:  2.7min\n",
      "[Parallel(n_jobs=6)]: Done   6 out of   6 | elapsed:  1.4min finished\n",
      "[Parallel(n_jobs=6)]: Done   2 out of   6 | elapsed:    4.7s remaining:    9.6s\n",
      "[Parallel(n_jobs=6)]: Done   6 out of   6 | elapsed:    8.8s finished\n",
      "[Parallel(n_jobs=6)]: Done   2 out of   6 | elapsed:  1.3min remaining:  2.6min\n",
      "[Parallel(n_jobs=6)]: Done   6 out of   6 | elapsed:  1.4min finished\n",
      "[Parallel(n_jobs=6)]: Done   2 out of   6 | elapsed:    4.2s remaining:    8.5s\n",
      "[Parallel(n_jobs=6)]: Done   6 out of   6 | elapsed:    8.3s finished\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:  4.7min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of CV: 0.7122 (+/- 0.0024)\n",
      "TNR: 0.74, FPR:0.26, FNR:0.34, TPR:0.66\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import BaggingClassifier\n",
    "bb=BaggingClassifier(base_estimator=lgb.sklearn.LGBMClassifier(\n",
    "                             class_weight='balanced',\n",
    "                             n_jobs=-1,\n",
    "                            ), n_estimators=70, max_features=35, bootstrap=True, bootstrap_features=False, n_jobs=-1, random_state=3, verbose=1)\n",
    "bb.fit(x_train, y_train)  \n",
    "bb.score(x_train, y_train)  \n",
    "pre = bb.predict(x_train)\n",
    "skf_bb = StratifiedKFold(n_splits = 3, shuffle = True)\n",
    "\n",
    "print(\"Result of bb\")\n",
    "\n",
    "predict_label_train = bb.predict_proba(x_train)[:,1] \n",
    "\n",
    "train_auc = metrics.roc_auc_score(y_train,predict_label_train)#\n",
    "print(\"Training Accuracy: %.3f\"%train_auc)\n",
    "scores_bb = cross_val_score(bb, x_train_norm, y_train, cv = skf_bb, verbose = 1,scoring='roc_auc')\n",
    "print(\"Accuracy of CV: %0.4f (+/- %0.4f)\" % (scores_bb.mean(), scores_bb.std()))\n",
    "\n",
    "tn, fp, fn, tp = metrics.confusion_matrix(y_train, pre, labels = None, sample_weight = None).ravel()\n",
    "[tn, fp, fn, tp]\n",
    "print(\"TNR: %0.2f, FPR:%0.2f, FNR:%0.2f, TPR:%0.2f\"%(tn/(tn+fp),fp/(tn+fp),fn/(fn+tp),tp/(fn+tp)))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# predict_label_test = bb.predict_proba(x_test_norm)[:,1] \n",
    "# result = pd.DataFrame({'user_id':test_for_output['user_id'].as_matrix(), \n",
    "#                        'merchant_id':test_for_output['merchant_id'].as_matrix(), \n",
    "#                        'label':predict_label_test.astype(np.float32)},\n",
    "#                      columns=['user_id','merchant_id','label'])\n",
    "# result.to_csv(\"./bagging_lgb_0617_cv_0.71.csv\", index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3-fold cross validation:\n",
      "\n",
      "Accuracy: 0.72 (+/- 0.00) [LR]\n",
      "Accuracy: 0.71 (+/- 0.00) [Random Forest]\n",
      "Accuracy: 0.71 (+/- 0.00) [LGB]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\install\\anacoda3\\lib\\site-packages\\sklearn\\preprocessing\\label.py:151: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if diff:\n",
      "D:\\install\\anacoda3\\lib\\site-packages\\sklearn\\preprocessing\\label.py:151: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if diff:\n",
      "D:\\install\\anacoda3\\lib\\site-packages\\sklearn\\preprocessing\\label.py:151: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if diff:\n",
      "D:\\install\\anacoda3\\lib\\site-packages\\sklearn\\preprocessing\\label.py:151: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if diff:\n",
      "D:\\install\\anacoda3\\lib\\site-packages\\sklearn\\preprocessing\\label.py:151: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if diff:\n",
      "D:\\install\\anacoda3\\lib\\site-packages\\sklearn\\preprocessing\\label.py:151: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if diff:\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.65 (+/- 0.00) [StackingClassifier]\n"
     ]
    },
    {
     "ename": "NotFittedError",
     "evalue": "This StackingClassifier instance is not fitted yet. Call 'fit' with appropriate arguments before using this method.",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNotFittedError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-216-759285313040>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     45\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     46\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 47\u001b[1;33m \u001b[0mpredict_label_test\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msclf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict_proba\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx_test\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32mD:\\install\\anacoda3\\lib\\site-packages\\mlxtend\\classifier\\stacking_classification.py\u001b[0m in \u001b[0;36mpredict_proba\u001b[1;34m(self, X)\u001b[0m\n\u001b[0;32m    248\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    249\u001b[0m         \"\"\"\n\u001b[1;32m--> 250\u001b[1;33m         \u001b[0mcheck_is_fitted\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'clfs_'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    251\u001b[0m         \u001b[0mmeta_features\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict_meta_features\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    252\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\install\\anacoda3\\lib\\site-packages\\mlxtend\\externals\\estimator_checks.py\u001b[0m in \u001b[0;36mcheck_is_fitted\u001b[1;34m(estimator, attributes, msg, all_or_any)\u001b[0m\n\u001b[0;32m     71\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     72\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mall_or_any\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mhasattr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mestimator\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mattr\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mattr\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mattributes\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 73\u001b[1;33m         \u001b[1;32mraise\u001b[0m \u001b[0mNotFittedError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmsg\u001b[0m \u001b[1;33m%\u001b[0m \u001b[1;33m{\u001b[0m\u001b[1;34m'name'\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mtype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mestimator\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__name__\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mNotFittedError\u001b[0m: This StackingClassifier instance is not fitted yet. Call 'fit' with appropriate arguments before using this method."
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from mlxtend.classifier import StackingClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "import lightgbm as lgb\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "# clf1 = KNeighborsClassifier(n_neighbors=10,n_jobs=-1)\n",
    "\n",
    "clf1 = LogisticRegression(penalty = 'l2',class_weight = 'balanced')\n",
    "clf2 = RandomForestClassifier(n_estimators=50, \n",
    "                             random_state = 18, \n",
    "                             class_weight='balanced',\n",
    "                             n_jobs=-1,\n",
    "                             max_features='sqrt',\n",
    "                            min_samples_leaf = 40,\n",
    "                            max_depth=10)\n",
    "clf3 = lgb.sklearn.LGBMClassifier(class_weight='balanced',n_jobs=-1)\n",
    "# clf4 = AdaBoostClassifier(base_estimator=DecisionTreeClassifier(class_weight='balanced'))\n",
    "# clf5 = ExtraTreesClassifier(n_jobs=-1,class_weight='balanced')\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "sclf = StackingClassifier(classifiers=[clf1, clf2, clf3], \n",
    "                          meta_classifier=AdaBoostClassifier(base_estimator=DecisionTreeClassifier(class_weight='balanced')))\n",
    "skf_stack = StratifiedKFold(n_splits = 3, shuffle = True)\n",
    "print('3-fold cross validation:\\n')\n",
    "\n",
    "for clf, label in zip([clf1, clf2, clf3, sclf], \n",
    "                      ['LR', \n",
    "                       'Random Forest', \n",
    "                       'LGB',\n",
    "                       'StackingClassifier']):\n",
    "\n",
    "    scores = cross_val_score(clf, x_train, y_train, \n",
    "                                              cv=skf_stack, scoring='roc_auc')\n",
    "    b\n",
    "    \n",
    "\n",
    "\n",
    "\n",
    "\n",
    "predict_label_test = sclf.predict_proba(x_test)[:,1] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# result = pd.DataFrame({'user_id':test_for_output['user_id'].as_matrix(), \n",
    "#                        'merchant_id':test_for_output['merchant_id'].as_matrix(), \n",
    "#                        'label':predict_label_test.astype(np.float32)},\n",
    "#                      columns=['user_id','merchant_id','label'])\n",
    "# result.to_csv(\"./0607_649_leave_stk_with_PCA_with_SMOTE_max_fea.csv\", index = False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}