Day-31/Titanic_survival_competition_2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ebe55349",
   "metadata": {},
   "source": [
    "# [Titanic - Machine learning from Disaster](https://www.kaggle.com/competitions/titanic/)\n",
    "\n",
    "- [My notebook](https://www.kaggle.com/code/param302/titanic-survival-competition-2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "36aebaac",
   "metadata": {
    "papermill": {
     "duration": 0.007995,
     "end_time": "2022-07-05T16:46:52.019499",
     "exception": false,
     "start_time": "2022-07-05T16:46:52.011504",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Importing required libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "92dde964",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-05T16:46:52.038070Z",
     "iopub.status.busy": "2022-07-05T16:46:52.037216Z",
     "iopub.status.idle": "2022-07-05T16:46:53.834807Z",
     "shell.execute_reply": "2022-07-05T16:46:53.833434Z"
    },
    "papermill": {
     "duration": 1.810069,
     "end_time": "2022-07-05T16:46:53.837789",
     "exception": false,
     "start_time": "2022-07-05T16:46:52.027720",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.preprocessing import OrdinalEncoder, OneHotEncoder\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import cross_val_score"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a201d888",
   "metadata": {
    "papermill": {
     "duration": 0.00762,
     "end_time": "2022-07-05T16:46:53.853408",
     "exception": false,
     "start_time": "2022-07-05T16:46:53.845788",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Approach\n",
    "1. Load data\n",
    "2. Select useful features\n",
    "3. Extract numerical and categorical columns\n",
    "4. Make pipeline\n",
    "5. Do Hyperparameter tuning with cross-validation\n",
    "- a. One with Ordinal Encoding\n",
    "- b. One with OneHot Encoding, use best\n",
    "6. Check MAE\n",
    "7. Find best parameters\n",
    "8. Train model\n",
    "9. Predicting `survived` on `test_data`\n",
    "10. Submit predictions"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e5a8a354",
   "metadata": {
    "papermill": {
     "duration": 0.007206,
     "end_time": "2022-07-05T16:46:53.868117",
     "exception": false,
     "start_time": "2022-07-05T16:46:53.860911",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## 1. Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5b22cd03",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-05T16:46:53.885867Z",
     "iopub.status.busy": "2022-07-05T16:46:53.884659Z",
     "iopub.status.idle": "2022-07-05T16:46:53.952023Z",
     "shell.execute_reply": "2022-07-05T16:46:53.950683Z"
    },
    "papermill": {
     "duration": 0.079267,
     "end_time": "2022-07-05T16:46:53.954752",
     "exception": false,
     "start_time": "2022-07-05T16:46:53.875485",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Name</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Ticket</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PassengerId</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Braund, Mr. Owen Harris</td>\n",
       "      <td>male</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>A/5 21171</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Cumings, Mrs. John Bradley (Florence Briggs Th...</td>\n",
       "      <td>female</td>\n",
       "      <td>38.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>PC 17599</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>C85</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>Heikkinen, Miss. Laina</td>\n",
       "      <td>female</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>STON/O2. 3101282</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Futrelle, Mrs. Jacques Heath (Lily May Peel)</td>\n",
       "      <td>female</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>113803</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>C123</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Allen, Mr. William Henry</td>\n",
       "      <td>male</td>\n",
       "      <td>35.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>373450</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             Survived  Pclass  \\\n",
       "PassengerId                     \n",
       "1                   0       3   \n",
       "2                   1       1   \n",
       "3                   1       3   \n",
       "4                   1       1   \n",
       "5                   0       3   \n",
       "\n",
       "                                                          Name     Sex   Age  \\\n",
       "PassengerId                                                                    \n",
       "1                                      Braund, Mr. Owen Harris    male  22.0   \n",
       "2            Cumings, Mrs. John Bradley (Florence Briggs Th...  female  38.0   \n",
       "3                                       Heikkinen, Miss. Laina  female  26.0   \n",
       "4                 Futrelle, Mrs. Jacques Heath (Lily May Peel)  female  35.0   \n",
       "5                                     Allen, Mr. William Henry    male  35.0   \n",
       "\n",
       "             SibSp  Parch            Ticket     Fare Cabin Embarked  \n",
       "PassengerId                                                          \n",
       "1                1      0         A/5 21171   7.2500   NaN        S  \n",
       "2                1      0          PC 17599  71.2833   C85        C  \n",
       "3                0      0  STON/O2. 3101282   7.9250   NaN        S  \n",
       "4                1      0            113803  53.1000  C123        S  \n",
       "5                0      0            373450   8.0500   NaN        S  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "titanic_data = pd.read_csv(\"../input/titanic/train.csv\", index_col=\"PassengerId\")\n",
    "test_data = pd.read_csv(\"../input/titanic/test.csv\", index_col=\"PassengerId\")\n",
    "\n",
    "titanic_data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8c18cdfb",
   "metadata": {
    "papermill": {
     "duration": 0.007647,
     "end_time": "2022-07-05T16:46:53.970730",
     "exception": false,
     "start_time": "2022-07-05T16:46:53.963083",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## 2. Selecting useful features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "3b410cbd",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-05T16:46:53.989535Z",
     "iopub.status.busy": "2022-07-05T16:46:53.989038Z",
     "iopub.status.idle": "2022-07-05T16:46:54.001639Z",
     "shell.execute_reply": "2022-07-05T16:46:54.000410Z"
    },
    "papermill": {
     "duration": 0.025749,
     "end_time": "2022-07-05T16:46:54.004466",
     "exception": false,
     "start_time": "2022-07-05T16:46:53.978717",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "useful_features = [\"Pclass\", \"Sex\", \"Age\", \"SibSp\", \"Parch\", \"Ticket\", \"Fare\", \"Cabin\", \"Embarked\"]\n",
    "X = titanic_data[useful_features]\n",
    "Y = titanic_data[\"Survived\"]\n",
    "test_data = test_data[useful_features]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0760fb72",
   "metadata": {
    "papermill": {
     "duration": 0.00778,
     "end_time": "2022-07-05T16:46:54.020340",
     "exception": false,
     "start_time": "2022-07-05T16:46:54.012560",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## 3. Extract numerical and categorical columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0443f23e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-05T16:46:54.038991Z",
     "iopub.status.busy": "2022-07-05T16:46:54.038505Z",
     "iopub.status.idle": "2022-07-05T16:46:54.051148Z",
     "shell.execute_reply": "2022-07-05T16:46:54.049675Z"
    },
    "papermill": {
     "duration": 0.025067,
     "end_time": "2022-07-05T16:46:54.053545",
     "exception": false,
     "start_time": "2022-07-05T16:46:54.028478",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Index(['Pclass', 'Age', 'SibSp', 'Parch', 'Fare'], dtype='object'),\n",
       " Index(['Sex', 'Ticket', 'Cabin', 'Embarked'], dtype='object'))"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_cols = X.select_dtypes(exclude=\"object\").columns\n",
    "cat_cols = X.select_dtypes(\"object\").columns\n",
    "num_cols, cat_cols"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fcccdd2b",
   "metadata": {
    "papermill": {
     "duration": 0.008029,
     "end_time": "2022-07-05T16:46:54.069705",
     "exception": false,
     "start_time": "2022-07-05T16:46:54.061676",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## 4. Make Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "28d86293",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-05T16:46:54.089095Z",
     "iopub.status.busy": "2022-07-05T16:46:54.088236Z",
     "iopub.status.idle": "2022-07-05T16:46:54.102377Z",
     "shell.execute_reply": "2022-07-05T16:46:54.101413Z"
    },
    "papermill": {
     "duration": 0.026791,
     "end_time": "2022-07-05T16:46:54.104698",
     "exception": false,
     "start_time": "2022-07-05T16:46:54.077907",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class CreatePipeline:\n",
    "    \"\"\"Create Pipeline\n",
    "    methods:\n",
    "        pipeline: Create Final Pipeline\n",
    "        \n",
    "        create_model: Create the provided model\n",
    "        \n",
    "        numerical_transformer: Transform numerical cols\n",
    "        \n",
    "        categorical_transformer: Transform categorical cols \\\n",
    "        OneHotEncoding / OrdinalEncoding\n",
    "        \n",
    "        data_preprocessor: Preprocess the data using ColumnTransformer     \n",
    "        \"\"\"\n",
    "    \n",
    "    def pipeline(self, *, preprocessor, model, verbose=False):\n",
    "        \"\"\"Creates pipeline\n",
    "        params:\n",
    "            preprocessor\n",
    "            model\n",
    "        \"\"\"\n",
    "        steps = [(\"preprocessor\", preprocessor),\n",
    "                 (\"model\", model)]\n",
    "        return Pipeline(steps=steps, verbose=verbose)\n",
    "    \n",
    "    \n",
    "    def numerical_transformer(self, *, strategy=\"mean\", **params):\n",
    "        \"\"\"Transform numerical columns using `SimpleImputer`.\n",
    "        params:\n",
    "            strategy: \"mean\" | \"median\" | \"most_frequent\" | \"constant\"\n",
    "            **params: extra keyword args for SimpleImputer\"\"\"\n",
    "        \n",
    "        transformer = SimpleImputer(strategy=strategy, **params)\n",
    "        return transformer\n",
    "\n",
    "    \n",
    "    def categorical_transformer(self, *, \n",
    "                                imp_strategy=\"most_frequent\", \n",
    "                                encoder_type=\"Ordinal\", \n",
    "                                imp_params={}, encoder_params={}):\n",
    "        \"\"\"Transform categorical columns by making Pipeline\n",
    "        `SimpleImputer` | `OneHotEncoder` | `OrdinalEncoder`.\n",
    "        args:\n",
    "            imp_strategy: strategy for imputer values can be\n",
    "                \"most_frequent\" | \"constant\"\n",
    "            encoder_type: encoder type,\n",
    "                \"Ordinal\" | \"OneHot\"\n",
    "        kwargs:\n",
    "            imp_params: keyword args for `SimpleImputer`.\n",
    "            encoder_params: keyword args for encoder.`\n",
    "        \"\"\"\n",
    "        if not encoder_type in (\"Ordinal\", \"OneHot\"):\n",
    "            raise ValueError(f\"Inappropriate value for encoder_type passed: {encoder_type}\\\n",
    "            Takes one of 'Ordinal' | 'OneHot'.\")\n",
    "        \n",
    "        encoder = OrdinalEncoder if encoder_type==\"Ordinal\" else OneHotEncoder\n",
    "        transformer = Pipeline(steps=[\n",
    "            (\"imputer\", SimpleImputer(strategy=imp_strategy, **imp_params)),\n",
    "            (encoder_type, encoder(**encoder_params))\n",
    "        ])\n",
    "        return transformer\n",
    "    \n",
    "    \n",
    "    def data_preprocessor(self, *, transformers):\n",
    "        \"\"\"Preprocess the data using `ColumnTransformer`.\n",
    "        Pass extact list of transformers\n",
    "        to be passed in `ColumnTransformer`.\n",
    "        each tuple consist of: (transformer_name,\n",
    "                                transformer,\n",
    "                                list_of_columns).\"\"\"\n",
    "        preprocessor = ColumnTransformer(transformers=transformers)\n",
    "        return preprocessor\n",
    "    \n",
    "    \n",
    "    def create_model(self, *, model, random_state=0, n_estimators=1000, **kwargs):\n",
    "        \"\"\"Creates the model.\n",
    "        **kwargs: keyword args for model.\"\"\"\n",
    "        my_model = model(random_state=random_state, n_estimators=n_estimators, **kwargs)\n",
    "        return my_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "992d5a84",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-05T16:46:54.123783Z",
     "iopub.status.busy": "2022-07-05T16:46:54.122627Z",
     "iopub.status.idle": "2022-07-05T16:46:54.131521Z",
     "shell.execute_reply": "2022-07-05T16:46:54.130253Z"
    },
    "papermill": {
     "duration": 0.021038,
     "end_time": "2022-07-05T16:46:54.133936",
     "exception": false,
     "start_time": "2022-07-05T16:46:54.112898",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "cp = CreatePipeline()\n",
    "num_transformer = cp.numerical_transformer()\n",
    "cat_oh_transformer = cp.categorical_transformer(encoder_type=\"OneHot\", encoder_params={\"handle_unknown\": \"ignore\"})\n",
    "cat_ord_transformer = cp.categorical_transformer(encoder_type=\"Ordinal\", encoder_params={\"handle_unknown\":\"use_encoded_value\", \"unknown_value\":-1})\n",
    "oh_preprocessor = cp.data_preprocessor(\n",
    "                    transformers=[(\"num\", num_transformer, num_cols),\n",
    "                                  (\"cat\", cat_oh_transformer, cat_cols)\n",
    "                                 ])\n",
    "ord_preprocessor = cp.data_preprocessor(\n",
    "                    transformers=[(\"num\", num_transformer, num_cols),\n",
    "                                  (\"cat\", cat_ord_transformer, cat_cols)\n",
    "                                 ])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bc586dfc",
   "metadata": {
    "papermill": {
     "duration": 0.007685,
     "end_time": "2022-07-05T16:46:54.149587",
     "exception": false,
     "start_time": "2022-07-05T16:46:54.141902",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## 5. Hyperparamter tuning with Cross-validation\n",
    "### a. with `OneHot Encoding`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "c334d6f3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-05T16:46:54.168468Z",
     "iopub.status.busy": "2022-07-05T16:46:54.167988Z",
     "iopub.status.idle": "2022-07-05T16:48:43.780061Z",
     "shell.execute_reply": "2022-07-05T16:48:43.778469Z"
    },
    "papermill": {
     "duration": 109.634032,
     "end_time": "2022-07-05T16:48:43.791617",
     "exception": false,
     "start_time": "2022-07-05T16:46:54.157585",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    2.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    2.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    2.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    5.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    5.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    6.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:   10.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:   11.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:   12.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:   15.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:   16.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:   18.8s finished\n"
     ]
    }
   ],
   "source": [
    "n_estimators = [100, 250, 500, 750]\n",
    "max_depths = [5, 10, 20]\n",
    "oh_maes = {}\n",
    "i = 0\n",
    "for n in n_estimators:\n",
    "    for md in max_depths:\n",
    "        i += 1\n",
    "        model = cp.create_model(model=RandomForestClassifier, n_estimators=n, max_depth=md)\n",
    "        pipeline = cp.pipeline(preprocessor=oh_preprocessor, model=model)\n",
    "        scores = -1 * cross_val_score(pipeline, X, Y, cv=10, verbose=True,\n",
    "                                scoring=\"neg_mean_absolute_error\")\n",
    "        mae = scores.mean()\n",
    "        oh_maes[i] = [n, md, mae]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8edd2093",
   "metadata": {
    "papermill": {
     "duration": 0.008995,
     "end_time": "2022-07-05T16:48:43.810034",
     "exception": false,
     "start_time": "2022-07-05T16:48:43.801039",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### b. with `Ordinal Encoding`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "38e4c98a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-05T16:48:43.830407Z",
     "iopub.status.busy": "2022-07-05T16:48:43.829642Z",
     "iopub.status.idle": "2022-07-05T16:50:41.023712Z",
     "shell.execute_reply": "2022-07-05T16:50:41.022035Z"
    },
    "papermill": {
     "duration": 117.216836,
     "end_time": "2022-07-05T16:50:41.036064",
     "exception": false,
     "start_time": "2022-07-05T16:48:43.819228",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    2.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    2.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    2.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    5.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    6.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    6.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:   11.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:   12.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:   12.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:   16.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:   18.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:   18.7s finished\n"
     ]
    }
   ],
   "source": [
    "n_estimators = [100, 250, 500, 750]\n",
    "max_depths = [5, 10, 20]\n",
    "ord_maes = {}\n",
    "i = 0\n",
    "for n in n_estimators:\n",
    "    for md in max_depths:\n",
    "        i += 1\n",
    "        model = cp.create_model(model=RandomForestClassifier, n_estimators=n, max_depth=md)\n",
    "        pipeline = cp.pipeline(preprocessor=ord_preprocessor, model=model)\n",
    "        scores = -1 * cross_val_score(pipeline, X, Y, cv=10, verbose=True,\n",
    "                                scoring=\"neg_mean_absolute_error\")\n",
    "        mae = scores.mean()\n",
    "        ord_maes[i] = [n, md, mae]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "50648568",
   "metadata": {
    "papermill": {
     "duration": 0.009557,
     "end_time": "2022-07-05T16:50:41.056612",
     "exception": false,
     "start_time": "2022-07-05T16:50:41.047055",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## 6. Checking MAEs\n",
    "### a. `OneHot Encoding`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "eab355b2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-05T16:50:41.079287Z",
     "iopub.status.busy": "2022-07-05T16:50:41.078500Z",
     "iopub.status.idle": "2022-07-05T16:50:41.085858Z",
     "shell.execute_reply": "2022-07-05T16:50:41.083484Z"
    },
    "papermill": {
     "duration": 0.022061,
     "end_time": "2022-07-05T16:50:41.088635",
     "exception": false,
     "start_time": "2022-07-05T16:50:41.066574",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.\tN_estimators: 100\tmax_depth: 5\tMAE: 0.23676654182272155\n",
      "2.\tN_estimators: 100\tmax_depth: 10\tMAE: 0.19410736579275906\n",
      "3.\tN_estimators: 100\tmax_depth: 20\tMAE: 0.17283395755305866\n",
      "4.\tN_estimators: 250\tmax_depth: 5\tMAE: 0.23901373283395758\n",
      "5.\tN_estimators: 250\tmax_depth: 10\tMAE: 0.19749063670411984\n",
      "6.\tN_estimators: 250\tmax_depth: 20\tMAE: 0.1705992509363296\n",
      "7.\tN_estimators: 500\tmax_depth: 5\tMAE: 0.23227215980024968\n",
      "8.\tN_estimators: 500\tmax_depth: 10\tMAE: 0.18965043695380773\n",
      "9.\tN_estimators: 500\tmax_depth: 20\tMAE: 0.1683395755305868\n",
      "10.\tN_estimators: 750\tmax_depth: 5\tMAE: 0.22891385767790257\n",
      "11.\tN_estimators: 750\tmax_depth: 10\tMAE: 0.18965043695380773\n",
      "12.\tN_estimators: 750\tmax_depth: 20\tMAE: 0.1728339575530587\n"
     ]
    }
   ],
   "source": [
    "for i in oh_maes:\n",
    "    n, md, mae = oh_maes[i]\n",
    "    print(f\"{i}.\\tN_estimators: {n}\\tmax_depth: {md}\\tMAE: {mae}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c0ac4046",
   "metadata": {
    "papermill": {
     "duration": 0.009705,
     "end_time": "2022-07-05T16:50:41.108207",
     "exception": false,
     "start_time": "2022-07-05T16:50:41.098502",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### b. `Ordinal Encoding`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "6d407c1e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-05T16:50:41.130341Z",
     "iopub.status.busy": "2022-07-05T16:50:41.129848Z",
     "iopub.status.idle": "2022-07-05T16:50:41.137619Z",
     "shell.execute_reply": "2022-07-05T16:50:41.134945Z"
    },
    "papermill": {
     "duration": 0.023518,
     "end_time": "2022-07-05T16:50:41.141571",
     "exception": false,
     "start_time": "2022-07-05T16:50:41.118053",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.\tN_estimators: 100\tmax_depth: 5\tMAE: 0.17843945068664166\n",
      "2.\tN_estimators: 100\tmax_depth: 10\tMAE: 0.18067415730337077\n",
      "3.\tN_estimators: 100\tmax_depth: 20\tMAE: 0.17506866416978778\n",
      "4.\tN_estimators: 250\tmax_depth: 5\tMAE: 0.17394506866416976\n",
      "5.\tN_estimators: 250\tmax_depth: 10\tMAE: 0.17280898876404496\n",
      "6.\tN_estimators: 250\tmax_depth: 20\tMAE: 0.17730337078651687\n",
      "7.\tN_estimators: 500\tmax_depth: 5\tMAE: 0.17394506866416976\n",
      "8.\tN_estimators: 500\tmax_depth: 10\tMAE: 0.17393258426966293\n",
      "9.\tN_estimators: 500\tmax_depth: 20\tMAE: 0.1784269662921348\n",
      "10.\tN_estimators: 750\tmax_depth: 5\tMAE: 0.1728214731585518\n",
      "11.\tN_estimators: 750\tmax_depth: 10\tMAE: 0.1739325842696629\n",
      "12.\tN_estimators: 750\tmax_depth: 20\tMAE: 0.1750561797752809\n"
     ]
    }
   ],
   "source": [
    "for i in ord_maes:\n",
    "    n, md, mae = ord_maes[i]\n",
    "    print(f\"{i}.\\tN_estimators: {n}\\tmax_depth: {md}\\tMAE: {mae}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d870cd5b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-05T16:50:41.165856Z",
     "iopub.status.busy": "2022-07-05T16:50:41.164731Z",
     "iopub.status.idle": "2022-07-05T16:50:41.171849Z",
     "shell.execute_reply": "2022-07-05T16:50:41.171045Z"
    },
    "papermill": {
     "duration": 0.021441,
     "end_time": "2022-07-05T16:50:41.174069",
     "exception": false,
     "start_time": "2022-07-05T16:50:41.152628",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "min(oh_maes, key=lambda x: oh_maes[x][2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "da9369e9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-05T16:50:41.197679Z",
     "iopub.status.busy": "2022-07-05T16:50:41.197116Z",
     "iopub.status.idle": "2022-07-05T16:50:41.203481Z",
     "shell.execute_reply": "2022-07-05T16:50:41.202263Z"
    },
    "papermill": {
     "duration": 0.02121,
     "end_time": "2022-07-05T16:50:41.205918",
     "exception": false,
     "start_time": "2022-07-05T16:50:41.184708",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "min(ord_maes, key=lambda x: ord_maes[x][2])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5b5e2bff",
   "metadata": {
    "papermill": {
     "duration": 0.010243,
     "end_time": "2022-07-05T16:50:41.226841",
     "exception": false,
     "start_time": "2022-07-05T16:50:41.216598",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## 7. Best parameters:\n",
    "- a. `Ordinal Encoding`\n",
    "- - `N_estimators: 250` & `max_depth:10` & `MAE:0.17280898876404496`\n",
    "- b. `One Hot Encoding`\n",
    "- - `N_estimators: 500` &\t`max_depth: 20` & `MAE: 0.1683395755305868`\n",
    "\n",
    "### `OneHotEncoding` did better"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "012cdfeb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-05T16:50:41.250208Z",
     "iopub.status.busy": "2022-07-05T16:50:41.249738Z",
     "iopub.status.idle": "2022-07-05T16:50:41.255505Z",
     "shell.execute_reply": "2022-07-05T16:50:41.253958Z"
    },
    "papermill": {
     "duration": 0.02036,
     "end_time": "2022-07-05T16:50:41.257913",
     "exception": false,
     "start_time": "2022-07-05T16:50:41.237553",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# using onehot encoding\n",
    "best_n_estimators = 500\n",
    "best_max_depth = 20"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e05a7403",
   "metadata": {
    "papermill": {
     "duration": 0.010657,
     "end_time": "2022-07-05T16:50:41.279269",
     "exception": false,
     "start_time": "2022-07-05T16:50:41.268612",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## 8. Training model on best parameters with `OneHotEncoding` as preprocessor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "b5afd138",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-05T16:50:41.302951Z",
     "iopub.status.busy": "2022-07-05T16:50:41.302326Z",
     "iopub.status.idle": "2022-07-05T16:50:42.534821Z",
     "shell.execute_reply": "2022-07-05T16:50:42.533248Z"
    },
    "papermill": {
     "duration": 1.247535,
     "end_time": "2022-07-05T16:50:42.537510",
     "exception": false,
     "start_time": "2022-07-05T16:50:41.289975",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('preprocessor',\n",
       "                 ColumnTransformer(transformers=[('num', SimpleImputer(),\n",
       "                                                  Index(['Pclass', 'Age', 'SibSp', 'Parch', 'Fare'], dtype='object')),\n",
       "                                                 ('cat',\n",
       "                                                  Pipeline(steps=[('imputer',\n",
       "                                                                   SimpleImputer(strategy='most_frequent')),\n",
       "                                                                  ('OneHot',\n",
       "                                                                   OneHotEncoder(handle_unknown='ignore'))]),\n",
       "                                                  Index(['Sex', 'Ticket', 'Cabin', 'Embarked'], dtype='object'))])),\n",
       "                ('model',\n",
       "                 RandomForestClassifier(max_depth=20, n_estimators=500,\n",
       "                                        random_state=0))])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = cp.create_model(model=RandomForestClassifier, n_estimators=best_n_estimators, max_depth=best_max_depth)\n",
    "pipeline = cp.pipeline(preprocessor=oh_preprocessor, model=model)\n",
    "pipeline.fit(X, Y)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f5dafab6",
   "metadata": {
    "papermill": {
     "duration": 0.010161,
     "end_time": "2022-07-05T16:50:42.558526",
     "exception": false,
     "start_time": "2022-07-05T16:50:42.548365",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## 9. Predicting `survived` on `test_data`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "7f29276a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-05T16:50:42.581800Z",
     "iopub.status.busy": "2022-07-05T16:50:42.581361Z",
     "iopub.status.idle": "2022-07-05T16:50:42.703283Z",
     "shell.execute_reply": "2022-07-05T16:50:42.701853Z"
    },
    "papermill": {
     "duration": 0.136964,
     "end_time": "2022-07-05T16:50:42.706003",
     "exception": false,
     "start_time": "2022-07-05T16:50:42.569039",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 1, 0, 0,\n",
       "       1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1,\n",
       "       1, 0, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1,\n",
       "       1, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0,\n",
       "       1, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0,\n",
       "       0, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1,\n",
       "       0, 0, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1,\n",
       "       1, 0, 1, 1, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0,\n",
       "       0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 0,\n",
       "       1, 0, 1, 0, 1, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1,\n",
       "       0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0,\n",
       "       0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0,\n",
       "       1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 1, 0,\n",
       "       0, 0, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0,\n",
       "       1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1,\n",
       "       0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_preds = pipeline.predict(test_data)\n",
    "test_preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "1dba10af",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-05T16:50:42.730937Z",
     "iopub.status.busy": "2022-07-05T16:50:42.730521Z",
     "iopub.status.idle": "2022-07-05T16:50:42.743392Z",
     "shell.execute_reply": "2022-07-05T16:50:42.742400Z"
    },
    "papermill": {
     "duration": 0.028572,
     "end_time": "2022-07-05T16:50:42.745990",
     "exception": false,
     "start_time": "2022-07-05T16:50:42.717418",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "output = pd.DataFrame({\"PassengerId\": test_data.index, \"Survived\": test_preds})\n",
    "output.to_csv(\"./data/submission_2.csv\", index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.10.2 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.2"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 243.214233,
   "end_time": "2022-07-05T16:50:43.783335",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2022-07-05T16:46:40.569102",
   "version": "2.3.4"
  },
  "vscode": {
   "interpreter": {
    "hash": "d88aeb6bf73c3e2735560d04f72b234e9e70c0b4c2e9aa8924f896649b343841"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}