NN-House-Price-Short-and-Kaggle-Submission.ipynb

{"cells":[{"cell_type":"markdown","id":"c8a4c025-9d4f-40f6-92a8-73a308aeca18","metadata":{"jp-MarkdownHeadingCollapsed":true,"tags":[],"id":"c8a4c025-9d4f-40f6-92a8-73a308aeca18"},"source":["<span style=\"font-size:50px;\"> House Prices</span>"]},{"cell_type":"markdown","id":"ccc47802-7ba9-48c6-aa86-f2a78174183f","metadata":{"id":"ccc47802-7ba9-48c6-aa86-f2a78174183f"},"source":["**Kaggle Competition**: [House Prices - Advanced Regression Techniques](https://www.kaggle.com/c/house-prices-advanced-regression-techniques/data)"]},{"cell_type":"markdown","id":"1c7ab448-03da-4e97-a5f0-e912954f9b60","metadata":{"id":"1c7ab448-03da-4e97-a5f0-e912954f9b60"},"source":["# PROJECT SETUP"]},{"cell_type":"markdown","id":"322d24a0-d01c-463c-8919-0493a9eb0ba3","metadata":{"id":"322d24a0-d01c-463c-8919-0493a9eb0ba3"},"source":["## Load Library"]},{"cell_type":"code","execution_count":null,"id":"c1b64f2e-465b-4a30-a286-f66d70fddcc1","metadata":{"id":"c1b64f2e-465b-4a30-a286-f66d70fddcc1"},"outputs":[],"source":["import pandas as pd\n","import numpy as np\n","import matplotlib.pyplot as plt\n","import seaborn as sns\n","\n","# Pipeline Diagram\n","from sklearn import set_config; set_config(display='diagram')\n","\n","# Random Search\n","from scipy import stats\n","from sklearn.model_selection import RandomizedSearchCV, GridSearchCV\n","\n","# Feature Selection\n","from statsmodels.stats.outliers_influence import variance_inflation_factor as vif\n","\n","# Preprocess\n","from sklearn.model_selection import train_test_split\n","from sklearn.preprocessing import MinMaxScaler, StandardScaler, RobustScaler\n","from sklearn.impute import SimpleImputer, KNNImputer\n","from sklearn.preprocessing import RobustScaler, OneHotEncoder, OrdinalEncoder\n","from sklearn.feature_selection import SelectPercentile, mutual_info_regression\n","\n","# Pipeline\n","from sklearn.pipeline import Pipeline, make_pipeline\n","from sklearn.compose import ColumnTransformer, make_column_transformer, make_column_selector\n","\n","# Models\n","from sklearn.ensemble import GradientBoostingRegressor\n","from sklearn.tree import DecisionTreeRegressor\n","from sklearn.svm import SVR\n","\n","# Score\n","from sklearn.model_selection import cross_val_score\n","from sklearn.metrics import mean_squared_log_error, mean_squared_error, make_scorer\n","\n","# Hide warnings\n","import warnings\n","warnings.filterwarnings('ignore')\n","warnings.simplefilter(action='ignore', category=FutureWarning)"]},{"cell_type":"markdown","id":"9a6bff08-52b7-4acd-812d-97f2ed8ac648","metadata":{"id":"9a6bff08-52b7-4acd-812d-97f2ed8ac648"},"source":["## Scoring: `rmsle` and `rmse`"]},{"cell_type":"code","execution_count":null,"id":"52b10541-9f1a-4c1a-9091-9d4435bc8594","metadata":{"id":"52b10541-9f1a-4c1a-9091-9d4435bc8594"},"outputs":[],"source":["from sklearn.metrics import mean_squared_log_error, mean_squared_error, make_scorer\n","\n","rmsle = make_scorer(lambda y_true, y_pred: mean_squared_log_error(y_true, y_pred, squared=False))\n","\n","rmse = make_scorer(lambda y_true, y_pred: mean_squared_error(y_true, y_pred)**0.5)\n","rmse_neg = make_scorer(lambda y_true, y_pred: -1 * mean_squared_error(y_true, y_pred)**0.5)"]},{"cell_type":"markdown","id":"353cfe34-a0ad-4b76-81e4-d8360e0cfd65","metadata":{"tags":[],"id":"353cfe34-a0ad-4b76-81e4-d8360e0cfd65"},"source":["## Load Dataset\n","- Train set: https://storage.googleapis.com/efcabinet/data/house_price_train.csv\n","- Test set: https://storage.googleapis.com/efcabinet/data/house-price-test.csv\n","- Submission sample: https://storage.googleapis.com/efcabinet/data/house-price-sample_submission.csv\n","- Data descripttion: https://storage.googleapis.com/efcabinet/data/house-price-data-description.txt"]},{"cell_type":"markdown","id":"f2b25f71-1b89-4593-8170-56fb49d82fc3","metadata":{"id":"f2b25f71-1b89-4593-8170-56fb49d82fc3"},"source":["**Load the train dataset**"]},{"cell_type":"code","execution_count":null,"id":"752967db-3777-41ee-8319-53d1abdfed0f","metadata":{"id":"752967db-3777-41ee-8319-53d1abdfed0f","outputId":"a02dc3f5-d00b-40c9-961d-00b2c7914590"},"outputs":[{"data":{"text/plain":["(1460, 81)"]},"execution_count":56,"metadata":{},"output_type":"execute_result"}],"source":["data = pd.read_csv(\"https://storage.googleapis.com/efcabinet/data/house_price_train.csv\")\n","data.shape"]},{"cell_type":"markdown","id":"2ebe757a-0bac-48e2-96ff-5b8aab68349f","metadata":{"id":"2ebe757a-0bac-48e2-96ff-5b8aab68349f"},"source":["**Feature Engineering**"]},{"cell_type":"code","execution_count":null,"id":"04ee9bc6-5d14-40bd-9014-288ba629e066","metadata":{"id":"04ee9bc6-5d14-40bd-9014-288ba629e066","outputId":"ab7ff8c5-b0fb-468d-deeb-55eb1cca4dd4"},"outputs":[{"data":{"text/plain":["(1460, 81)"]},"execution_count":57,"metadata":{},"output_type":"execute_result"}],"source":["# Converting non-numeric predictors stored as numbers into string\n","# data['MSSubClass'] = data['MSSubClass'].apply(str)\n","\n","data.shape"]},{"cell_type":"markdown","id":"be6cdd3a-36a8-414c-afc4-4ec5dfcb0c0e","metadata":{"id":"be6cdd3a-36a8-414c-afc4-4ec5dfcb0c0e"},"source":["**Create `X` for features and `y` for target**"]},{"cell_type":"code","execution_count":null,"id":"a3dea4bd-1d17-4c01-9978-7e1ed6919f05","metadata":{"id":"a3dea4bd-1d17-4c01-9978-7e1ed6919f05"},"outputs":[],"source":["X = data.drop(columns=['SalePrice', 'Id'])\n","y = data.SalePrice\n","y_log = np.log(y)"]},{"cell_type":"code","execution_count":null,"id":"e561e6d1-bd26-4ecc-b948-751a6bfcc3eb","metadata":{"id":"e561e6d1-bd26-4ecc-b948-751a6bfcc3eb","outputId":"4b535c7f-805b-40a0-ef51-f2d3b6046994"},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>MSSubClass</th>\n","      <th>MSZoning</th>\n","      <th>LotFrontage</th>\n","      <th>LotArea</th>\n","      <th>Street</th>\n","      <th>Alley</th>\n","      <th>LotShape</th>\n","      <th>LandContour</th>\n","      <th>Utilities</th>\n","      <th>LotConfig</th>\n","      <th>...</th>\n","      <th>PoolArea</th>\n","      <th>PoolQC</th>\n","      <th>Fence</th>\n","      <th>MiscFeature</th>\n","      <th>MiscVal</th>\n","      <th>YrSold</th>\n","      <th>SaleType</th>\n","      <th>SaleCondition</th>\n","      <th>MoSold_sin</th>\n","      <th>MoSold_cos</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>60</td>\n","      <td>RL</td>\n","      <td>65.0</td>\n","      <td>8450</td>\n","      <td>Pave</td>\n","      <td>NaN</td>\n","      <td>Reg</td>\n","      <td>Lvl</td>\n","      <td>AllPub</td>\n","      <td>Inside</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>NaN</td>\n","      <td>NaN</td>\n","      <td>NaN</td>\n","      <td>0</td>\n","      <td>2008</td>\n","      <td>WD</td>\n","      <td>Normal</td>\n","      <td>0.500000</td>\n","      <td>0.866025</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>20</td>\n","      <td>RL</td>\n","      <td>80.0</td>\n","      <td>9600</td>\n","      <td>Pave</td>\n","      <td>NaN</td>\n","      <td>Reg</td>\n","      <td>Lvl</td>\n","      <td>AllPub</td>\n","      <td>FR2</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>NaN</td>\n","      <td>NaN</td>\n","      <td>NaN</td>\n","      <td>0</td>\n","      <td>2007</td>\n","      <td>WD</td>\n","      <td>Normal</td>\n","      <td>0.866025</td>\n","      <td>-0.500000</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>60</td>\n","      <td>RL</td>\n","      <td>68.0</td>\n","      <td>11250</td>\n","      <td>Pave</td>\n","      <td>NaN</td>\n","      <td>IR1</td>\n","      <td>Lvl</td>\n","      <td>AllPub</td>\n","      <td>Inside</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>NaN</td>\n","      <td>NaN</td>\n","      <td>NaN</td>\n","      <td>0</td>\n","      <td>2008</td>\n","      <td>WD</td>\n","      <td>Normal</td>\n","      <td>-0.866025</td>\n","      <td>-0.500000</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>3 rows × 80 columns</p>\n","</div>"],"text/plain":["   MSSubClass MSZoning  LotFrontage  LotArea Street Alley LotShape  \\\n","0          60       RL         65.0     8450   Pave   NaN      Reg   \n","1          20       RL         80.0     9600   Pave   NaN      Reg   \n","2          60       RL         68.0    11250   Pave   NaN      IR1   \n","\n","  LandContour Utilities LotConfig  ... PoolArea PoolQC Fence MiscFeature  \\\n","0         Lvl    AllPub    Inside  ...        0    NaN   NaN         NaN   \n","1         Lvl    AllPub       FR2  ...        0    NaN   NaN         NaN   \n","2         Lvl    AllPub    Inside  ...        0    NaN   NaN         NaN   \n","\n","  MiscVal YrSold  SaleType  SaleCondition  MoSold_sin  MoSold_cos  \n","0       0   2008        WD         Normal    0.500000    0.866025  \n","1       0   2007        WD         Normal    0.866025   -0.500000  \n","2       0   2008        WD         Normal   -0.866025   -0.500000  \n","\n","[3 rows x 80 columns]"]},"execution_count":59,"metadata":{},"output_type":"execute_result"}],"source":["X['MoSold_sin'] = np.sin((X.MoSold-1)*(2.*np.pi/12))\n","X['MoSold_cos'] = np.cos((X.MoSold-1)*(2.*np.pi/12))\n","X.drop(columns=['MoSold'], inplace=True)\n","X.head(3)"]},{"cell_type":"markdown","id":"63dc4c8e-4174-425b-a10b-5d15d89d8c89","metadata":{"id":"63dc4c8e-4174-425b-a10b-5d15d89d8c89"},"source":["---"]},{"cell_type":"markdown","id":"de5fdbbf-1ad8-44c0-8d1c-51c1f487e23f","metadata":{"tags":[],"id":"de5fdbbf-1ad8-44c0-8d1c-51c1f487e23f"},"source":["# ITERATIONS"]},{"cell_type":"markdown","id":"666a2bb0-8563-495f-8c5a-5665b0c99928","metadata":{"tags":[],"id":"666a2bb0-8563-495f-8c5a-5665b0c99928"},"source":["## Preprocessing"]},{"cell_type":"markdown","id":"f666a683-de49-4102-8762-2d51734b27dd","metadata":{"id":"f666a683-de49-4102-8762-2d51734b27dd"},"source":["### Encoding"]},{"cell_type":"code","execution_count":null,"id":"e80917c0-a7d6-46c5-803f-90f8dd613fe0","metadata":{"id":"e80917c0-a7d6-46c5-803f-90f8dd613fe0"},"outputs":[],"source":["# In the pipeline for ordinal preprocessing, we'll fill_value='missing' to replace NaN\n","\n","feat_ordinal_dict = {\n","    # considers \"missing\" as \"neutral\"\n","    \"BsmtCond\": ['missing', 'Po', 'Fa', 'TA', 'Gd'], #No Ex\n","    \"BsmtExposure\": ['missing', 'No', 'Mn', 'Av', 'Gd'], # \n","    \"BsmtFinType1\": ['missing', 'Unf', 'LwQ', 'Rec', 'BLQ', 'ALQ', 'GLQ'], #No Partial\n","    # BsmtFinType1: \"BLQ\": \"Below Average Living Quarters\" vs \"Rec\": \"Average Rec Room\",\n","    \"BsmtFinType2\": ['missing', 'Unf', 'LwQ', 'Rec', 'BLQ', 'ALQ', 'GLQ'], #\n","    \"BsmtQual\": ['missing', 'Fa', 'TA', 'Gd', 'Ex'], #No Po\n","    \"Electrical\": ['missing', 'Mix', 'FuseP', 'FuseF', 'FuseA', 'SBrkr'], #\n","    \"ExterCond\": ['missing', 'Po', 'Fa', 'TA', 'Gd', 'Ex'], #\n","    \"ExterQual\": ['missing', 'Fa', 'TA', 'Gd', 'Ex'], #\n","    \"Fence\": ['missing', 'MnWw', 'GdWo', 'MnPrv', 'GdPrv'], #Min Wood/Wire vs Min Privacy \n","    \"FireplaceQu\": ['missing', 'Po', 'Fa', 'TA', 'Gd', 'Ex'], # \n","    \"Functional\": ['missing', 'Sev', 'Maj2', 'Maj1', 'Mod', 'Min2', 'Min1', 'Typ'], #No Sal\n","    \"GarageCond\": ['missing', 'Po', 'Fa', 'TA', 'Gd', 'Ex'], #\n","    \"GarageFinish\": ['missing', 'Unf', 'RFn', 'Fin'], # \n","    \"GarageQual\": ['missing', 'Po', 'Fa', 'TA', 'Gd', 'Ex'], # \n","    \"HeatingQC\": ['missing', 'Po', 'Fa', 'TA', 'Gd', 'Ex'], #\n","    \"KitchenQual\": ['missing', 'Fa', 'TA', 'Gd', 'Ex'], #No Po\n","    \"LandContour\": ['missing', 'Low', 'Bnk', 'HLS', 'Lvl'], # \n","    \"LandSlope\": ['missing', 'Sev', 'Mod', 'Gtl'], #\n","    \"LotShape\": ['missing', 'IR3', 'IR2', 'IR1', 'Reg'], #\n","    \"PavedDrive\": ['missing', 'N', 'P', 'Y'],\n","    \"PoolQC\": ['missing', 'Fa', 'Gd', 'Ex'], #No Po\n","}\n","\n","# OrdinalEncoder input (list of string or array-like of intergers or strings)\n","feat_ordinal = sorted(feat_ordinal_dict.keys()) # sort alphabetically\n","\n","# OrdinalEncoder 'categories' parameter (a list of lists/array-like)\n","feat_ordinal_values_sorted = [feat_ordinal_dict[i] for i in feat_ordinal]"]},{"cell_type":"code","execution_count":null,"id":"c0ac7066-7e51-4f7a-a984-ea5215a89a0c","metadata":{"id":"c0ac7066-7e51-4f7a-a984-ea5215a89a0c"},"outputs":[],"source":["feat_nominal = sorted(list(set(X.select_dtypes(include=[\"object\"]).columns) - set(feat_ordinal))) # Convert to sorted list"]},{"cell_type":"code","execution_count":null,"id":"be093398-f6d6-4310-b425-dbacc7bb785b","metadata":{"id":"be093398-f6d6-4310-b425-dbacc7bb785b"},"outputs":[],"source":["feat_numerical = sorted(X.select_dtypes(include=[\"int64\", \"float64\"]).columns)"]},{"cell_type":"code","execution_count":null,"id":"0fe8da79-4934-4f02-b3cf-005a3f4fa5cd","metadata":{"id":"0fe8da79-4934-4f02-b3cf-005a3f4fa5cd","outputId":"0fd1222b-5fdf-4375-cab9-e4e0f53389be"},"outputs":[{"data":{"text/plain":["['MoSold']"]},"execution_count":65,"metadata":{},"output_type":"execute_result"}],"source":["feat_cyclical = sorted([\n","    \"MoSold\"  # Month Sold (MM)\n","])\n","feat_cyclical"]},{"cell_type":"markdown","id":"b47a5e38-5e4f-4970-af37-ed9b0db82e90","metadata":{"id":"b47a5e38-5e4f-4970-af37-ed9b0db82e90"},"source":["## Full Preprocessing Pipeline"]},{"cell_type":"code","execution_count":null,"id":"e8ce104f-2150-4b56-ae7b-209b04be8a61","metadata":{"id":"e8ce104f-2150-4b56-ae7b-209b04be8a61","outputId":"6b80eeab-628b-483d-ad51-1f600256dba7"},"outputs":[{"data":{"text/html":["<style>#sk-container-id-4 {color: black;background-color: white;}#sk-container-id-4 pre{padding: 0;}#sk-container-id-4 div.sk-toggleable {background-color: white;}#sk-container-id-4 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-4 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-4 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-4 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-4 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-4 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-4 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-4 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-4 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-4 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-4 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-4 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-4 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-4 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-4 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-4 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-4 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-4 div.sk-item {position: relative;z-index: 1;}#sk-container-id-4 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-4 div.sk-item::before, #sk-container-id-4 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-4 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-4 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-4 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-4 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-4 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-4 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-4 div.sk-label-container {text-align: center;}#sk-container-id-4 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-4 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-4\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>Pipeline(steps=[(&#x27;columntransformer&#x27;,\n","                 ColumnTransformer(transformers=[(&#x27;pipeline-1&#x27;,\n","                                                  Pipeline(steps=[(&#x27;knnimputer&#x27;,\n","                                                                   KNNImputer()),\n","                                                                  (&#x27;minmaxscaler&#x27;,\n","                                                                   MinMaxScaler())]),\n","                                                  [&#x27;1stFlrSF&#x27;, &#x27;2ndFlrSF&#x27;,\n","                                                   &#x27;3SsnPorch&#x27;, &#x27;BedroomAbvGr&#x27;,\n","                                                   &#x27;BsmtFinSF1&#x27;, &#x27;BsmtFinSF2&#x27;,\n","                                                   &#x27;BsmtFullBath&#x27;,\n","                                                   &#x27;BsmtHalfBath&#x27;, &#x27;BsmtUnfSF&#x27;,\n","                                                   &#x27;EnclosedPorch&#x27;,\n","                                                   &#x27;Fireplaces&#x27;, &#x27;FullBath&#x27;,\n","                                                   &#x27;GarageArea&#x27;, &#x27;GarageCars&#x27;,\n","                                                   &#x27;Gara...\n","                                                                   SimpleImputer(strategy=&#x27;most_frequent&#x27;)),\n","                                                                  (&#x27;onehotencoder&#x27;,\n","                                                                   OneHotEncoder(handle_unknown=&#x27;ignore&#x27;))]),\n","                                                  [&#x27;Alley&#x27;, &#x27;BldgType&#x27;,\n","                                                   &#x27;CentralAir&#x27;, &#x27;Condition1&#x27;,\n","                                                   &#x27;Condition2&#x27;, &#x27;Exterior1st&#x27;,\n","                                                   &#x27;Exterior2nd&#x27;, &#x27;Foundation&#x27;,\n","                                                   &#x27;GarageType&#x27;, &#x27;Heating&#x27;,\n","                                                   &#x27;HouseStyle&#x27;, &#x27;LotConfig&#x27;,\n","                                                   &#x27;MSZoning&#x27;, &#x27;MasVnrType&#x27;,\n","                                                   &#x27;MiscFeature&#x27;,\n","                                                   &#x27;Neighborhood&#x27;, &#x27;RoofMatl&#x27;,\n","                                                   &#x27;RoofStyle&#x27;, &#x27;SaleCondition&#x27;,\n","                                                   &#x27;SaleType&#x27;, &#x27;Street&#x27;,\n","                                                   &#x27;Utilities&#x27;])]))])</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-37\" type=\"checkbox\" ><label for=\"sk-estimator-id-37\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">Pipeline</label><div class=\"sk-toggleable__content\"><pre>Pipeline(steps=[(&#x27;columntransformer&#x27;,\n","                 ColumnTransformer(transformers=[(&#x27;pipeline-1&#x27;,\n","                                                  Pipeline(steps=[(&#x27;knnimputer&#x27;,\n","                                                                   KNNImputer()),\n","                                                                  (&#x27;minmaxscaler&#x27;,\n","                                                                   MinMaxScaler())]),\n","                                                  [&#x27;1stFlrSF&#x27;, &#x27;2ndFlrSF&#x27;,\n","                                                   &#x27;3SsnPorch&#x27;, &#x27;BedroomAbvGr&#x27;,\n","                                                   &#x27;BsmtFinSF1&#x27;, &#x27;BsmtFinSF2&#x27;,\n","                                                   &#x27;BsmtFullBath&#x27;,\n","                                                   &#x27;BsmtHalfBath&#x27;, &#x27;BsmtUnfSF&#x27;,\n","                                                   &#x27;EnclosedPorch&#x27;,\n","                                                   &#x27;Fireplaces&#x27;, &#x27;FullBath&#x27;,\n","                                                   &#x27;GarageArea&#x27;, &#x27;GarageCars&#x27;,\n","                                                   &#x27;Gara...\n","                                                                   SimpleImputer(strategy=&#x27;most_frequent&#x27;)),\n","                                                                  (&#x27;onehotencoder&#x27;,\n","                                                                   OneHotEncoder(handle_unknown=&#x27;ignore&#x27;))]),\n","                                                  [&#x27;Alley&#x27;, &#x27;BldgType&#x27;,\n","                                                   &#x27;CentralAir&#x27;, &#x27;Condition1&#x27;,\n","                                                   &#x27;Condition2&#x27;, &#x27;Exterior1st&#x27;,\n","                                                   &#x27;Exterior2nd&#x27;, &#x27;Foundation&#x27;,\n","                                                   &#x27;GarageType&#x27;, &#x27;Heating&#x27;,\n","                                                   &#x27;HouseStyle&#x27;, &#x27;LotConfig&#x27;,\n","                                                   &#x27;MSZoning&#x27;, &#x27;MasVnrType&#x27;,\n","                                                   &#x27;MiscFeature&#x27;,\n","                                                   &#x27;Neighborhood&#x27;, &#x27;RoofMatl&#x27;,\n","                                                   &#x27;RoofStyle&#x27;, &#x27;SaleCondition&#x27;,\n","                                                   &#x27;SaleType&#x27;, &#x27;Street&#x27;,\n","                                                   &#x27;Utilities&#x27;])]))])</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-38\" type=\"checkbox\" ><label for=\"sk-estimator-id-38\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">columntransformer: ColumnTransformer</label><div class=\"sk-toggleable__content\"><pre>ColumnTransformer(transformers=[(&#x27;pipeline-1&#x27;,\n","                                 Pipeline(steps=[(&#x27;knnimputer&#x27;, KNNImputer()),\n","                                                 (&#x27;minmaxscaler&#x27;,\n","                                                  MinMaxScaler())]),\n","                                 [&#x27;1stFlrSF&#x27;, &#x27;2ndFlrSF&#x27;, &#x27;3SsnPorch&#x27;,\n","                                  &#x27;BedroomAbvGr&#x27;, &#x27;BsmtFinSF1&#x27;, &#x27;BsmtFinSF2&#x27;,\n","                                  &#x27;BsmtFullBath&#x27;, &#x27;BsmtHalfBath&#x27;, &#x27;BsmtUnfSF&#x27;,\n","                                  &#x27;EnclosedPorch&#x27;, &#x27;Fireplaces&#x27;, &#x27;FullBath&#x27;,\n","                                  &#x27;GarageArea&#x27;, &#x27;GarageCars&#x27;, &#x27;GarageYrBlt&#x27;,\n","                                  &#x27;GrLivArea&#x27;, &#x27;HalfBath&#x27;, &#x27;Kitc...\n","                                                  SimpleImputer(strategy=&#x27;most_frequent&#x27;)),\n","                                                 (&#x27;onehotencoder&#x27;,\n","                                                  OneHotEncoder(handle_unknown=&#x27;ignore&#x27;))]),\n","                                 [&#x27;Alley&#x27;, &#x27;BldgType&#x27;, &#x27;CentralAir&#x27;,\n","                                  &#x27;Condition1&#x27;, &#x27;Condition2&#x27;, &#x27;Exterior1st&#x27;,\n","                                  &#x27;Exterior2nd&#x27;, &#x27;Foundation&#x27;, &#x27;GarageType&#x27;,\n","                                  &#x27;Heating&#x27;, &#x27;HouseStyle&#x27;, &#x27;LotConfig&#x27;,\n","                                  &#x27;MSZoning&#x27;, &#x27;MasVnrType&#x27;, &#x27;MiscFeature&#x27;,\n","                                  &#x27;Neighborhood&#x27;, &#x27;RoofMatl&#x27;, &#x27;RoofStyle&#x27;,\n","                                  &#x27;SaleCondition&#x27;, &#x27;SaleType&#x27;, &#x27;Street&#x27;,\n","                                  &#x27;Utilities&#x27;])])</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-39\" type=\"checkbox\" ><label for=\"sk-estimator-id-39\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">pipeline-1</label><div class=\"sk-toggleable__content\"><pre>[&#x27;1stFlrSF&#x27;, &#x27;2ndFlrSF&#x27;, &#x27;3SsnPorch&#x27;, &#x27;BedroomAbvGr&#x27;, &#x27;BsmtFinSF1&#x27;, &#x27;BsmtFinSF2&#x27;, &#x27;BsmtFullBath&#x27;, &#x27;BsmtHalfBath&#x27;, &#x27;BsmtUnfSF&#x27;, &#x27;EnclosedPorch&#x27;, &#x27;Fireplaces&#x27;, &#x27;FullBath&#x27;, &#x27;GarageArea&#x27;, &#x27;GarageCars&#x27;, &#x27;GarageYrBlt&#x27;, &#x27;GrLivArea&#x27;, &#x27;HalfBath&#x27;, &#x27;KitchenAbvGr&#x27;, &#x27;LotArea&#x27;, &#x27;LotFrontage&#x27;, &#x27;LowQualFinSF&#x27;, &#x27;MSSubClass&#x27;, &#x27;MasVnrArea&#x27;, &#x27;MiscVal&#x27;, &#x27;MoSold_cos&#x27;, &#x27;MoSold_sin&#x27;, &#x27;OpenPorchSF&#x27;, &#x27;OverallCond&#x27;, &#x27;OverallQual&#x27;, &#x27;PoolArea&#x27;, &#x27;ScreenPorch&#x27;, &#x27;TotRmsAbvGrd&#x27;, &#x27;TotalBsmtSF&#x27;, &#x27;WoodDeckSF&#x27;, &#x27;YearBuilt&#x27;, &#x27;YearRemodAdd&#x27;, &#x27;YrSold&#x27;]</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-40\" type=\"checkbox\" ><label for=\"sk-estimator-id-40\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">KNNImputer</label><div class=\"sk-toggleable__content\"><pre>KNNImputer()</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-41\" type=\"checkbox\" ><label for=\"sk-estimator-id-41\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">MinMaxScaler</label><div class=\"sk-toggleable__content\"><pre>MinMaxScaler()</pre></div></div></div></div></div></div></div></div><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-42\" type=\"checkbox\" ><label for=\"sk-estimator-id-42\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">pipeline-2</label><div class=\"sk-toggleable__content\"><pre>[&#x27;BsmtCond&#x27;, &#x27;BsmtExposure&#x27;, &#x27;BsmtFinType1&#x27;, &#x27;BsmtFinType2&#x27;, &#x27;BsmtQual&#x27;, &#x27;Electrical&#x27;, &#x27;ExterCond&#x27;, &#x27;ExterQual&#x27;, &#x27;Fence&#x27;, &#x27;FireplaceQu&#x27;, &#x27;Functional&#x27;, &#x27;GarageCond&#x27;, &#x27;GarageFinish&#x27;, &#x27;GarageQual&#x27;, &#x27;HeatingQC&#x27;, &#x27;KitchenQual&#x27;, &#x27;LandContour&#x27;, &#x27;LandSlope&#x27;, &#x27;LotShape&#x27;, &#x27;PavedDrive&#x27;, &#x27;PoolQC&#x27;]</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-43\" type=\"checkbox\" ><label for=\"sk-estimator-id-43\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">SimpleImputer</label><div class=\"sk-toggleable__content\"><pre>SimpleImputer(fill_value=&#x27;missing&#x27;, strategy=&#x27;constant&#x27;)</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-44\" type=\"checkbox\" ><label for=\"sk-estimator-id-44\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">OrdinalEncoder</label><div class=\"sk-toggleable__content\"><pre>OrdinalEncoder(categories=[[&#x27;missing&#x27;, &#x27;Po&#x27;, &#x27;Fa&#x27;, &#x27;TA&#x27;, &#x27;Gd&#x27;],\n","                           [&#x27;missing&#x27;, &#x27;No&#x27;, &#x27;Mn&#x27;, &#x27;Av&#x27;, &#x27;Gd&#x27;],\n","                           [&#x27;missing&#x27;, &#x27;Unf&#x27;, &#x27;LwQ&#x27;, &#x27;Rec&#x27;, &#x27;BLQ&#x27;, &#x27;ALQ&#x27;,\n","                            &#x27;GLQ&#x27;],\n","                           [&#x27;missing&#x27;, &#x27;Unf&#x27;, &#x27;LwQ&#x27;, &#x27;Rec&#x27;, &#x27;BLQ&#x27;, &#x27;ALQ&#x27;,\n","                            &#x27;GLQ&#x27;],\n","                           [&#x27;missing&#x27;, &#x27;Fa&#x27;, &#x27;TA&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;],\n","                           [&#x27;missing&#x27;, &#x27;Mix&#x27;, &#x27;FuseP&#x27;, &#x27;FuseF&#x27;, &#x27;FuseA&#x27;,\n","                            &#x27;SBrkr&#x27;],\n","                           [&#x27;missing&#x27;, &#x27;Po&#x27;, &#x27;Fa&#x27;, &#x27;TA&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;],\n","                           [&#x27;missing&#x27;, &#x27;Fa&#x27;, &#x27;TA&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;],\n","                           [&#x27;missing&#x27;, &#x27;...\n","                           [&#x27;missing&#x27;, &#x27;Unf&#x27;, &#x27;RFn&#x27;, &#x27;Fin&#x27;],\n","                           [&#x27;missing&#x27;, &#x27;Po&#x27;, &#x27;Fa&#x27;, &#x27;TA&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;],\n","                           [&#x27;missing&#x27;, &#x27;Po&#x27;, &#x27;Fa&#x27;, &#x27;TA&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;],\n","                           [&#x27;missing&#x27;, &#x27;Fa&#x27;, &#x27;TA&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;],\n","                           [&#x27;missing&#x27;, &#x27;Low&#x27;, &#x27;Bnk&#x27;, &#x27;HLS&#x27;, &#x27;Lvl&#x27;],\n","                           [&#x27;missing&#x27;, &#x27;Sev&#x27;, &#x27;Mod&#x27;, &#x27;Gtl&#x27;],\n","                           [&#x27;missing&#x27;, &#x27;IR3&#x27;, &#x27;IR2&#x27;, &#x27;IR1&#x27;, &#x27;Reg&#x27;],\n","                           [&#x27;missing&#x27;, &#x27;N&#x27;, &#x27;P&#x27;, &#x27;Y&#x27;],\n","                           [&#x27;missing&#x27;, &#x27;Fa&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;]],\n","               dtype=&lt;class &#x27;numpy.int64&#x27;&gt;, handle_unknown=&#x27;use_encoded_value&#x27;,\n","               unknown_value=-1)</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-45\" type=\"checkbox\" ><label for=\"sk-estimator-id-45\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">MinMaxScaler</label><div class=\"sk-toggleable__content\"><pre>MinMaxScaler()</pre></div></div></div></div></div></div></div></div><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-46\" type=\"checkbox\" ><label for=\"sk-estimator-id-46\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">pipeline-3</label><div class=\"sk-toggleable__content\"><pre>[&#x27;Alley&#x27;, &#x27;BldgType&#x27;, &#x27;CentralAir&#x27;, &#x27;Condition1&#x27;, &#x27;Condition2&#x27;, &#x27;Exterior1st&#x27;, &#x27;Exterior2nd&#x27;, &#x27;Foundation&#x27;, &#x27;GarageType&#x27;, &#x27;Heating&#x27;, &#x27;HouseStyle&#x27;, &#x27;LotConfig&#x27;, &#x27;MSZoning&#x27;, &#x27;MasVnrType&#x27;, &#x27;MiscFeature&#x27;, &#x27;Neighborhood&#x27;, &#x27;RoofMatl&#x27;, &#x27;RoofStyle&#x27;, &#x27;SaleCondition&#x27;, &#x27;SaleType&#x27;, &#x27;Street&#x27;, &#x27;Utilities&#x27;]</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-47\" type=\"checkbox\" ><label for=\"sk-estimator-id-47\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">SimpleImputer</label><div class=\"sk-toggleable__content\"><pre>SimpleImputer(strategy=&#x27;most_frequent&#x27;)</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-48\" type=\"checkbox\" ><label for=\"sk-estimator-id-48\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">OneHotEncoder</label><div class=\"sk-toggleable__content\"><pre>OneHotEncoder(handle_unknown=&#x27;ignore&#x27;)</pre></div></div></div></div></div></div></div></div></div></div></div></div></div></div>"],"text/plain":["Pipeline(steps=[('columntransformer',\n","                 ColumnTransformer(transformers=[('pipeline-1',\n","                                                  Pipeline(steps=[('knnimputer',\n","                                                                   KNNImputer()),\n","                                                                  ('minmaxscaler',\n","                                                                   MinMaxScaler())]),\n","                                                  ['1stFlrSF', '2ndFlrSF',\n","                                                   '3SsnPorch', 'BedroomAbvGr',\n","                                                   'BsmtFinSF1', 'BsmtFinSF2',\n","                                                   'BsmtFullBath',\n","                                                   'BsmtHalfBath', 'BsmtUnfSF',\n","                                                   'EnclosedPorch',\n","                                                   'Fireplaces', 'FullBath',\n","                                                   'GarageArea', 'GarageCars',\n","                                                   'Gara...\n","                                                                   SimpleImputer(strategy='most_frequent')),\n","                                                                  ('onehotencoder',\n","                                                                   OneHotEncoder(handle_unknown='ignore'))]),\n","                                                  ['Alley', 'BldgType',\n","                                                   'CentralAir', 'Condition1',\n","                                                   'Condition2', 'Exterior1st',\n","                                                   'Exterior2nd', 'Foundation',\n","                                                   'GarageType', 'Heating',\n","                                                   'HouseStyle', 'LotConfig',\n","                                                   'MSZoning', 'MasVnrType',\n","                                                   'MiscFeature',\n","                                                   'Neighborhood', 'RoofMatl',\n","                                                   'RoofStyle', 'SaleCondition',\n","                                                   'SaleType', 'Street',\n","                                                   'Utilities'])]))])"]},"execution_count":66,"metadata":{},"output_type":"execute_result"}],"source":["encoder_ordinal = OrdinalEncoder(\n","    categories=feat_ordinal_values_sorted,\n","    dtype= np.int64,\n","    handle_unknown=\"use_encoded_value\",\n","    unknown_value=-1 # Considers unknown values as worse than \"missing\"\n",")\n","\n","preproc_ordinal = make_pipeline(\n","    SimpleImputer(strategy=\"constant\", fill_value=\"missing\"),\n","    encoder_ordinal,\n","    MinMaxScaler()\n",")\n","\n","preproc_numerical = make_pipeline(\n","    KNNImputer(),\n","    MinMaxScaler()\n",")\n","\n","preproc_nominal = make_pipeline(\n","    SimpleImputer(strategy=\"most_frequent\"),\n","    OneHotEncoder(handle_unknown=\"ignore\") # To ignore error message\n",")\n","\n","preproc_transformer = make_column_transformer(\n","    (preproc_numerical, feat_numerical),\n","    (preproc_ordinal, feat_ordinal),\n","    (preproc_nominal, feat_nominal),\n","    remainder=\"drop\")\n","\n","# preproc_selector = SelectPercentile(\n","#     mutual_info_regression,\n","#     percentile=25, # keep only xx% of all features\n","# )\n","\n","preproc = make_pipeline(\n","    preproc_transformer \n","    # preproc_selector\n",")\n","preproc"]},{"cell_type":"markdown","id":"c6db2bb5-50e2-4bb0-a99c-d53a199bc4c6","metadata":{"id":"c6db2bb5-50e2-4bb0-a99c-d53a199bc4c6"},"source":["**Encode whole dataset with the full preprocessing pipeline**"]},{"cell_type":"code","execution_count":null,"id":"f324838f-bcf5-43c0-9cf1-fe3edf84b6a0","metadata":{"id":"f324838f-bcf5-43c0-9cf1-fe3edf84b6a0","outputId":"d1f75625-4d17-43ab-c0be-bc5e3ee2fe47"},"outputs":[{"data":{"text/plain":["(1460, 217)"]},"execution_count":67,"metadata":{},"output_type":"execute_result"}],"source":["pd.DataFrame(preproc.fit_transform(X, y_log)).shape"]},{"cell_type":"code","execution_count":null,"id":"4b8e5d87-272a-4ed3-ad20-f81d20faf025","metadata":{"id":"4b8e5d87-272a-4ed3-ad20-f81d20faf025","outputId":"7e780970-40c4-40ab-b096-f21b80861215"},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>0</th>\n","      <th>1</th>\n","      <th>2</th>\n","      <th>3</th>\n","      <th>4</th>\n","      <th>5</th>\n","      <th>6</th>\n","      <th>7</th>\n","      <th>8</th>\n","      <th>9</th>\n","      <th>...</th>\n","      <th>207</th>\n","      <th>208</th>\n","      <th>209</th>\n","      <th>210</th>\n","      <th>211</th>\n","      <th>212</th>\n","      <th>213</th>\n","      <th>214</th>\n","      <th>215</th>\n","      <th>216</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>0.119780</td>\n","      <td>0.413559</td>\n","      <td>0.0</td>\n","      <td>0.375</td>\n","      <td>0.125089</td>\n","      <td>0.0</td>\n","      <td>0.333333</td>\n","      <td>0.0</td>\n","      <td>0.064212</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>1.0</td>\n","      <td>0.0</td>\n","      <td>1.0</td>\n","      <td>1.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>0.212942</td>\n","      <td>0.000000</td>\n","      <td>0.0</td>\n","      <td>0.375</td>\n","      <td>0.173281</td>\n","      <td>0.0</td>\n","      <td>0.000000</td>\n","      <td>0.5</td>\n","      <td>0.121575</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>1.0</td>\n","      <td>0.0</td>\n","      <td>1.0</td>\n","      <td>1.0</td>\n","      <td>0.0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>2 rows × 217 columns</p>\n","</div>"],"text/plain":["        0         1    2      3         4    5         6    7         8    \\\n","0  0.119780  0.413559  0.0  0.375  0.125089  0.0  0.333333  0.0  0.064212   \n","1  0.212942  0.000000  0.0  0.375  0.173281  0.0  0.000000  0.5  0.121575   \n","\n","   9    ...  207  208  209  210  211  212  213  214  215  216  \n","0  0.0  ...  0.0  0.0  0.0  0.0  0.0  1.0  0.0  1.0  1.0  0.0  \n","1  0.0  ...  0.0  0.0  0.0  0.0  0.0  1.0  0.0  1.0  1.0  0.0  \n","\n","[2 rows x 217 columns]"]},"execution_count":68,"metadata":{},"output_type":"execute_result"}],"source":["pd.DataFrame(preproc.fit_transform(X, y_log)).head(2)"]},{"cell_type":"markdown","id":"d478fd55-d808-48c0-87f3-e9240c20dcec","metadata":{"id":"d478fd55-d808-48c0-87f3-e9240c20dcec"},"source":["# TESTING MODELS"]},{"cell_type":"code","execution_count":null,"id":"479232e3-3231-44ed-8c86-599aa6c54383","metadata":{"id":"479232e3-3231-44ed-8c86-599aa6c54383"},"outputs":[],"source":["allow_grid_searching = True # use True to activate girdsearch, False to deactive\n","\n","# Cache the preprocessing step of the pipeline\n","from tempfile import mkdtemp\n","from shutil import rmtree\n","cachedir = mkdtemp()"]},{"cell_type":"markdown","id":"dfc64be8-3a6a-48f8-abaa-af53bf019066","metadata":{"tags":[],"id":"dfc64be8-3a6a-48f8-abaa-af53bf019066"},"source":["## Linear Models (Lasso, Ridge, ElasticNet, SGDRegressor...)"]},{"cell_type":"code","execution_count":null,"id":"bf2318aa-7039-4a2a-b785-259b0b76bba9","metadata":{"id":"bf2318aa-7039-4a2a-b785-259b0b76bba9"},"outputs":[],"source":["from sklearn.linear_model import Ridge, Lasso, LinearRegression"]},{"cell_type":"code","execution_count":null,"id":"f46b0814-121c-4a1b-91a4-45577a12bead","metadata":{"id":"f46b0814-121c-4a1b-91a4-45577a12bead","outputId":"f5c61503-f047-40f8-e8a1-dcd51d70bc44"},"outputs":[{"data":{"text/plain":["1412187430.1168573"]},"execution_count":94,"metadata":{},"output_type":"execute_result"}],"source":["model_linear_regression = LinearRegression()\n","pipe_linreg = make_pipeline(preproc, model_linear_regression, memory=cachedir)\n","cross_val_score(pipe_linreg, X, y_log, cv=10, scoring=rmse).mean()"]},{"cell_type":"code","execution_count":null,"id":"91e7f28f-2378-41c9-829c-8ad6adb02fd6","metadata":{"id":"91e7f28f-2378-41c9-829c-8ad6adb02fd6","outputId":"7646c02d-bf5e-4e7a-d304-b95baeb19558"},"outputs":[{"data":{"text/plain":["0.13855378989109343"]},"execution_count":96,"metadata":{},"output_type":"execute_result"}],"source":["model_ridge = Ridge()\n","pipe_ridge = make_pipeline(preproc, model_ridge, memory=cachedir)\n","cross_val_score(pipe_ridge, X, y_log, cv=5, scoring=rmse).mean()"]},{"cell_type":"code","execution_count":null,"id":"9c8bfd93-fec6-48f4-b683-fbf0f4cf1a33","metadata":{"id":"9c8bfd93-fec6-48f4-b683-fbf0f4cf1a33","outputId":"e8f4a0b4-28cc-4e4a-8cb2-6d27f723006f"},"outputs":[{"name":"stdout","output_type":"stream","text":["Fitting 5 folds for each of 20 candidates, totalling 100 fits\n","Best params:  {'ridge__alpha': 2.0}\n","Best RMSE (of log):  0.13734118600507728\n"]}],"source":["# GridSearch the Ridge regularization\n","if allow_grid_searching:\n","    param_grid =  {'ridge__alpha': np.linspace(0.5,2,num=20)}\n","    search_ridge = GridSearchCV(pipe_ridge, param_grid=param_grid, \n","                              cv=5, n_jobs=-1, verbose=2, scoring=rmse_neg)\n","\n","    search_ridge.fit(X, y_log);\n","    # print('Best estimator: ', search_ridge.best_estimator_)\n","    print('Best params: ', search_ridge.best_params_)\n","    print('Best RMSE (of log): ', -search_ridge.best_score_) # Inverse to turn into positive"]},{"cell_type":"markdown","id":"11a61e11-cc59-4da0-b478-dce42444ac38","metadata":{"id":"11a61e11-cc59-4da0-b478-dce42444ac38"},"source":["## KNN"]},{"cell_type":"code","execution_count":null,"id":"21fa89b2-8e51-481b-aebe-925a7b188e19","metadata":{"id":"21fa89b2-8e51-481b-aebe-925a7b188e19","outputId":"6c819a36-833b-4a60-f715-463e2511d24d"},"outputs":[{"name":"stdout","output_type":"stream","text":["Best RMSE (of log):  0.20831550684405498\n"]}],"source":["from sklearn.neighbors import KNeighborsRegressor\n","\n","model_knn = KNeighborsRegressor()\n","pipe_knn = make_pipeline(preproc, model_knn)\n","scores = cross_val_score(pipe_knn, X, y_log, cv=5, scoring=rmse, verbose=False)\n","print('Best RMSE (of log): ', scores.mean())"]},{"cell_type":"code","execution_count":null,"id":"72b88d1f-a49d-4a00-a935-7933bb0bcd22","metadata":{"id":"72b88d1f-a49d-4a00-a935-7933bb0bcd22","outputId":"36837c6a-b009-49aa-c27d-b010c384a091"},"outputs":[{"name":"stdout","output_type":"stream","text":["Fitting 3 folds for each of 11 candidates, totalling 33 fits\n","Best params:  {'kneighborsregressor__n_neighbors': 7}\n","Best RMSE (of log):  0.21030598163719624\n"]}],"source":["# GridSearch the KNN\n","if allow_grid_searching:\n","    param_grid =  {'kneighborsregressor__n_neighbors': [3,4,5,6,7,8,9,10,15,20,30]}\n","    search_knn = GridSearchCV(pipe_knn, param_grid=param_grid, \n","                              cv=3, n_jobs=-1, verbose=2, scoring=rmse_neg)\n","\n","    search_knn.fit(X, y_log);\n","    print('Best params: ', search_knn.best_params_)\n","    print('Best RMSE (of log): ', -search_knn.best_score_) # Inverse to turn into positive"]},{"cell_type":"markdown","id":"60bf0b3e-5ae9-4b1a-b808-c0f873853222","metadata":{"id":"60bf0b3e-5ae9-4b1a-b808-c0f873853222"},"source":["## SVM"]},{"cell_type":"code","execution_count":null,"id":"537c01e4-ad25-48ca-9df0-f6574b6ba921","metadata":{"id":"537c01e4-ad25-48ca-9df0-f6574b6ba921","outputId":"9c5441c8-5fd7-4a8d-e167-10776d1c1b9c"},"outputs":[{"data":{"text/plain":["0.1440302176420198"]},"execution_count":100,"metadata":{},"output_type":"execute_result"}],"source":["model_svr_l = SVR(kernel='linear')\n","pipe_svm = make_pipeline(preproc, model_svr_l, memory=cachedir)\n","cross_val_score(pipe_svm, X, y_log, cv=5, scoring=rmse).mean()"]},{"cell_type":"code","execution_count":null,"id":"85360768-7900-4b0c-8af2-20426ef6a40b","metadata":{"id":"85360768-7900-4b0c-8af2-20426ef6a40b","outputId":"a97a5aea-eee9-49bd-c5d7-a078331d2d40"},"outputs":[{"data":{"text/plain":["0.13400913508010154"]},"execution_count":101,"metadata":{},"output_type":"execute_result"}],"source":["model_svr_rbf = SVR(kernel='rbf', C = 10)\n","pipe_svm_rbf = make_pipeline(preproc, model_svr_rbf, memory=cachedir)\n","cross_val_score(pipe_svm_rbf, X, y_log, cv=5, scoring=rmse).mean()"]},{"cell_type":"code","execution_count":null,"id":"e3f43dd6-ab37-4c68-b11e-53bfd68c851d","metadata":{"id":"e3f43dd6-ab37-4c68-b11e-53bfd68c851d","outputId":"7254fc03-5118-436c-a6e6-1eafe2a028ab"},"outputs":[{"name":"stdout","output_type":"stream","text":["Best estimator:  Pipeline(memory='/var/folders/18/xyw20kr11mz0qymy_10fss5w0000gn/T/tmp4ie2iwo_',\n","         steps=[('pipeline',\n","                 Pipeline(steps=[('columntransformer',\n","                                  ColumnTransformer(transformers=[('pipeline-1',\n","                                                                   Pipeline(steps=[('knnimputer',\n","                                                                                    KNNImputer()),\n","                                                                                   ('minmaxscaler',\n","                                                                                    MinMaxScaler())]),\n","                                                                   ['1stFlrSF',\n","                                                                    '2ndFlrSF',\n","                                                                    '3SsnPorch',\n","                                                                    'BedroomAbvGr',\n","                                                                    'BsmtFinSF1',\n","                                                                    'BsmtFinSF2',\n","                                                                    'BsmtFullBath',...\n","                                                                                    OneHotEncoder(handle_unknown='ignore'))]),\n","                                                                   ['Alley',\n","                                                                    'BldgType',\n","                                                                    'CentralAir',\n","                                                                    'Condition1',\n","                                                                    'Condition2',\n","                                                                    'Exterior1st',\n","                                                                    'Exterior2nd',\n","                                                                    'Foundation',\n","                                                                    'GarageType',\n","                                                                    'Heating',\n","                                                                    'HouseStyle',\n","                                                                    'LotConfig',\n","                                                                    'MSZoning',\n","                                                                    'MasVnrType',\n","                                                                    'MiscFeature',\n","                                                                    'Neighborhood',\n","                                                                    'RoofMatl',\n","                                                                    'RoofStyle',\n","                                                                    'SaleCondition',\n","                                                                    'SaleType',\n","                                                                    'Street',\n","                                                                    'Utilities'])]))])),\n","                ('svr', SVR(C=1, epsilon=0.01))])\n","Best params:  {'svr__C': 1, 'svr__coef0': 0.0, 'svr__epsilon': 0.01}\n","Best RMSE (of log):  0.12514530788743156\n"]}],"source":["# GridSearch\n","if allow_grid_searching:\n","    param_grid =  {'svr__C': [0.5, 0.7, 1, 2, 5, 10],\n","                  'svr__epsilon': [0.01, 0.05, 0.1, 0.2, 0.5],\n","                   'svr__coef0': [0.0, 0.1, 0.5,1],\n","                  }\n","    search_svm_rbf = GridSearchCV(pipe_svm_rbf, param_grid=param_grid, \n","                              cv=5, n_jobs=-1, verbose=False, scoring=rmse_neg)\n","\n","    search_svm_rbf.fit(X, y_log);\n","    print('Best estimator: ', search_svm_rbf.best_estimator_)\n","    print('Best params: ', search_svm_rbf.best_params_)\n","    print('Best RMSE (of log): ', -search_svm_rbf.best_score_) # Inverse to turn into positive"]},{"cell_type":"markdown","id":"fadc5d2f-37db-445d-9a6c-2c44ad5ca9e2","metadata":{"id":"fadc5d2f-37db-445d-9a6c-2c44ad5ca9e2"},"source":["## Trees"]},{"cell_type":"code","execution_count":null,"id":"53f280a4-1e7c-4dbc-b887-1f7e0202c2d9","metadata":{"id":"53f280a4-1e7c-4dbc-b887-1f7e0202c2d9","outputId":"03086f71-ad61-4993-fc03-176abedbbb67"},"outputs":[{"data":{"text/plain":["0.18931479383895605"]},"execution_count":103,"metadata":{},"output_type":"execute_result"}],"source":["from sklearn.tree import DecisionTreeRegressor\n","\n","model_decision_tree_regressor = DecisionTreeRegressor(max_depth=50, min_samples_leaf=20)\n","pipe_decision_tree_regressor = make_pipeline(preproc, model_decision_tree_regressor, memory=cachedir)\n","score = cross_val_score(pipe_decision_tree_regressor, X, y_log, cv=5, scoring=rmse)\n","score.mean()"]},{"cell_type":"markdown","id":"8711144c-d90a-449f-a31d-6ed0f2224b97","metadata":{"id":"8711144c-d90a-449f-a31d-6ed0f2224b97"},"source":["## Random Forest"]},{"cell_type":"code","execution_count":null,"id":"59e3d0a0-1dbf-4ba7-95f2-baf901108e2c","metadata":{"id":"59e3d0a0-1dbf-4ba7-95f2-baf901108e2c","outputId":"247a36c0-4652-4b57-c508-af71dea21c27"},"outputs":[{"data":{"text/plain":["0.16224793725790768"]},"execution_count":104,"metadata":{},"output_type":"execute_result"}],"source":["from sklearn.ensemble import RandomForestRegressor\n","\n","model_rf = RandomForestRegressor(max_depth=50,min_samples_leaf=20)\n","pipe_rf = make_pipeline(preproc, model_rf, memory=cachedir)\n","score = cross_val_score(pipe_rf, X, y_log, cv=5, scoring=rmse)\n","score.mean()"]},{"cell_type":"markdown","id":"60ccdcc3-3381-48b2-85fc-ffc411149446","metadata":{"id":"60ccdcc3-3381-48b2-85fc-ffc411149446"},"source":["## Boosted Trees"]},{"cell_type":"code","execution_count":null,"id":"ce7b7b88-c7c3-4c1f-87af-b2bacfbea43b","metadata":{"id":"ce7b7b88-c7c3-4c1f-87af-b2bacfbea43b","outputId":"a06a2da4-356c-4d46-d217-2aa2c6bf934a"},"outputs":[{"data":{"text/plain":["0.1402346794225529"]},"execution_count":105,"metadata":{},"output_type":"execute_result"}],"source":["from sklearn.ensemble import AdaBoostRegressor\n","\n","model_ada = AdaBoostRegressor(\n","    base_estimator=DecisionTreeRegressor(max_depth=None))\n","\n","pipe_ada = make_pipeline(preproc, model_ada, memory=cachedir)\n","score = cross_val_score(pipe_ada, X, y_log, cv=5, scoring=rmse)\n","score.mean()"]},{"cell_type":"code","execution_count":null,"id":"7fe41887-600b-4d31-a048-5370a17566e7","metadata":{"id":"7fe41887-600b-4d31-a048-5370a17566e7","outputId":"f6a3c270-8308-4daf-daad-50e2debd269a"},"outputs":[{"data":{"text/plain":["0.12639867093332918"]},"execution_count":106,"metadata":{},"output_type":"execute_result"}],"source":["from sklearn.ensemble import GradientBoostingRegressor\n","\n","model_xgboost = GradientBoostingRegressor(n_estimators=100, verbose=0)\n","pipe_gb = make_pipeline(preproc, model_xgboost, memory=cachedir)\n","score = cross_val_score(pipe_gb, X, y_log, cv=5, scoring=rmse)\n","score.mean()"]},{"cell_type":"code","execution_count":null,"id":"717bfff0-0dec-47ff-bd75-a8b2c82e9cdf","metadata":{"tags":[],"id":"717bfff0-0dec-47ff-bd75-a8b2c82e9cdf","outputId":"bdc065b2-cb0e-45db-a7ac-95ca1de84ee2"},"outputs":[{"name":"stdout","output_type":"stream","text":["Best params:  {'gradientboostingregressor__learning_rate': 0.0998865672595152, 'gradientboostingregressor__loss': 'huber', 'gradientboostingregressor__max_depth': 3, 'gradientboostingregressor__max_features': 64, 'gradientboostingregressor__min_samples_split': 4, 'gradientboostingregressor__n_estimators': 244, 'gradientboostingregressor__subsample': 0.95}\n","RMSE (of log):  0.12114903151024078\n"]},{"data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAZEAAAEHCAYAAABvHnsJAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAxkUlEQVR4nO3deZhcVZ3/8fe3QocOWSAbSQBDE4gGEI3YYlCDCIhMBAEXFB0MKpPgFhUdYX7MM6IzLujIaFwBB4kOYtgCATEqcaFVQDsSIQEhIQsi2QNkbQip7++Pcypd6VRV13Jr6/68nqeerrp1q+65VdXne89u7o6IiEg5UvVOgIiINC8FERERKZuCiIiIlE1BREREyqYgIiIiZduv3gmopVGjRnlbW1u9kyEi0lQWLVq00d1H53quXwWRtrY2Ojs7650MEZGmYmar8z2n6iwRESmbgoiIiJRNQURERMpWtyBiZiPM7Fdmtiz+HZ5nvwVm9qyZ3dVj+w1m9piZLTGz68yspTYpFxGRjHqWRC4DFrr7RGBhfJzL14ALcmy/AZgEHAcMAi6qRiJFRCS/egaRs4E58f4c4JxcO7n7QmBrju13ewT8CTisSunsn9Jp2LgMVnaEv+l0vVMkUlXptLNiwzbue2IjKzZsI53W5LTFqGcX3zHuvibeXwuMKedNYjXWBcAn8jw/A5gBMH78+HIO0f+k0/C3O2HeTNi1E1oGwblXw6SzIKVmNOl70mlnwdK1XHLTYrp2pWltSXHVeZM549ixpFJW7+Q1tKrmCGZ2T2yz6Hk7O3u/WJooN+x/F7jX3TtyPenu17h7u7u3jx6dc6yM9LT5ie4AAuHvvJlhu0gftGrT9j0BBKBrV5pLblrMqk3b65yyxlfVkoi7n5bvOTNbZ2bj3H2NmY0D1pf6/mb2OWA0MLOCZEpPW9d2B5CMXTth21oYNTHZY6XTIThtXQtDx8KII6tS2kmnnVWbtrNuSxdjhrXSNnKwrjBlj3VbuvYEkIyuXWnWb+1iwughdUpVc6hnddZ8YDrwlfj3jlJebGYXAW8BTnX3+lfY1ygzrImhY0MVVnYgaRkEQ8Yme5waVZupqkJ6M2ZYK60tqb0CSWtLioOHttYxVXtr1Ashq9fKhmY2ErgJGA+sBs5z981m1g5c7O4Xxf06CL2whgCbgA+5+y/M7MX4ukyj+23u/oVCx2xvb/eqTHtSbGbYLIEmqcy9t/PduAyunrpvsJrZkWiJZ8WGbUyb3bFPBnH3rKlNdZXZqJlIs0unnZUbt/Pomi0sW7+Vmzqf4pkdLzTUhUa9L4TMbJG7t+d6rm4lEXffBJyaY3snWd113X1qntc3zrxf+doQZh7TnRk2U2N1KhXSNfOYUIU1pIyAt/tFeGQezP94/vOtUbVZX6iqyJeJnH70GJ58ZocCS5lyfa5fOvc4jh9/EONHNM5nma/NZlIDXAg1TkbczIrJDIsJNJVIupSTSoV09Za2XMcFWHVvdwCB3Oebr9ps6CGhlJLQuVSzqqJWpYNcmciVCx5l1+40l976UN2r6ZqplJSd1gMGDtjnc/1/8x7m7llTGyr9jXwhpCCShGLaEKp51V2vUk6+444+Gp68v/fzHXFk2D/79e+8HtY+DLdf3L3tbd+CY86FAeX9XNtGDuaq8ybvcxXfNnJwhadfvSqGnpnypu3P75OJnPmKQ/cEEKjf1Wm9q1pK0TOts049qmEz52yN3GbTYPUoTSqTGbYMCo8zmWnmqhy6A022pBqr69UlN99xn1kJvrv3891TbdYBF94V/o6Y0B1AMu85/+OhZFPmgMdUyjjj2LHcPWsqP53xWu6eNTWRDK5a3UIzGd202R2cf+0DTJvdwdPPdnH4yL0/zwEp8maAtdRM3WN7pjXtITPO1iiZc7bMhVAmrUldCCVBQaQSmVHdq/8Qrr4v/kN3ZtizFFBMoClXoVJONeU77v5DYMltcNJn9z7ft31r3/PNVJu1TQ1/t63L/Z5P3l9RUEyljAmjhzBlwqg9V5iVjk4uVMVQiVyZ8qW3PsR/nn3cXpnIaw4f0RAZYLU+h2romdZbFz3FrFMmNmTmnK1aF0JJUHVWuUqtQkqisTqfWnXJLeW4p10B91wBUz4KqQEwfgq0nVT4fNNpGDg493umdyfW4J5U9Uu1qhjyZcotA4y7Z01l/dYuDh7ayvjhB1Slmq5UjVzV0lPPtK55rou5nU8yd8YUdu7azcFDS2/PqVV7UOZCqJGq2aCOXXzrIdEuvjXqnlqURmsTmXRWeH7zE8UHzN0vhiqrtUtg2CGw8fFQJWb7wQEj4YHvwXtvSuSzTarLb7XaAkpJXyYDywSWTAZWy4buZmwTuXLBo5z5ikMZkILXHD6CEyeMZL/9Sv9faaZzr0ShLr4KIuVa2QFzztx3+4V3haqZclTSwyrz2gpKOQUznnxpS+C4pNOw9NbQ9jFoBJz4Mfj1F7oD08mXh+Ax8fREguJ9T2zk/Gsf2Gf7T2e8likTRpWY9NyZeCUqzZjqkbEl8TnUKvC9+GKany1Zk0ivtr4yBqk3DTlOpOklXYVUTmkiV8Ze5pV6wYwHL5y2YroCF7L5ie7uwFPO7w4gEP7+9osw497ESlVJVr9Uo4ohU/89KavqqpQMNV9D98gPnMDooftXJXOu9HOoZeB78pkdifVqa+Sut7WihvVi5JoWPemG8lJ7WGWCztVTQ4no6qnhcZk9mAr2sKl276/sBnojTyeBdckci8bu6ZLRsyNAKRlpvoytY/lGps3uYMHStQ03zXkte3it29LF8AMG8tE3HcXHTgm34QcMLKsjQOaCJFsjtQfVYnp7lUR6U6iEkKuhHMobKFfqOJKEBy8WvKKyKo8s71mqy9mwvit8FwmURiq90i9HLdso8pW03BtrpHO2Wl7RjzuwlfefeDjfXLhsT6nnE6dOZOyw0jP+ao1BSkKtSncqifSm0FV4z+6p0HvpIN9iT6WOI0m4W2/BK6pqjnGBvUt1i2+Et3x57xLeSZ+Fn3060XEvlVzpZxR7lZdr3Ec1SwO5SlqzTpnIbX95CmjM7re1vKLfnWZPAIHweXxz4TJ2l1GIb+Sut7Uq3SmI9KaUzLq3ap9CVVClVo8lnLEXrOIpp+qulJURswcdvv1qGN4Wugaf9Jnw988/gM0rqj/upQSlBIZaD8bLzth+eGE7M06awI/vX82a50LgaKTqloxaVjGu35q71LNhW3mBtdAFST1XS6zV+B1VZ/WmlAb03qqkequCKmUcSa4pQypokylcxWOlpa2cTgLZDfQbl8H936n9uJcSlDIhXj0aXzMZW9vIwezcleaZHS8Ajdn+A7WtYqzVuJZ6d/+t1XmqJNKbUq7Ceysd9Faq6Vk9Vqj+P9eUIRWOCylYxVNs2tJpWPMgrFsKJ34Uhh1aekN8NUf3J6SUq7x8VTWGVf3qtJGrW3pKooqxGLUq9dR7OphanadKIr0pZaR5b6WD7FLNsEPhleeH0dwtg8trNE6ie22ScpVATvpsqI7a8o/iG+KrObo/IaVc5eVqfP3EqRP55NzFNVm3Irv7bTPNtlsttSr11Lv7b63OU4MNk1Zo8F0mk73nCnjVBXDvV6s3wrweC2DlG8U/5aOheqoeo/mrpNSqikzmvXrTdh78+7Pc3PnUXm0UmcFp1czk61290t/0pYGIGmxYS4VKB5kr7AMPg+vfWt21ReoxDUq+6rrUgGSqoxpoZchSr/IypYF1W7qYvXD5Xs9lrk7bRg6uaibfyAsb9UWN3P03SQoitZZKwQs7CjfAV5pZVnsBrDzSQ8aSytEJIT3xLaQOeVVlGX6OwOjnfJ8nx5zK0889X5eqmXJGaReqBqt2Jl/v6pX+ph7jkeqhcSqZ+5NCDfBJjESvdAxJKd1zs/zdxrLqpKv2ahBfddJV/L31pZWXGHIERrv9Yh5cvKgmYy+SUqixs9pdMht9dHVfVKvOAoVUu5uxSiL1UKgBPolSRCXzelVQFfb0c8/z6d+P5tOvuZHD9u9i7PAhbNi0mbZtq2HEMZUFkjyBcaQ/C7Q0TdVMoavTJLpkFmpT6S/VK9KtFu1gCiLlqLS6qVDvoySW0a1kDEkFQWzMsFY273iRry+CG17/DIff+QEO37UT/phAm0yewLjeDwR2AM1TNZOvGqzSTD5XhnHlO17BW18+jv32S/Wb6hXpVot2MAWRUiXVaJ2vAT6J2YEr6SJbQRDLZIIvrHuMIzouSbZNJkdgXHXSVXz99y/u2aXZq2aqMXvvpbc+xPADBvKGo0aRSlnDLmwk1VGLdjAFkVJVs9E6nQZLwZnfgLs+WdlI9BLGkGRXgbx8/1EMLTOIZTLBrv0fg98nPGFjj8CYHjyWFRuHsnnHg0DjjsQuVSWZfL4Mo3P1Zg4bPkiBox+qxah1BZFSJVHdlEt2CWfQCHjdrLBu+5hjYeRRVevK2rMK5IiRrdw17TsMvvujZQWxVMo4YMSh1VmuNyswpoCTR/pey8X296qZfBnG7jRNUc0nyatFO5gGG5aqnGVxi2lDqdNyu7kGRB0xspX55x/C0Bc3ljdavF7jVPq5dNq586Gn91qxb9YpE5nb+SQ/vPAEBZEGVu1BppWuOqnBhkkqtdG62Ay1WiWcXuSqAlm5qYulL4xmyoSjy3vTJpi2pC9KpYy3vnwcww8YSOfqzexOw9zOJ7n0jKObvpqvL6t2D6pqt4MpiJSq1Ayy2DaUpJfbLVLV6kwbbV6vfmK//VK84ahRHDZ8EOu3dvGO4w/t99V8ja7ZZxLQpWE5Spltt9iBf3WaubYZloqV0jTCADcpXq3W/aiWupVEzGwEMBdoA1YB57n7Mzn2WwBMAX7v7mfmeH428EF3b8yQXWwJo05VQBo7IFJftVr3o1rqWRK5DFjo7hOBhfFxLl8DLsj1hJm1A8Ork7yElFLCKKWEkyBduYrUT7PXBtStd5aZPQac7O5rzGwc8Ft3f1mefU8GPpNdEjGzAcA9wHuBZcWURGoyFXwuhaaHF5F+L4keVNXUqL2zxrj7mnh/LTCmxNd/DJgfg1DencxsBjADYPz48aWnMonpx5uskVkLF4nUVjPPJFDVIGJm9wC5uhddnv3A3d3Mii4SmdkhwLuAk3vb192vAa6BUBIp9hhAvxzvoIWLRKQUVc0J3f00d395jtsdwLpYjUX8u76Et34VcBSw3MxWAQeY2fLCLylDvu65xa4V3oTqvS60iDSXel5Ozwemx/vTgTuKfaG7/8zdx7p7m7u3ATvc/ajEU1jpuhxNqNm7G4pIbdUziHwFeLOZLQNOi48xs3Yz+0FmJzPrAG4GTjWzp8zsLTVLYc/Fo4YdCm+8NKxMWMJiTc1ECxeJ9A3VXowqQ3NnFdJzUsQT/gV+d2Wfbh9Rm4hI80v6/7hQ7ywFkd7s6Z67Dm54Z80nSKyHRu9uKCKF5ZpYtbUlxd1lTqVSKIj0nUvoasl0z3XvN+0jGnwo0txq2bapIFKsnu0jUJMJEkVESlXLtk0FkWLVaYJEEZFS1XIqFbWJlELTl4hIk0iybbNRpz1pPk02fYmI9F+1mkpFl9EiIlI2BRERESmbgoiIiJRNQURERMqmICIiImVTEBERkbIpiIiISNkUREREpGwKIiIiUjYFERERKZuCiIiIlE1BREREyqYgIiIiZes1iFjwz2b2H/HxeDM7ofpJExGRRldMSeS7wInA+fHxVuA7VUuRiIg0jWLWE3mtux9vZg8CuPszZjawyukSEZEmUExJZJeZDQAcwMxGA+nCLxERkf6gmCAyG5gHHGxmXwR+D3ypqqkSEZGmULA6y8xSwErgs8CpgAHnuPujNUibiIg0uIJBxN3TZvYdd38V8LcapUlERJpEMdVZC83sHWZmVU+NiIg0lWKCyEzgZuAFM9sab1uqnC4REWkCvQYRdx/q7il3b4n3h7r7sEoPbGYjzOxXZrYs/h2eZ78FZvasmd3VY7uZ2RfN7HEze9TMZlWaJhERKU1R056Y2dvM7L/j7cyEjn0ZsNDdJwIL4+NcvgZckGP7hcBLgEnufjTw04TSJSIiRSpm2pOvAJ8AHom3T5jZlxM49tnAnHh/DnBOrp3cfSFhlHxPHwa+4O7puN/6BNIkIiIlKKYkMg14s7tf5+7XAWcAb03g2GPcfU28vxYYU+LrjwTebWadZvZzM5uYayczmxH36dywYUMl6RURkR6KmfYE4CBgc7x/YLFvbmb3AGNzPHV59gN3dzPzYt832h/ocvd2M3s7cB0wtedO7n4NcA1Ae3t7qccQEZECigkiXwYeNLPfEAYbnkT+9ou9uPtp+Z4zs3VmNs7d15jZOKDU6qingNvi/XnAD0t8vYiIVKiY3lk3AlMIGfatwInuPjeBY88Hpsf704E7Snz97cCb4v03Ao8nkCYRESlBMQ3r5wI73H2+u88HuszsnASO/RXgzWa2DDgtPsbM2s3sB1nH7yCMUznVzJ4ys7dkvf4dZvYwobR0UQJpEhGREph74WYCM1vs7pN7bHswToXSVNrb272zs7PeyRARaSpmtsjd23M9V0zvrFz7FNsgLyIifVgxQaTTzK4ysyPj7X+ARdVOmIiINL5igsjHgReAufHWBXy0mokSEZHm0Gu1lLtvJ3bpjSscDo7bRESknyumd9ZPzGyYmQ0GHgYeMbN/rX7SRESk0RVTnXWMu28hzG31c+AIck+IKCIi/UwxQaTFzFoIQWS+u+8CNH2IiIgUFUSuBlYBg4F7zexwQItSiYhIUdOezHb3Q919moeRiU/SPd0IZjY9/6tFRKQvK2pRqmwevJi16RMJpkdERJpIyUEkB0vgPUREpAklEUTUyC4i0k+pJCIiImUrZrDhEb1s+0OiKRIRkaZRTEnk1hzbbsnccfePJZccERFpJnnnzjKzScCxwIFxDfOMYUBrtRMmIiKNr9AEjC8DzgQOAs7K2r4V+JcqpklERJpE3iDi7ncAd5jZie5+Xw3TJCIiTaKYNpFz4yy+LWa20Mw2mNk/Vz1lIiLS8IoJIqfHWXzPJMyhdRSgqeBFRKS4WXzj37cCN7v7c1VMj4iINJFeVzYE7jSzvwE7gQ+b2WjCErkiItLPFTOL72XA64D2uJbIDuDsaidMREQaXzEj1g8APgJ8L246BGivZqJERKQ5FNMm8kPgBUJpBOAfwH9VLUUiItI0igkiR7r7V4FdAO6+A026KCIiFBdEXjCzQcQp383sSOD5qqZKRESaQjG9s64AFgAvMbMbgNcDH6hmoqTJpNOw+QnYuhaGjoURR0IqiVUGRBpDOu2s2rSddVu6GDOslbaRg0mlVCEDRQQRd/+lmS0CphCqsT7h7hsrPbCZjQDmAm2EQYznufszOfZbEI/9e3c/M2v7qcDXCKWpbcCF7r680nRJidJp+NudMG8m7NoJLYPgnO/D2ONgy9MKKlK2Rsm402lnwdK1XHLTYrp2pWltSXHVeZM549ixCiQU1ztrobtvcvefuftd7r7RzBYmcOzLgIXuPhFYGB/n8jXgghzbvwe8z90nAz8B/j2BNEmpNj/RHUAg/L39YnhoLsw5E66eGoJMOl3fdEpTyWTc02Z3cP61DzBtdgcLlq4lna79QqqrNm3fE0AAunalueSmxazatL3maWlEeYOImbXG0sIoMxtuZiPirQ04NIFjnw3MiffnAOfk2sndFxJmDt7nKcK09AAHAk8nkCYp1da13QEkY9dO8N3d9+fNDMFGpEiNlHGv29K1Jx0ZXbvSrN+af8x1Ou2s2LCN+57YyIoN2+oS/GqlUHXWTOCThHEhi+jukbUF+HYCxx7j7mvi/bXAmBJffxFwt5ntjGmakmsnM5sBzAAYP358mUmVvIaMCVVY2YGkZVDshhHt2gnb1sKoiTVPnjSnQhn3hNFDapqWMcNaaW1J7ZWe1pYUBw/NvaxSf6v+ylsScfdvuvsRwGfcfYK7HxFvr3T3PUHEzN6c7z3M7B4zW5LjtteId3d39s52ivEpYJq7H0YYy3JVnvO4xt3b3b199OjRJR5CepUaAG+8NAQOCH/feCn89cbufVoGwZCx9UmfNKVMxp2tUMZdTW0jB3PVeZP3pCcTFNpGDs65fyOVomqhmIb1b/Wyy5XAr/K89rR8LzKzdWY2zt3XmNk4YH1vacl67Wjgle7+QNw0l9CDTHqTdE+qLU/Dn66FKR8NZdX9BkHLYNi5OTzfMgjOvTocR6RImYy759V8voy7mlIp44xjxzJp1lTWb+3i4KGFG/mTLEU1SueCQorp4tubcs9oPjAd+Er8e0cJr32GsGzvS939ceDNwKNlpqN0zdqlNVdPqnOvhklnlZ/+oWNDwOj47+5tIybAhT+DXTtCCaRZPh9pGKVm3LVIz4TRQ3oNAum0c8DAAcw69SjSDrcueoo1z3WVVYpqlmoxCzVJFbyB2V/c/fgyXjcSuAkYD6wmdPHdbGbtwMXuflHcrwOYBAwBNgEfcvdfmNm5wBeANCGofNDdVxQ6Znt7u3d2dpaa1L1VIyOulY3LQm+pnu0XMzvKb69o5s9DJEG5Mv1Zp0xkbueTXHrG0SVn/is2bGPa7I592mLunjW15u1CZrbI3XPOmZhESaQs7r4JODXH9k5Co3nm8dQ8r58HzKtaAvPJ1aV13kyYeUzjNxzn60lVSaN3KhUCxsxjwvuo5CH9VK62kNm/XsbcGVM47tCDSi49NFLngkKSCCKrEniP5lGNjLhWho7N3ZOq0kbvVCqce6Ofv0gV5cv0d+7aXVb1U6m9wuqlqMtFM3udmb3XzN6fuWWec/e3Vy95DSiTEWdrlt5HI44MVU3ZPanU6C2SiKR7lJXaK6xeem0TMbMfA0cCi4E4ggx391nVTVry+n2bCHR3ClDVk0iiqtEQnumdVe/OBYXaRIoJIo8Cx3ilLfANIJEgAo2VETdrTzGRJlFKN9tGyfSTVmnD+hJgLLCmtx37jUZpA2j2UpFIgyu1dFFsV+C+pJicZhTwiJn9wszmZ27VTpgUIV9PMc1TJZKI/jb6vBzFricijaiZe4qJNIFm6WZbT8VMe/K7WiREylCtLrsiAjRPN9t6KmY9kSlm9mcz22ZmL5jZbjPbUovESS/UZVekqpqlm209FdM7qxN4D3Az0A68H3ipu/9b9ZOXrMR6ZzWSRuopVm3qiSZ1UG6Pq2aYPLFYFU974u7LzWyAu+8GfmhmDwJNF0T6pEbpKVZt6okmdVJOj6tmmTwxCcX89+0ws4HAYjP7qpl9qsjX9V/pdJjscGVH+KulYcuWWSFu/eql6okmTaM/9eoqJhhcEPf7GLAdeAnwjmomquEVChKZK+arp2qN8Qplr7P9+PLl+XuiSU79aYnWRlPOkrrNqpjeWavNbBAwzt0/X4M0NbbeqlWSnuW3H7cDZF/NrWN4v+mJlkRden+qTmlE/alXVzG9s84izJu1ID6e3K8HG/Y2wK/Q2I1SpdOw7Jfw8M2w8rfw8C3hcT8p1WRfzX29cxcrp17V53uiZZe+zr/2AabN7mDB0rUllyJWbuw/1SmNqD/16ip2sOEJwG8B3H2xmR1RxTQ1tt4G+CU5duOZlbDhUfjj7O5SzxsvDccZ2bcyz1yyr+aefu4F3veH0Xz2tT/l9PFwwIhD+2SpLF9d+qQSFiJKp51H12zRILk6arSVGaupmP/AXe7+XI9t/bdytbep4JMcu7F1Dfzuyr1LPb+7MmzvB3pezW3e8SIDx7yM1qPeGAJpHwsgkExd+qpN21m2fmui05JL6TK9uqZMGMWE0UP6ZACB4koiS83svcAAM5sIzAL+WN1kNbBMkOjZJpIJEkmu9PfC9tylnhd2VH4eTaA/Xc1lJFGXvm5LFzd1PsWsUyYy+9fL9rSJfOnc4/pkdYrUVzFB5OPA5cDzwE+AXwD/Wc1ENbRigkRSYzeGH5G7amx4W2Xv20T626yomdJXzwbxUjL/McNaeWbHC/z4/tV86A0TMIOUwfHjS1+iVaQ3xYxYbycEkTa6g467+yuqm7TkNd2I9XQaHp0Pt1/cXeo55/tw9Nv6ZFWOBJWuSaGeWZK0Shelegz4DGFdkT1lbHdfnWQia6Hpggj0r2lNJDF9dXEkqY9Kpz3Z4O53JpwmKVZ/mdZEEtXfqgGlfooJIp8zsx8ACwntIgC4+21VS5WIiDSFYoLIB4BJQAvd1VkOKIiIiPRzxQSR17j7y6qeEhERaTrFtND+0cyOqXpKRESk6RRTEplCmAZ+JaFNxGjSLr4iIpKsYoLIGVVPhYiINKVeq7PcfXWuW6UHNrMRZvYrM1sW/w7Psc9kM7vPzJaa2UNm9u6s544wswfMbLmZzY0LZ4mISA3Vc9TaZcBCd59I6D58WY59dgDvd/djCSWib5jZQfG5K4H/cfejgGeAD1U/ySIikq2eQeRsYE68Pwc4p+cO7v64uy+L958G1gOjzcyAU4BbCr1eRESqq55BZIy7Z+Y0XwuMKbSzmZ0ADASeAEYCz7r7i/Hpp4BD87xuhpl1mlnnhg0bkkm5iIgAxTWsl83M7gFyrcZ0efYDd3czyzuJl5mNA34MTHf3dCiIFMfdrwGugTB3VtEvFBGRXlU1iLj7afmeM7N1ZjbO3dfEILE+z37DgJ8Bl7v7/XHzJuAgM9svlkYOA/6RcPJFRKQX9azOmg9Mj/enA3f03CH2uJoH/MjdM+0feJh6+DfAOwu9XkREqqueQeQrwJvNbBlwWnyMmbXHCR8BzgNOAi40s8XxNjk+dylwiZktJ7SR/G9NUy8iIr2vJ9KXNOV6IiIidVZoPRGtbiQiImVTEBERkbIpiIiISNkUREREpGwKIiIiUjYFERERKZuCiIiIlE1BREREyqYgIiIiZavqBIzSD6TTsPkJ2LoWho6FEUdCqjGvTdJpZ9Wm7azb0sWYYa20jRxMKlX8jNAisi8FESlfOg1/uxPmzYRdO6FlEJx7NUw6q+ECSTrtLFi6lktuWkzXrjStLSmuOm8yZxw7VoFEpAKN9Z8uzWXzE90BBMLfeTPD9gazatP2PQEEoGtXmktuWsyqTdvrnDKR5qYgIuXburY7gGTs2gnb1tYnPQWs29K1J4BkdO1Ks35rV51SJNI3qDpLyjd0bKjCyg4kLYNgSK7FLItUpTaWMcNaaW1J7RVIWltSHDy0teL3FmkE9WrzU0lEyjfiyNAG0jIoPM60iYw4srz3y7SxXD0V5pwZ/v7tzrC9Qm0jB3PVeZNpbQk/+UybSNvIwRW/t0i9Zdr8ps3u4PxrH2Da7A4WLF1LOl39pT60nohUJlNy2LY2lEAqKTlsXBYCR8+SzcwOGDUxgaSGK7X1W7s4eKh6Z0nfsWLDNqbN7tinpH33rKlMGD2k4vcvtJ6IqrOkMqlUyOATyOQLtrEk8P6plDFh9JBE/qlEGkmhNr9q/95VnSWNI9PGkq3SNhaRfiDT5petVm1+CiLSOJJuYxHpJ+rZ5qc2EWksSbaxiPQj1WzzU5uINI8k21hE+pF6tfnpEk9ERMqmICIiImVTEBERkbIpiIiISNkUREREpGwKIiIiUjYFERERKVvdgoiZjTCzX5nZsvh3eI59JpvZfWa21MweMrN3Zz13g5k9ZmZLzOw6M2up7RnIPtLpMIniyo7wN4HZd6WxpdPOig3buO+JjazYsK0ms8ZKY6lnSeQyYKG7TwQWxsc97QDe7+7HAmcA3zCzg+JzNwCTgOOAQcBFVU+x5FfFadylMdVz+nFpHPUMImcDc+L9OcA5PXdw98fdfVm8/zSwHhgdH9/tEfAn4LBaJFryaKKlciUZWnJYoL5BZIy7r4n31wJjCu1sZicAA4EnemxvAS4AFuR53Qwz6zSzzg0bNlSeasmtiZbKlWRoyWGBKs+dZWb3ALnm8b48+4G7u5nlLQOb2Tjgx8B0d+9ZP/Jd4F5378j1Wne/BrgGwgSMJSRfSlGNpXKloWnJYYEql0Tc/TR3f3mO2x3AuhgcMkFifa73MLNhwM+Ay939/h7PfY5QvXVJNc9DiqBp3PsdLTksUN9ZfOcD04GvxL939NzBzAYC84AfufstPZ67CHgLcGqO0onUWioFk86CmcdoGvd+IpUyzjh2LJNmTdWSw/1Y3dYTMbORwE3AeGA1cJ67bzazduBid7/IzP4Z+CGwNOulF7r7YjN7Mb5ua9x+m7t/odAxtZ6IiEjpCq0nokWpRESkoEJBRHUNIiJSNgUREREpm5bH7UcyazCv29LFIQfuz0t8Lalta0P3XDWCi0gZFET6icwUFZfctJgRB+zHT96wgdS9l4RxHZnuuJPOUiARkZIox+gnsqeo+PSr96MtE0BAU5SISNkURPqJ7CkqxtizmqJERBKh6qy+Ip0OJYmtuds4sqeoWMdwTVHShLLbtMYM08A+aQwqifQFRUzDnj1Fxdc7d7HqpKs0RUkT0bTr0qg02LAv2LgsBI6eJYuZHTBq4p5NmSvZ9Vu7GDcs9s7arilKmsGKDduYNrtjn8kO7541lQmjh9QxZdIfFBpsqOqsvqDQNOxZQSSVMiaMHpKV6QyF0RORxldo2nUFEaknXXr2BZlp2LOpjaNPybRpZdO069IIFET6Ak3D3udp2nVpVGoT6SsyvbM0DXufld2mpWnXpZbUJtIfpFKh/WOU2jj6qn3btETqT5eqIiJSNgUREREpm4KIiIiUTUFERETKpiAiIiJl61ddfM1sA7C6iF1HARurnJxa62vn1NfOB/reOfW184G+d07Fns/h7j461xP9KogUy8w68/WJblZ97Zz62vlA3zunvnY+0PfOKYnzUXWWiIiUTUFERETKpiCS2zX1TkAV9LVz6mvnA33vnPra+UDfO6eKz0dtIiIiUjaVREREpGwKIiIiUjYFkR7M7Awze8zMlpvZZfVOTznMbJWZPWxmi82sM24bYWa/MrNl8e/weqezEDO7zszWm9mSrG05z8GC2fE7e8jMjq9fynPLcz5XmNk/4ve02MymZT33b/F8HjOzt9Qn1fmZ2UvM7Ddm9oiZLTWzT8Ttzfwd5TunpvyezKzVzP5kZn+N5/P5uP0IM3sgpnuumQ2M2/ePj5fH59uKOpC76xZvwADgCWACMBD4K3BMvdNVxnmsAkb12PZV4LJ4/zLgynqns5dzOAk4HljS2zkA04CfAwZMAR6od/qLPJ8rgM/k2PeY+NvbHzgi/iYH1PsceqRxHHB8vD8UeDymu5m/o3zn1JTfU/ysh8T7LcAD8bO/CXhP3P594MPx/keA78f77wHmFnMclUT2dgKw3N1XuPsLwE+Bs+ucpqScDcyJ9+cA59QvKb1z93uBzT025zuHs4EfeXA/cJCZjatJQouU53zyORv4qbs/7+4rgeWE32bDcPc17v6XeH8r8ChwKM39HeU7p3wa+nuKn/W2+LAl3hw4Bbglbu/5HWW+u1uAU82s11XPFET2dijw96zHT1H4R9SoHPilmS0ysxlx2xh3XxPvrwXG1CdpFcl3Ds38vX0sVu9cl1XF2FTnE6s9XkW40u0T31GPc4Im/Z7MbICZLQbWA78ilJaedfcX4y7Zad5zPvH554CRvR1DQaRveoO7Hw/8E/BRMzsp+0kP5dWm7tvdF84B+B5wJDAZWAN8va6pKYOZDQFuBT7p7luyn2vW7yjHOTXt9+Tuu919MnAYoZQ0KeljKIjs7R/AS7IeHxa3NRV3/0f8ux6YR/jxrMtUH8S/6+uXwrLlO4em/N7cfV38J08D19JdFdIU52NmLYTM9gZ3vy1uburvKNc5Nfv3BODuzwK/AU4kVCVmlkbPTvOe84nPHwhs6u29FUT29mdgYuy9MJDQuDS/zmkqiZkNNrOhmfvA6cASwnlMj7tNB+6oTworku8c5gPvjz2ApgDPZVWpNKwebQLnEr4nCOfznthb5ghgIvCnWqevkFhX/r/Ao+5+VdZTTfsd5TunZv2ezGy0mR0U7w8C3kxo5/kN8M64W8/vKPPdvRP4dSxNFlbvHgSNdiP0InmcUHd4eb3TU0b6JxB6jPwVWJo5B0Ld5kJgGXAPMKLeae3lPG4kVB3sItTbfijfORB6oXwnfmcPA+31Tn+R5/PjmN6H4j/wuKz9L4/n8xjwT/VOf47zeQOhquohYHG8TWvy7yjfOTXl9wS8AngwpnsJ8B9x+wRCsFsO3AzsH7e3xsfL4/MTijmOpj0REZGyqTpLRETKpiAiIiJlUxAREZGyKYiIiEjZFERERKRsCiIiIlI2BRFpGGZ2vZm9M97/gZkdU+b7nGxmr8v1vkkzs4PM7CNZjw8xs1sKvUaqx8w+aWYHZD2+OzPgrsL3nZw9Bbx0UxDpB7KmOGiaY7v7Re7+SJmHPRl4XW87JeQgwhTaALj70+6eSMCq9Hur5fdez99YD58E9gQRd5/mYcqPSk0mDDwsWgN9JtVV71GVuhU9+rQN+BtwA2HqglsI/yz/QZiuZQlwDewZQPpb4BtAJ/Bp4CzCjKQPEkYSj4n7XUGY/rkDWA28nbAmxMPAAqClQJqmxTQtAmYDd2W954+BPxBGarfF9/9LvL0u7mfAtwmjfe8B7gbemZX+9nj/dOC++Nqb6V4jYRXw+bj9YcLkcm2E2WP/QRhxPBW4nrBuQidhNoIz4+tbgR/G1z4IvKmX7ccSRvIuJowCnkhYLmBn3Pa1ePwlcf8Lgdvi57gM+GrWZ/ehmJY/EeZj+nbcnknrA8BVhIn/FsTPuAOYFPc7Erg/pvG/gG1x+8lxv/nx/QfEdP05pnlm3G8ccG9M95L4OQ2Ix18S3/dTcd/J8VgPEeZiG57rN1bCb/l6wu/lj8AK4ndeYP9/zUr/5+O2wcDPCDMzLAHeDcwCXohp/03Wb2QU3f8/18fP5QbgNMJvdBlwQtz/BMJv7cGYvpcR1hZ6EtgQP693AyOA22Oa7gdekee3v89vpt55SeJ5U70ToFuRX1T4J3Dg9fHxdcBnyJq+JP54z4r3fwt8N+u54XQHmIuAr8f7VwC/J6w18EpgB3H6hphhnJMnPa2EaaOPiI9vZO8gsggYFB8fALTG+xOBznj/7YTpqQcAhwDP0iOIxAzgXmBw3H4p3dM3rAI+Hu9/BPhB1vE/k5XW6wkZcSoe/6mY/k8D18V9JsWMotD2bwHvi9sHAoPIChpZ31N2EFlBmMiulRCkXxLPdRUhI2ohZPrZQeQu4uJGhClEJsb7ryXMZ0Tc5/x4/2L2DiLbs76XGcC/x/v7EzL8I+I5ZqbEGUBYhOnVwK+yzuWg+Pch4I3x/heAb+T6jZXwW76ecDGQIizstLzAvqcTL47i/ncRFvh6B3Bt1n4HZv0mRmVtX0V3EHkROC6+zyLC/5AR1tG4Pe4/DNgv3j8NuDXru/x21vt+C/hcvH8KsDjPb3+f30y985Kkb/2juNV3/N3d/xDv/x/hymulmX2WkFGPIMyXdWfcZ27Waw8D5sbJ5AYCK7Oe+7m77zKzhwkZyoK4/WHCP18uk4AVHhbjgRBEZmQ9P9/dd8b7LcC3zWwysBt4adx+EnCju+8GnjazX+c4zhRCRvOHuD7OQMKVYkZm9thFhKCUz00eZmFdZmYrYvrfQPgnx93/ZmarY9rybb8PuNzMDgNuc/dlRazZs9DdnwMws0eAwwmZ2u/cfXPcfnPWZwJws7vvjlOSvw64Oes4+8e/J9K9mNBPgP/Oev2fsr6X04FXZLUJHUgIpH8Grouz1t7u7ovj5zLBzL5FuMr/pZkdSAgmv4uvn0MIABnZv7FS3B6/j0fMrNDaNqfH24Px8ZCY/g7g62Z2JeHipaOIY65094cBzGwp4bvx+Ltvi/scCMwxs4mEi7aWPO/1BkIgw91/bWYjzWxYfC77t7/Pb6aIdDYVtYk0l54TnTnwXcLV+3GEapHWrOe3Z93/FuFK6jhgZo/9ngeI/9S7PF42AWko+0Ij+9ifAtYRSjrthEBQLCNcHU+Ot2Pc/UM9004IToXSmuuzK4m7/wR4G6H66m4zO6WIlz2fdb+3NGZkPrsUYQGhyVm3o0t4PYTP7+NZrz/C3X/pYaXFkwjVfteb2fvd/RnCd/RbQunmByUeqxTZn0uhSGzAl7PSf5S7/6+7P05Ybvhh4L/M7D9KPGY663H27/w/CVVhLydUAWf/nxRrz2dS5m+mqSiINJfxZnZivP9eQjUUwMZ41VqoQfdAutcNmF5gv2I9RrhqbYuP393LsdfEIHUBobQDoZrq3XH1tXHAm3K89n7g9WZ2FOyZ6v6lOfbLtpVQPZPtXWaWMrMjCbOYPka4mn1ffN+XAuMLbTezCYTS12zC9NmvyHOs3vwZeKOZDY+Nr+/ItZOHBZFWmtm7YlrMzF4Zn74/63XvKXCsXwAfjiUOzOyl8TM8HFjn7tcSgsXxZjYKSLn7rcC/E9Ybfw54xsymxve7APjdvoepml8AH4y/b8zsUDM72MwOAXa4+/8R2nyOj/uX831ky/4/uTBre8/3zf6NnAxs9B6LcsXncv1m+hQFkebyGGGlwkcJbRzfI5Q+lhD+2f5c4LVXEKpFFgEbK01ILK5/BFgQ33MrYTnNXL4LTDezvxKqkTJXavMIjZqPAD9i72qqzHE2EP6ZbzSzh+I+va3Odidwrpktzsr8niQ0cP4cuNjdu2K6UrE6Yy5wobs/X2D7ecASC8uNvpywZvgmQlXbEjP7Wi/pypzTP4AvxfT8gVBvn++zex/wofjZLSXU30PohXRJ/EyOKvD6HxA+37+Y2RLgasJV98nAX83sQcIFwDcJy6P+Np7f/wH/Ft9jOvC1eKzJhHaRmnD3XxKq6+6L38cthMz8OOBPMa2fI3QugNB+ssDMflPmIb8KfDl+Ltmlxt8Ax8Tf1LsJ/0+vjp/JV8h/YbbPb6bMdDUsTQXfJOIV/12xmN0QzGyIu2+zUGH/HWCZu/9PvdPVDLI+u/0IwfQ6d59XwusPAHbGOv33EBrZz+7tdSJJU8O6VOJfzGw6oY3jQcJVrhTnCjM7jVDn/ktCd9FSvJrQWcEIvdo+mGjqRIqkkoj0yszmEbqFZrvU3X9Rj/RI4zGzy4F39dh8c65t7v7FHK8/jtBFPdvz7v7a5FIp1aAgIiIiZVPDuoiIlE1BREREyqYgIiIiZVMQERGRsv1/h5oKIDb+waEAAAAASUVORK5CYII=\n","text/plain":["<Figure size 432x288 with 1 Axes>"]},"metadata":{"needs_background":"light"},"output_type":"display_data"}],"source":["if allow_grid_searching:\n","    grid = {\n","        'gradientboostingregressor__n_estimators': stats.randint(50,300),\n","        'gradientboostingregressor__learning_rate':stats.uniform(0.05,0.3),\n","        'gradientboostingregressor__loss': ['ls', 'lad', 'huber', 'quantile'],\n","        'gradientboostingregressor__max_depth': stats.randint(3,5),\n","        'gradientboostingregressor__min_samples_split': stats.randint(2,10),\n","        'gradientboostingregressor__subsample': [0.95, 1], # 1 default\n","        'gradientboostingregressor__max_features': stats.randint(2,100), # default None, i.e = n_features\n","    }\n","\n","    search_gb = RandomizedSearchCV(pipe_gb, \n","                                   grid, \n","                                   scoring=rmse_neg, \n","                                   n_iter=50, \n","                                   cv=5, \n","                                   n_jobs=-1, \n","                                   verbose=False,)\n","\n","    # Fit data to Grid Search\n","    search_gb.fit(X, y_log);\n","    print('Best params: ', search_gb.best_params_)\n","    print('RMSE (of log): ', -search_gb.best_score_)\n","    \n","    # Plot results of gridsearch\n","    df_cv_results_ = pd.DataFrame(search_gb.cv_results_)\n","    sns.scatterplot(x=\"param_gradientboostingregressor__n_estimators\", y='mean_test_score', data=df_cv_results_)\n","    sns.scatterplot(x=\"param_gradientboostingregressor__max_features\", y='mean_test_score', data=df_cv_results_)"]},{"cell_type":"markdown","id":"9b1d3024-4f6b-4bdb-b90d-1f2e4c932cfd","metadata":{"id":"9b1d3024-4f6b-4bdb-b90d-1f2e4c932cfd"},"source":["## Stacking"]},{"cell_type":"code","execution_count":null,"id":"734b9f43-7875-4076-829f-7ee19cba8202","metadata":{"id":"734b9f43-7875-4076-829f-7ee19cba8202","outputId":"3ebf5206-cebb-4080-dc27-94067bb3ca44"},"outputs":[{"data":{"text/plain":["0.12025256163301815"]},"execution_count":109,"metadata":{},"output_type":"execute_result"}],"source":["from sklearn.ensemble import GradientBoostingRegressor, VotingRegressor\n","\n","gboost = GradientBoostingRegressor(n_estimators=100)\n","ridge = Ridge()\n","svm = SVR(C=1, epsilon=0.05)\n","adaboost = AdaBoostRegressor(base_estimator=DecisionTreeRegressor(max_depth=None))\n","\n","\n","model_voting_reg = VotingRegressor(\n","    estimators = [(\"gboost\", gboost),(\"adaboost\", adaboost),(\"ridge\", ridge), (\"svm_rbf\", svm)],\n","    weights = [1,1,1,1], # to equally weight the two models\n","    n_jobs=-1\n",")\n","\n","pipe_ensemble = make_pipeline(preproc, model_voting_reg, memory=cachedir)\n","\n","score = cross_val_score(pipe_ensemble, X, y_log, cv=5, scoring=rmse, n_jobs=-1)\n","score.mean()"]},{"cell_type":"code","execution_count":null,"id":"e25c8053-c46d-478c-bba9-389a0cebf49c","metadata":{"id":"e25c8053-c46d-478c-bba9-389a0cebf49c","outputId":"635aa198-b215-49cc-a6a6-85b792434bef"},"outputs":[{"data":{"text/plain":["0.1190762653604445"]},"execution_count":110,"metadata":{},"output_type":"execute_result"}],"source":["from sklearn.ensemble import StackingRegressor\n","\n","gboost = GradientBoostingRegressor(n_estimators=100)\n","ridge = Ridge()\n","svm = SVR(C=1, epsilon=0.05)\n","adaboost = AdaBoostRegressor(base_estimator=DecisionTreeRegressor(max_depth=None))\n","\n","model_stacking_reg = StackingRegressor(\n","    estimators=[(\"gboost\", gboost),\n","                (\"adaboost\", adaboost),\n","                (\"ridge\", ridge),\n","                (\"svm_rbf\", svm)],\n","    final_estimator=LinearRegression(),\n","    cv=5,\n","    n_jobs=-1\n",")\n","\n","pipe_stacking_1 = make_pipeline(preproc, model_stacking_reg, memory=cachedir)\n","score = cross_val_score(pipe_stacking_1, X, y_log, cv=5, scoring=rmse, n_jobs=-1)\n","score.mean()"]},{"cell_type":"markdown","id":"229fb393-bc8d-4501-b850-d374c6e91d44","metadata":{"id":"229fb393-bc8d-4501-b850-d374c6e91d44"},"source":["## XGBoost"]},{"cell_type":"code","execution_count":null,"id":"32118b74-a6c0-4f60-bc58-913904b7d500","metadata":{"id":"32118b74-a6c0-4f60-bc58-913904b7d500","outputId":"aef67cf4-73a9-4d92-f752-66756b47c9da"},"outputs":[{"data":{"text/plain":["0.12906759130738804"]},"execution_count":111,"metadata":{},"output_type":"execute_result"}],"source":["from xgboost import XGBRegressor\n","\n","model_xgb = XGBRegressor(max_depth=10, n_estimators=300, learning_rate=0.1)\n","pipe_xgb = make_pipeline(preproc, model_xgb)\n","cross_val_score(pipe_xgb, X, y_log, cv=5, scoring=rmse, n_jobs=-1).mean()"]},{"cell_type":"code","execution_count":null,"id":"a8de434e-330a-476c-b30f-dc7f6e88aeaa","metadata":{"id":"a8de434e-330a-476c-b30f-dc7f6e88aeaa","outputId":"a8c49187-77bc-44fe-d799-c502bb7ac5a6"},"outputs":[{"name":"stdout","output_type":"stream","text":["Best Validation Score 0.140842\n"]},{"data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAX4AAAEICAYAAABYoZ8gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAApaklEQVR4nO3deXxV9Z3/8dfn3mwkISGEsISwiyAoCETcFcXWtZV2rJXWjlY7VqerrWOrUx91fo/pYn+Oo7addpzWtf5krHtdarVCFxeUTWSXnbCGAElYQrbP7497AlckIZDce5J738/H45izn/fh+Picc7/33HPM3RERkfQRCTuAiIgklwq/iEiaUeEXEUkzKvwiImlGhV9EJM2o8IuIpBkVfhGRNKPCL0ljZvlmttbMvhg3rqeZrTezK+LGlZvZi2a208x2mdkSM/uRmRUF0681syYz2x10q83spgRnn2JmFUeY52Ez+/cE57jTzH6XyG1I6lPhl6Rx993AV4F7zawkGP0zYI67PwVgZmcAs4A3gdHu3gu4CGgExset7m13z3f3fOAfgJ+Z2YSk7IhIN6fCL0nl7q8CLwH3m9kU4Ergn+Nm+RnwkLv/xN23Bsusd/cfuvusVtY5H1gKnNAyzsw+bWaLg08Ms8wsftoJwbhdwTyfjpt2SfAJo9bMNprZLWaWB7wClMZ9yig9mv02s38ys5VmtsPMXohf3sw+aWbLzazazP7LzP5iZl85mvW3Y5+/F+xPbbCtqcH4yWY2x8xqzGyrmd1ztNuV7keFX8JwMzAFeAq4xd23AAQF9nTg6aNZmZmdAhwPzAmGjweeAL4NlAAvA38wsywzywT+APwJ6At8A3jczEYFq/st8FV37wmcCLzh7nuAi4FNLZ8y3H3TUeQ7H/gJsZPcAGAdMCOY1if4d7gNKAaWA2cczf63Y59HAV8HTgn260JgbbDofcB97l4AjACePNptS/ejwi9J5+47gcVALvBM3KQiYv9PbmkZYWY/C65g95jZD+LmPS0YXwu8CzwGfBhM+zzwkru/5u4NwN1AD2IF9TQgH/ipu9e7+xvAi8D0YNkGYIyZFbj7Tnef1wm7/EXgQXef5+77iRX5081sKHAJsNjdn3H3RuD++P0/Cm3tcxOQTWy/Mt19rbuvCpZrAI4zsz7uvtvd3+nIjkr3oMIvSWdmVwNDgdeBu+Im7QSaiV0VA+Dutwbt/M8CGXHzvuPuvYIr2P7AWODHwbRSYlfVLetoBjYAA4NpG4JxLdYF0yD2fcElwLqgyeX0Du3s4fPsBqri88RNc6DNL5HbuY0D++zuK4l9ErgT2GZmM+Kamq4n9mlpmZm9Z2aXHcO2pZtR4ZekMrO+wH8C/0Tsi94rzexsgKBJZTbw2aNZZ/BdwNPAp4JRm4Ahcds0YBCwMZg2yMzi/98fHEzD3d9z98uJNQM9x8Gmj448xvbQPHnEmnU2ApuBskOylh26gmPYRvw+4+7/z93PCuZxghOuu3/o7tOJ7e9dwFNBPklhKvySbL8AnnP3me6+GbgV+B8zyw6m3wpcZ2bfD04SmFkZMKy1FZpZMfAZYs1HECvWl5rZ1KBN/7vAfuAtYieWvcCtZpYZfMH8KWBG0B7+RTMrDJpLaoh9AgHYChSbWeER9i9qZjlxXRaxtvcvm9nJwX7+GJjt7muJfdF9kplNM7MM4GvEPsG0JXLINrLb2mczG2Vm5wfz1QH7WvbLzK42s5LgE8KuYP3NH9uipBZ3V6cuKR0wjdiVaa9Dxr8B/Chu+FRiX07uCrpFwI+A4mD6tcTarXcH3TZixbVv3Do+AywBqoG/AGPjpo0NxlUH83wmGJ8F/JFYk1MN8B5wVtxyDxJrotkFlB5m/x4mdjUd3/09mHYjsArYQew7hbK45S4CVgR5/gt4G/hSK/+Gdx5mGxVt7TMwjtj3ILVx2y8Npv0u+PfbTezEOS3s/0/UJb6z4OCLSBcQNEFVAF9095lh55HUpKYekZCZ2YVm1itoirkdMEB310jCqPCLhO90Ys1A24l93zDN3feFG0lSmZp6RETSjK74RUTSTMaRZwlfnz59fOjQoWHHEBHpVubOnbvd3UsOHd8tCv/QoUOZM2dO2DFERLoVM1t3uPFq6hERSTMq/CIiaUaFX0QkzXSLNn4RkaPV0NBARUUFdXV1YUdJuJycHMrKysjMzGzX/Cr8IpKSKioq6NmzJ0OHDiX2sNLU5O5UVVVRUVHBsGGtPsvwI9TUIyIpqa6ujuLi4pQu+gBmRnFx8VF9slHhF5GUlepFv8XR7mdKF/7n5m/kiTeXhx1DRKRLSek2/uGvXQf7a+DMt8OOIiJppqqqiqlTpwKwZcsWotEoJSWxH9G+++67ZGVltbrsnDlzePTRR7n//vsTki2lC7/n96NszxIamprJjKb0hxsR6WKKi4tZsGABAHfeeSf5+fnccsstB6Y3NjaSkXH4ElxeXk55eXnCsqV0NbQ+x1NsNVRsOpZ3V4uIdK5rr72WG2+8kVNPPZVbb72Vd999l9NPP50JEyZwxhlnsHx5rGl61qxZXHZZ7L33d955J9dddx1Tpkxh+PDhnfIpIKWv+PPKxsAS2LZ6EcMGDQ47joiE5N/+sJglm2o6dZ1jSgv44afGHvVyFRUVvPXWW0SjUWpqavjb3/5GRkYGr7/+OrfffjtPP/30x5ZZtmwZM2fOpLa2llGjRnHTTTe1+579w0lY4TezB4HLgG3ufmIwrjfwv8BQYC1wpbvvTFSGvsNPAmDv5qXAJYnajIhIu33uc58jGo0CUF1dzTXXXMOHH36ImdHQ0HDYZS699FKys7PJzs6mb9++bN26lbKysmPOkMgr/oeBXwCPxo37PvBnd/+pmX0/GP5eogL07Duc/WRilR8mahMi0g0cy5V5ouTl5R3ov+OOOzjvvPN49tlnWbt2LVOmTDnsMtnZ2Qf6o9EojY2NHcqQsDZ+d/8rsOOQ0ZcDjwT9jwDTErV9ACJRtmYMJK92dUI3IyJyLKqrqxk4cCAADz/8cNK2m+wvd/u5++agfwvQL9EbrMkbSt/69egVkyLS1dx6663cdtttTJgwocNX8Ucjoe/cNbOhwItxbfy73L1X3PSd7l7UyrI3ADcADB48eNK6dYd9n8ARLXjkFk5c/Vt2fWc9fQp7HtM6RKT7Wbp0KSeccELYMZLmcPtrZnPd/WP3hSb7in+rmQ0IAg0AtrU2o7s/4O7l7l7e8qOHY5HdfzQZ1szGVUuOeR0iIqkk2YX/BeCaoP8a4PlEb7D3kNiXOtUbVPhFRCCBhd/MngDeBkaZWYWZXQ/8FPiEmX0IXBAMJ1TJ0BMBaNy2LNGbEhHpFhJ2O6e7T29l0tREbfNwIjk9qbQ+ZO1alczNioh0WSn9yIYWO3oMpve+tWHHEBHpEtKi8O8rHEFZUwV19cm7XUpEpKtKi8IfLRlFge1jw4Y1YUcRkTRx3nnn8eqrr35k3L333stNN9102PmnTJnCnDlzkhEtPQp/z7LYva1VaxeFnERE0sX06dOZMWPGR8bNmDGD6dNb+/ozedKi8PcbPg6AfZt0Z4+IJMcVV1zBSy+9RH19PQBr165l06ZNPPHEE5SXlzN27Fh++MMfhpItpR/L3KJH8SD2koPt0MPaRNLSK9+HLR907jr7nwQXt35Heu/evZk8eTKvvPIKl19+OTNmzODKK6/k9ttvp3fv3jQ1NTF16lQWLlzIuHHjOjfbEaTFFT9mbM0aRM/dauMXkeSJb+5paeZ58sknmThxIhMmTGDx4sUsWZL8H5emxRU/wO784QyomkNzsxOJHN0b6UWkm2vjyjyRLr/8cm6++WbmzZvH3r176d27N3fffTfvvfceRUVFXHvttdTV1SU9V3pc8QNeMppSq2LDlq1hRxGRNJGfn895553Hddddx/Tp06mpqSEvL4/CwkK2bt3KK6+8EkqutCn8+YNjbWibV84POYmIpJPp06fz/vvvM336dMaPH8+ECRMYPXo0X/jCFzjzzDNDyZQ2TT39j5sIr8He9QuBi8OOIyJpYtq0aR95H0hrL1yZNWtWcgKRRlf8uSVD2UsOke1Lw44iIhKqtCn8RCJszh5GYe3KsJOIiIQqfQo/sKdwJIMb17G/sSnsKCKSBOnyytWj3c+0KvzWbyzFVsP69WvDjiIiCZaTk0NVVVXKF393p6qqipycnHYvkzZf7gIUDhkHH8C2VfMZOXxE2HFEJIHKysqoqKigsrIy7CgJl5OTQ1lZWbvnT6vC3/+4iQDUVSwCrgg3jIgkVGZmJsOGDQs7RpeUVk09WYX92GUFZFbpYW0ikr7SqvBjxtacERTv1WsYRSR9pVfhB+qKjmdI03p219WHHUVEJBRpV/gzBowl3+pYt0rNPSKSntKu8BcPGw9A1Zr3Q04iIhKOtCv8fUdMAKBhk17DKCLpKe0Kf6RHIdsiJeTsXB52FBGRUKRd4Qeoyh1Byb7VYccQEQlFWhb+huITGOoVbNtZE3YUEZGkS8vC32PwBLKsiXXL5oYdRUQk6dKy8A84YTIANWtU+EUk/aRl4c/vPyr2UpatH4QdRUQk6dKy8BOJsClnJH1q9SMuEUk/oRR+M7vZzBab2SIze8LM2v8g6U6yt3gMw5vWUL2nLtmbFhEJVdILv5kNBL4JlLv7iUAUuCrZObLKJpBn+1m9XM09IpJewmrqyQB6mFkGkAtsSnaAvsfHvuDduerdZG9aRCRUSS/87r4RuBtYD2wGqt39T4fOZ2Y3mNkcM5uTiDfo9B46jnoyYPPCTl+3iEhXFkZTTxFwOTAMKAXyzOzqQ+dz9wfcvdzdy0tKSjo/SDSTTVnD6FWztPPXLSLShYXR1HMBsMbdK929AXgGOCOEHNT0GsOwhlXU1TeGsXkRkVCEUfjXA6eZWa6ZGTAVCOWyOzpwPEW2m1Wr9MA2EUkfYbTxzwaeAuYBHwQZHkh2DoA+x50CwPYP3wtj8yIiocgIY6Pu/kPgh2FsO17f4ybShNG4cUHYUUREkiY9f7kbsOx8NmcMoufOxWFHERFJmrQu/AC7CkYzaP9KGpqaw44iIpIUaV/4KZ3AANvBytV6MYuIpIe0L/wlo04HYMvSN0NOIiKSHGlf+PuOmkwjEZrW684eEUkPaV/4LSuPjVnDKdqpRzeISHpI+8IPUNN7HCMbV1C7b3/YUUREEk6FH8gcMpkC28fKJfPDjiIiknAq/EDp2LMA2LHirZCTiIgkngo/UFA2lt3kEtmsK34RSX0q/BB7B2/uCfSvWYS7h51GRCShVPgDdf1O5jhfx+aqnWFHERFJKBX+QP6I08i0JtZ98HbYUUREEkqFPzDwxNgXvHtWvxNyEhGRxFLhD2T3KmVbpITcygVhRxERSah2F34zyzOzaCLDhK2y4CSG7FtKo57UKSIprNXCb2YRM/uCmb1kZtuAZcBmM1tiZv/XzI5LXszk8IGTGGiVrFy9KuwoIiIJ09YV/0xgBHAb0N/dB7l7X+As4B3gLjO7OgkZk6bfiVMA2LRwZrhBREQSqK1XL17g7g2HjnT3HcDTwNNmlpmwZCEoGTmZOrJg/dvADWHHERFJiFYLf0vRN7Peh5lc6+4NhzsxdGsZWWzIHcOA6gW4O2YWdiIRkU7Xni935wGVwArgw6B/rZnNM7NJiQwXhroBp3K8r2Hd5q1hRxERSYj2FP7XgEvcvY+7FwMXAy8C/wz8VyLDhaH3CecSNWfdgllhRxERSYj2FP7T3P3VlgF3/xNwuru/A2QnLFlISk86h0YiNKz+e9hRREQSoq0vd1tsNrPvATOC4c8DW4N7+lPuhnfL7smGrJH02TEv7CgiIgnRniv+LwBlwHNBNzgYFwWuTFSwMNX2K+eEphVs21kddhQRkU53xMLv7tvd/RvAOcDZ7v51d69093p3X5n4iMmXP/Jssq2BlQvU3CMiqeeIhd/MTjKz+cAiYLGZzTWzExMfLTyDTj4fgL0f/jXkJCIina89TT3/DXzH3Ye4+xDgu8ADiY0VrsyCfmzMKKOgcm7YUUREOl17Cn+eux94hoG7zwLyEpaoi9hRPIlR9Yup2bc/7CgiIp2qPYV/tZndYWZDg+4HwOqObNTMepnZU2a2zMyWmtnpHVlfIuSMOItC28vS+XoBu4iklvYU/uuAEuCZoCsJxnXEfcAf3X00MB5Y2sH1dbpBky4CoHrJn0NOIiLSuY54H7+77wS+2VkbNLNCYncIXRusvx6o76z1d5ac4sFsyiij1xZd8YtIamm18JvZHwBvbbq7f/oYtzmM2PN+HjKz8cBc4FvuvueQ7d9A8IjMwYMHH+OmOqaq7xmcuPF5KnfVUtKrZygZREQ6W1tX/HcncJsTgW+4+2wzuw/4PnBH/Ezu/gDB3UPl5eWtnoASKf+EqeRuepK5c96g5ILLw4ggItLp2nos818StM0KoMLdZwfDTxEr/F3O4IkX0vRno275G6DCLyIpIukvW3f3LcAGMxsVjJoKLEl2jvaI5hWxPvt4+m5/B/dQPnSIiHS6pBf+wDeAx81sIXAy8OOQchzR7tKzGNu8gvV6Pr+IpIi2Xrb+WPD3W529UXdf4O7l7j7O3acFdw51ScXjPkmGNbNm7p/CjiIi0inauuKfZGalwHVmVmRmveO7ZAUM24ATz2U/WTSvStRXHiIiydXWXT2/Bv4MDCd2y2X8C2g9GJ/yLLMH6/LGMWjXbJqbnUhE7+EVke6t1St+d7/f3U8AHnT34e4+LK5Li6LfomHwWYxkAytWpeRTqEUkzbTnefw3mdl4M/t60I1LRrCupHTSpQBseO/FkJOIiHRce57H/03gcaBv0D1uZt9IdLCupGh4OTusiB7r9NweEen+2vPO3a8Ap7Y8UsHM7gLeBn6eyGBdSiTCpr5nM27La1RV76a4MD/sRCIix6w99/Eb0BQ33MRHv+hNC/knXkKB7WPxbN3WKSLdW3sK/0PAbDO708zuBN4BfpvQVF3Q4PJLaSCD+qV/DDuKiEiHtOexzPeY2SzgrGDUl919fkJTdUGRHgWszDuZYTv/TmNTMxnRsH70LCLSMW39cvdAQ7a7zwtu77w/vujHz5MOmkZ8ghFsZNGihWFHERE5Zm1dtj5vZv9hZueY2YF37JrZcDO73sxeBS5KfMSuY9Bp0wDYNu+FcIOIiHRAWz/gmkrsl7tfBRabWY2ZVQG/A/oD17j7U8mJ2TXkl45mc3QgRRvfCDuKiMgxa7ON391fBl5OUpZuYXvpFMatf5KN27YzsG+fsOOIiBw1fUN5lIonfIpsa2DZm8+HHUVE5Jio8B+l0vEXUGv5ZKx4KewoIiLHRIX/aEUzWV8yhQl732bbrpqw04iIHLW2buc8P65/2CHTPpvIUF1dwaQrKLC9LPqb7u4Rke6nrSv+u+P6nz5k2g8SkKXbKJt4MbvJJbJUhV9Eup+2Cr+10n+44bRimTms63MO4/e8yc6aPWHHERE5Km0Vfm+l/3DDaSf/5H+gyHbz/pv6kldEupe27uMfbmYvELu6b+knGB7W+mLpYfDky9j7eg6++Dm4+Mqw44iItFtbhf/yuP67D5l26HDasaxc1hadyUk7/kbN3joKcnPCjiQi0i5tPbLhL/Ed8BZQAywNhtNe9vjP0sdqmP83/bhZRLqPtm7n/LWZjQ36C4H3gUeB+WY2PUn5urRhp01jLzk0Lfx92FFERNqtrS93z3b3xUH/l4EV7n4SMAm4NeHJuoFITj5r+pzHpN2z2LZjV9hxRETapa3CXx/X/wngOQB335LIQN1N0elXU2h7WTjzybCjiIi0S1uFf5eZXWZmE4AzgT8CmFkG0CMZ4bqD0pMvYocVkbvs0N+4iYh0TW0V/q8CXyf2zt1vx13pTwV083qLaAabBl1Gef17rFy3Puw0IiJH1NZdPSvc/SJ3P9ndH44b/6q7fzcp6bqJgedeS5Y1sWrmY2FHERE5olbv4zez+9ta0N2/2ZENm1kUmANsdPfLOrKusBUNn0RF5lBK1z1Pc/PtRCJp/UQLEeni2mrquRE4C9hErEDPPaTrqG8BSzthPeEzo+b4z3KSL2f+wnlhpxERaVNbhX8A8ABwIfAlIBN43t0fcfdHOrJRMysDLgV+05H1dCXDzvsyzW5s//vDYUcREWlTW238Ve7+a3c/j9h9/L2AJWb2pU7Y7r3EfgvQ3NoMZnaDmc0xszmVlZWdsMnE6tFnMCsLTuXkyj/oiZ0i0qUd8Q1cZjaRWLPM1cArdLCZx8wuA7a5e5vrcfcH3L3c3ctLSko6ssmk6XHGV+hnO5nz2hNhRxERaVVbj2z4P2Y2F/gO8Beg3N2vd/clHdzmmcCnzWwtMAM438x+18F1dgmDJn+G7ZFiCpf8Dve0f3K1iHRRbV3x/4BY88544CfAPDNbaGYfmNnCY92gu9/m7mXuPhS4CnjD3a8+1vV1KdEMth73ecobF/D+wgVhpxEROay2Hsuc9s/cPxbDP3kTvuJXVP7lARj/q7DjiIh8TKuF393XHW68mUWA6cBhpx8Nd58FzOroerqSHn0Gs6zwLCZUvcSOmt30LsgPO5KIyEe01cZfYGa3mdkvzOyTFvMNYDWgV061If/Mf6KPVTPvVf2SV0S6nrba+B8DRgEfAF8BZgJXANPc/fI2lkt7ZeWXsSU6gL5LHqaxqdU7VkVEQtFW4R/u7te6+38Ta9oZA1zo7guSkqw7i0TZddL1jPNlvPO3P4adRkTkI9oq/A0tPe7eBFS4e13iI6WG4y+6kVrysLd+qVs7RaRLaavwjzezmqCrBca19JtZTbICdleRnJ6sG/Z5Ttv/Jh8sOua7X0VEOl1bj2yIuntB0PV094y4/oJkhuyuRlx2M80WYfvr94YdRUTkgCM+skGOXY/iwazo80lO3fUSGzZuDDuOiAigwp9wAy7+F/JsP8tevC/sKCIigAp/wvUeMYnl+ZOZuOkJtm6vCjuOiIgKfzL0uvgHFFsN7z97T9hRRERU+JOh39hzWZFXzqSKR9lapat+EQmXCn+SFFx0h676RaRLUOFPkv4nTWF5XjkTNzzGtqodYccRkTSmwp9EPS/6AX2smgXP/kfYUUQkjanwJ1HpSeexIq+cUzY8TMXmzWHHEZE0pcKfZEXTfkohe1j25J1hRxGRNKXCn2QlI09hSd9LOHvH0yxZ8kHYcUQkDanwh2DYlT/Bzah64Q49uVNEkk6FPwR5JUP4cPg1nF03k3fffD3sOCKSZlT4QzL6ih+w0wrJfeMO6uobjryAiEgnUeEPSWZuLyonf4+Tmpfy19//POw4IpJGVPhDdPyFN7EqZyzlK+5hfUVF2HFEJE2o8IcpEqHX535OAXtYOeNWfdErIkmhwh+y4hGTWDbki0ypfZm3/6IXs4tI4qnwdwGjr/oROyJFlMz6Hrtqd4cdR0RSnAp/F5CRW8ieT/yMkaxj9kPfCzuOiKQ4Ff4uYsgZn2NJv09xQdXjvDlLTT4ikjgq/F3IyH/8OTuifSiddTPbduwMO46IpCgV/i4kM6+I+st+zjA2sfChm3WXj4gkRNILv5kNMrOZZrbEzBab2beSnaErGzjxYhYPms4Ftc/y+rMPhR1HRFJQGFf8jcB33X0McBrwNTMbE0KOLmvMP97L2qzjOfX9f2XRovfDjiMiKSbphd/dN7v7vKC/FlgKDEx2jq7MMnPofe0TmBkZT1/LzuqasCOJSAoJtY3fzIYCE4DZh5l2g5nNMbM5lZWVSc8WtoLS46j6xH2M9tW8/z830tys9n4R6RyhFX4zyweeBr7t7h+7pHX3B9y93N3LS0pKkh+wCxh65udYNOzLTNn9Eq89+u9hxxGRFBFK4TezTGJF/3F3fyaMDN3F2KvvZlnBGUxdcw9/fWVG2HFEJAWEcVePAb8Flrr7Pcnefndj0QxG3DiDTVlDmPDOt3l/3sdaxUREjkoYV/xnAl8CzjezBUF3SQg5uo3M3EJ6Xf8MDZEser/wJdasXRV2JBHpxsK4q+fv7m7uPs7dTw66l5Odo7sp6D+c/Vc8Th920fTIZ9i4aWPYkUSkm9Ivd7uRAWPPZtulDzHYN1L9m2lUVlWFHUlEuiEV/m5myCmXsv78X3J800oqfjWNHbuqw44kIt2MCn83dNw5V7HqjLsY3/ABG35+Kdt15S8iR0GFv5sadeENfHjWPYxtXMzWX17Mlq2bw44kIt2ECn83NuoT17Hm/F8xsmkVNb++mA3rVocdSUS6ARX+bm7kuVdRcdGDDPJNZDx0AYvnvxV2JBHp4lT4U8Dw0y9n55XPk2HOkOc+y+w/6Re+ItI6Ff4UUTrmdDK+OottmaWUv3kjf33oDpqamsOOJSJdkAp/CinqP4TSm2eyqOBczll3P/Pu/hQ7d+iOHxH5KBX+FJOTV8i4m59l7uhbmLD3LWp/fhZL578ZdiwR6UJU+FOQRSJMuuoO1l46g1zfx/DnPs1fH7qDxoaGsKOJSBegwp/Cjpt8IdnffIflBWdwzrr7WfazKaxbuTjsWCISMhX+FNezd3/GfecFFkz6MUMbVtHvsXN586F/Zf/+urCjiUhIVPjTgRknf+pr7L/hbZb3PI0z1/2CjXdNZtFbL4WdTERCoMKfRopLhzH+lhdZeM6vyfW9nPinLzDvZ5dSsfKDsKOJSBKp8KehcedPp9e/LOCdoV9j1J459H3sXN795XVs37Qm7GgikgQq/GkqJzef0679Mftueo/5xZcyYdtz9PzvU3jvV19hx+a1YccTkQRS4U9zffoP5tRvPsbWa95kfq9PMmHL0/T89UTm33clm5a9G3Y8EUkAc/ewMxxReXm5z5kzJ+wYaWHtyiVUvPIfTNj+B/JsP8tzxkP5lzn+3OlYZk7Y8UTkKJjZXHcv/9h4FX45nMptW1j8h/sYueH3DKSSXVbAhkGfpmzKdRQNnxR2PBFpBxV+OSZ19Q3MeeMZovMeZtL+2WRZExWZw6gd9Q8MO+sqcvqPDDuiiLRChV86bNW69ayc+Qil617gJF8BQEXWcPaNuJiBk6eRO6QcIvraSKSrUOGXTtPU7Cz4YCFbZj9F6aY/Md6XEzGnJlLItn5n03PMBfQd90mscGDYUUXSmgq/JERjUzMLlq+iYs6L5K9/g4kN8+htuwGozCpjT79TKDj+LHqPPgeKj9MnApEkUuGXpFi3vZYP5r3NvuVvUFL1LuN8+YETwb5IHjsLxxIpm0DvEeVklY6LnQyiGSGnFklNKvySdM3NzvItNSxbNJc9K98id/tCjmtcwWhbT5Y1AdBgWdTkDcWLR5I7cCy5A0ZB7+GxrkevcHdApJtT4ZcuYUt1He+v28rmle9TX/EBPXYuZWDjBo6zjZTZdiJ28P/H/Rk9qc8vw4oGk1M8hIxeA6FgIPTsH+vy+0J2AZiFuEciXZcKv3RZlbX7Wbalhg8rKtm1cQUNlSvJrF5Dn8atDLTtDLJt9LcdFNi+jy3bFM2mOacYyy8hmt8H61EEub0hp1fsE0NOYazLyo+dJLLzITMXsvJifzOydeKQlNVa4VfjqoSupGc2JT1LOHtkCTDmwPide+pZU7WHRdv38PKOvWyrrGJf1XoaqzeTsW8bfXwnxY019Kmvobimmt62luLoYgrZTZ7vIUJ7LmoMMntARk7wNxsyekBGFkSzY8PRrKDLDLosiGTE+iMZrXSR2F+LgkViXSQaO8m0DMdPs8jBaXBwGDt4YrJI3HDcyerAicsOWSZ+nkP2+cByh+k/7D9T/LR2nCg77WTaiSflDmU63LKJvGiO296A8ZCV26lrD6Xwm9lFwH1AFPiNu/80jBzStRXlZVGUl8XEwUVxY08HYreUbq2pY0tNHVuq61hTXcc7tXVU1u6nsnY/22v2Ubenmqa9u8j33eRRR77tI5999LD99KCeXNtPYUYjhd5Az+ZG8hoayGtqoEdDA9nWSDYNZLGPDBpjnTcQ9Uai3kikuYGIN2LNTVhzA3gzNDdi3hTOP5akrq+9ByXHd+oqk174zSwK/BL4BFABvGdmL7j7kmRnke4rGjFKe/WgtFePNudrbnZq6hrYubeBHXvq2bmnnup9Deza10D13no272tg2b4Gauoaqa1roLaukdq6RvbUN7K7rpHG5qO7qouYk5th9MiAHlEnO8PIyXByIpCTYWRHISsKWRHIijqZESMr4mRGIdMgIwKZESMjAtGIk2EQjUSCYcgwJ2pGNGJEDCIWGx81I2IeGw46MyNiFvRzYH7DiEZiWSNYMO7gPLG/FlwgGxE8WNfB+WL7GltXy4eFSHCVai3ztfyjHFivtQweXC5uHguyHJjHHOu0K/4OXJ231RyeiGbCQ7eXgN/DhHHFPxlY6e6rAcxsBnA5oMIvnS4SMXrlZtErN4thffKOall3Z39jM/vqm9i9v5G99U3srW9kX0MT++qbDvyta2hif2MzdQ1N1DU0U9/UzP5gXH1jM/ubmtnf0ExDUzN7m5qpbmqmvr6ZhiansTn2t6GpOeicxqZmmpqdhmanKegOSUZimxm6lgMnErMDJ54DJ4SPnCha+u1jy7TMdKBRzCxu/o+elDiwroNF3Q6znZZtHX6ej46Pz2yH5IiLd+A/8dMfvMYYXNzqP88xCaPwDwQ2xA1XAKceOpOZ3QDcADB48ODkJBOJY2bkZEbJyYxSlJcVWg53pzE4ATQ2O01NTpPHThrNzdDkTnPzwXma3Wlsiv1t9pZxsfU0NXswf7CcezA+Nv3AfO64E0wH5+AytAw7B6c5ELd8MBj8jZ2kDq7r4EWt4wf7/fDrPLiug8tzyDo40B9sv43ttGzr4Pxx64nLfXDeA32HrCOu/5BlD812MLN/LH987pb++J6sjM7/0WOX/XLX3R8AHoDYXT0hxxEJjZmRGTUyo2EnkVQRxu/nNwKD4obLgnEiIpIEYRT+94CRZjbMzLKAq4AXQsghIpKWkt7U4+6NZvZ14FVit3M+6O6Lk51DRCRdhdLG7+4vAy+HsW0RkXSnZ+SKiKQZFX4RkTSjwi8ikmZU+EVE0ky3eCyzmVUC645x8T7A9k6M012k436n4z5Deu639rl9hrh7yaEju0Xh7wgzm3O451GnunTc73TcZ0jP/dY+d4yaekRE0owKv4hImkmHwv9A2AFCko77nY77DOm539rnDkj5Nn4REfmodLjiFxGROCr8IiJpJqULv5ldZGbLzWylmX0/7DyJYGaDzGymmS0xs8Vm9q1gfG8ze83MPgz+Fh1pXd2NmUXNbL6ZvRgMDzOz2cHx/t/gsd8pxcx6mdlTZrbMzJaa2empfqzN7Obg/+1FZvaEmeWk4rE2swfNbJuZLYobd9hjazH3B/u/0MwmHs22Urbwx73U/WJgDDDdzMaEmyohGoHvuvsY4DTga8F+fh/4s7uPBP4cDKeabwFL44bvAv7T3Y8DdgLXh5Iqse4D/ujuo4HxxPY/ZY+1mQ0EvgmUu/uJxB7lfhWpeawfBi46ZFxrx/ZiYGTQ3QD86mg2lLKFn7iXurt7PdDyUveU4u6b3X1e0F9LrBAMJLavjwSzPQJMCyVggphZGXAp8Jtg2IDzgaeCWVJxnwuBc4DfArh7vbvvIsWPNbHHx/cwswwgF9hMCh5rd/8rsOOQ0a0d28uBRz3mHaCXmQ1o77ZSufAf7qXuA0PKkhRmNhSYAMwG+rn75mDSFqBfWLkS5F7gVqA5GC4Gdrl7YzCcisd7GFAJPBQ0cf3GzPJI4WPt7huBu4H1xAp+NTCX1D/WLVo7th2qb6lc+NOKmeUDTwPfdvea+Gkeu2c3Ze7bNbPLgG3uPjfsLEmWAUwEfuXuE4A9HNKsk4LHuojY1e0woBTI4+PNIWmhM49tKhf+tHmpu5llEiv6j7v7M8HorS0f/YK/28LKlwBnAp82s7XEmvDOJ9b23StoDoDUPN4VQIW7zw6GnyJ2IkjlY30BsMbdK929AXiG2PFP9WPdorVj26H6lsqFPy1e6h60bf8WWOru98RNegG4Jui/Bng+2dkSxd1vc/cydx9K7Li+4e5fBGYCVwSzpdQ+A7j7FmCDmY0KRk0FlpDCx5pYE89pZpYb/L/ess8pfazjtHZsXwD+Mbi75zSgOq5J6MjcPWU74BJgBbAK+New8yRoH88i9vFvIbAg6C4h1ub9Z+BD4HWgd9hZE7T/U4AXg/7hwLvASuD3QHbY+RKwvycDc4Lj/RxQlOrHGvg3YBmwCHgMyE7FYw08Qex7jAZin+6ub+3YAkbsrsVVwAfE7npq97b0yAYRkTSTyk09IiJyGCr8IiJpRoVfRCTNqPCLiKQZFX4RkTSjwi8ikmZU+EVE0sz/B9XSnRG2B4+VAAAAAElFTkSuQmCC\n","text/plain":["<Figure size 432x288 with 1 Axes>"]},"metadata":{"needs_background":"light"},"output_type":"display_data"}],"source":["X_train, X_eval, y_train_log, y_eval_log = train_test_split(X,y_log,random_state=42)\n","\n","X_train_preproc = preproc.fit_transform(X_train, y_train_log)\n","X_eval_preproc = preproc.transform(X_eval)\n","\n","model_xgb.fit(X_train_preproc, y_train_log,\n","        verbose=False,\n","        eval_set=[(X_train_preproc, y_train_log), (X_eval_preproc, y_eval_log)],\n","        eval_metric=[\"rmse\"],\n","        early_stopping_rounds=10)\n","\n","# retrieve performance metrics\n","results = model_xgb.evals_result()\n","epochs = len(results['validation_0'][\"rmse\"])\n","x_axis = range(0, epochs)\n","\n","# plot rmsle loss\n","fig, ax = plt.subplots()\n","ax.plot(x_axis, results['validation_0']['rmse'], label='Train')\n","ax.plot(x_axis, results['validation_1']['rmse'], label='Val')\n","ax.legend(); plt.ylabel('RMSE (of log)'); plt.title('XGBoost Log Loss')\n","\n","print(\"Best Validation Score\", min(results['validation_1']['rmse']))"]},{"cell_type":"markdown","id":"15ec3816-4ba9-4f30-88aa-7bfa360ac39f","metadata":{"id":"15ec3816-4ba9-4f30-88aa-7bfa360ac39f"},"source":["# OTHER MODEL TESTINGS"]},{"cell_type":"code","execution_count":null,"id":"525b2443-bcee-44d6-813e-539a3928ac61","metadata":{"id":"525b2443-bcee-44d6-813e-539a3928ac61","outputId":"d3c131eb-2cca-4132-a0cb-186fe7f2afe8"},"outputs":[{"data":{"text/plain":["0.11700522598413146"]},"execution_count":113,"metadata":{},"output_type":"execute_result"}],"source":["from sklearn.ensemble import StackingRegressor\n","\n","gboost = GradientBoostingRegressor(n_estimators=100)\n","ridge = Ridge()\n","svm = SVR(C=1, epsilon=0.05)\n","adaboost = AdaBoostRegressor(base_estimator=DecisionTreeRegressor(max_depth=None))\n","knn = KNeighborsRegressor()\n","xgb = XGBRegressor(max_depth=3, n_estimators=2000, learning_rate=0.05)\n","\n","model_stacking_2 = StackingRegressor(\n","    estimators=[(\"gboost\", gboost),\n","                (\"adaboost\", adaboost),\n","                (\"ridge\", ridge),\n","                (\"svm_rbf\", svm),\n","                (\"knn\", knn),\n","                (\"xgb\", xgb)],\n","    final_estimator=LinearRegression(),\n","    cv=5,\n","    n_jobs=-1\n",")\n","\n","pipe_stacking_2 = make_pipeline(preproc, model_stacking_2, memory=cachedir)\n","score = cross_val_score(pipe_stacking_2, X, y_log, cv=5, scoring=rmse, n_jobs=-1)\n","score.mean()"]},{"cell_type":"markdown","id":"153a2488-6909-4375-997f-78e36c0c8c0c","metadata":{"id":"153a2488-6909-4375-997f-78e36c0c8c0c"},"source":["**Use best gboost params from grid search**"]},{"cell_type":"code","execution_count":null,"id":"a5fa13f7-f806-44a3-9ec8-0fed566d679a","metadata":{"id":"a5fa13f7-f806-44a3-9ec8-0fed566d679a","outputId":"12a4fd79-2b72-4712-b4ad-7ac30007dbb0"},"outputs":[{"data":{"text/plain":["0.1163791985608453"]},"execution_count":115,"metadata":{},"output_type":"execute_result"}],"source":["import warnings\n","warnings.filterwarnings('ignore')\n","warnings.simplefilter(action='ignore', category=FutureWarning)\n","\n","from sklearn.ensemble import StackingRegressor\n","\n","gboost = GradientBoostingRegressor(\n","    learning_rate = 0.092, \n","    loss = 'squared_error',\n","    max_depth = 4,\n","    max_features = 23,\n","    min_samples_split = 2,\n","    n_estimators = 274,\n","    subsample = 1)\n","ridge = Ridge()\n","svm = SVR(C=2, epsilon=0.05)\n","adaboost = AdaBoostRegressor(base_estimator=DecisionTreeRegressor(max_depth=None))\n","knn = KNeighborsRegressor()\n","xgb = XGBRegressor(max_depth=3, n_estimators=2000, learning_rate=0.05)\n","\n","model_stacking_3 = StackingRegressor(\n","    estimators=[(\"gboost\", gboost),\n","                (\"adaboost\", adaboost),\n","                (\"ridge\", ridge),\n","                (\"svm_rbf\", svm),\n","                (\"knn\", knn),\n","                (\"xgb\", xgb)],\n","    final_estimator=LinearRegression(),\n","    cv=5,\n","    n_jobs=-1\n",")\n","\n","pipe_stacking_3 = make_pipeline(preproc, model_stacking_3, memory=cachedir)\n","score = cross_val_score(pipe_stacking_3, X, y_log, cv=5, scoring=rmse, n_jobs=-1)\n","score.mean()"]},{"cell_type":"markdown","id":"93770537-aa44-4287-84fe-fac0406669c9","metadata":{"id":"93770537-aa44-4287-84fe-fac0406669c9"},"source":["## CatBoostRegressor"]},{"cell_type":"code","execution_count":null,"id":"79e0dd89-b1c3-43bb-8ffa-f74700d539bd","metadata":{"id":"79e0dd89-b1c3-43bb-8ffa-f74700d539bd"},"outputs":[],"source":["# !pip install catboost"]},{"cell_type":"code","execution_count":null,"id":"c2091b98-dc40-49a5-9e93-270e5c47fd1f","metadata":{"id":"c2091b98-dc40-49a5-9e93-270e5c47fd1f","outputId":"ebf9502b-09b9-4d89-9f27-f14df098e92d"},"outputs":[{"data":{"text/plain":["0.11799059634803322"]},"execution_count":116,"metadata":{},"output_type":"execute_result"}],"source":["import warnings\n","warnings.filterwarnings('ignore')\n","warnings.simplefilter(action='ignore', category=FutureWarning)\n","\n","\n","from catboost import CatBoostRegressor\n","\n","params = {'iterations': 6000,\n","          'learning_rate': 0.005,\n","          'depth': 4,\n","          'l2_leaf_reg': 1,\n","          'eval_metric':'RMSE',\n","          'early_stopping_rounds': 200,\n","          'verbose': False,\n","          'random_seed': 42}\n","cat_f = CatBoostRegressor(**params)\n","\n","pipe_cat = make_pipeline(preproc, cat_f, memory=cachedir)\n","score = cross_val_score(pipe_cat, X, y_log, cv=5, scoring=rmse)\n","score.mean()"]},{"cell_type":"code","execution_count":null,"id":"04311343-1f44-4b10-b2c9-551a2e175c4a","metadata":{"id":"04311343-1f44-4b10-b2c9-551a2e175c4a","outputId":"f6e88ce6-fb51-47a1-ca9c-e31d6ca6e393"},"outputs":[{"data":{"text/html":["<style>#sk-container-id-5 {color: black;background-color: white;}#sk-container-id-5 pre{padding: 0;}#sk-container-id-5 div.sk-toggleable {background-color: white;}#sk-container-id-5 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-5 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-5 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-5 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-5 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-5 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-5 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-5 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-5 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-5 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-5 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-5 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-5 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-5 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-5 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-5 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-5 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-5 div.sk-item {position: relative;z-index: 1;}#sk-container-id-5 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-5 div.sk-item::before, #sk-container-id-5 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-5 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-5 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-5 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-5 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-5 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-5 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-5 div.sk-label-container {text-align: center;}#sk-container-id-5 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-5 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-5\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>Pipeline(steps=[(&#x27;columntransformer&#x27;,\n","                 ColumnTransformer(transformers=[(&#x27;pipeline-1&#x27;,\n","                                                  Pipeline(steps=[(&#x27;knnimputer&#x27;,\n","                                                                   KNNImputer()),\n","                                                                  (&#x27;minmaxscaler&#x27;,\n","                                                                   MinMaxScaler())]),\n","                                                  [&#x27;1stFlrSF&#x27;, &#x27;2ndFlrSF&#x27;,\n","                                                   &#x27;3SsnPorch&#x27;, &#x27;BedroomAbvGr&#x27;,\n","                                                   &#x27;BsmtFinSF1&#x27;, &#x27;BsmtFinSF2&#x27;,\n","                                                   &#x27;BsmtFullBath&#x27;,\n","                                                   &#x27;BsmtHalfBath&#x27;, &#x27;BsmtUnfSF&#x27;,\n","                                                   &#x27;EnclosedPorch&#x27;,\n","                                                   &#x27;Fireplaces&#x27;, &#x27;FullBath&#x27;,\n","                                                   &#x27;GarageArea&#x27;, &#x27;GarageCars&#x27;,\n","                                                   &#x27;Gara...\n","                                                                   SimpleImputer(strategy=&#x27;most_frequent&#x27;)),\n","                                                                  (&#x27;onehotencoder&#x27;,\n","                                                                   OneHotEncoder(handle_unknown=&#x27;ignore&#x27;))]),\n","                                                  [&#x27;Alley&#x27;, &#x27;BldgType&#x27;,\n","                                                   &#x27;CentralAir&#x27;, &#x27;Condition1&#x27;,\n","                                                   &#x27;Condition2&#x27;, &#x27;Exterior1st&#x27;,\n","                                                   &#x27;Exterior2nd&#x27;, &#x27;Foundation&#x27;,\n","                                                   &#x27;GarageType&#x27;, &#x27;Heating&#x27;,\n","                                                   &#x27;HouseStyle&#x27;, &#x27;LotConfig&#x27;,\n","                                                   &#x27;MSZoning&#x27;, &#x27;MasVnrType&#x27;,\n","                                                   &#x27;MiscFeature&#x27;,\n","                                                   &#x27;Neighborhood&#x27;, &#x27;RoofMatl&#x27;,\n","                                                   &#x27;RoofStyle&#x27;, &#x27;SaleCondition&#x27;,\n","                                                   &#x27;SaleType&#x27;, &#x27;Street&#x27;,\n","                                                   &#x27;Utilities&#x27;])]))])</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-49\" type=\"checkbox\" ><label for=\"sk-estimator-id-49\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">Pipeline</label><div class=\"sk-toggleable__content\"><pre>Pipeline(steps=[(&#x27;columntransformer&#x27;,\n","                 ColumnTransformer(transformers=[(&#x27;pipeline-1&#x27;,\n","                                                  Pipeline(steps=[(&#x27;knnimputer&#x27;,\n","                                                                   KNNImputer()),\n","                                                                  (&#x27;minmaxscaler&#x27;,\n","                                                                   MinMaxScaler())]),\n","                                                  [&#x27;1stFlrSF&#x27;, &#x27;2ndFlrSF&#x27;,\n","                                                   &#x27;3SsnPorch&#x27;, &#x27;BedroomAbvGr&#x27;,\n","                                                   &#x27;BsmtFinSF1&#x27;, &#x27;BsmtFinSF2&#x27;,\n","                                                   &#x27;BsmtFullBath&#x27;,\n","                                                   &#x27;BsmtHalfBath&#x27;, &#x27;BsmtUnfSF&#x27;,\n","                                                   &#x27;EnclosedPorch&#x27;,\n","                                                   &#x27;Fireplaces&#x27;, &#x27;FullBath&#x27;,\n","                                                   &#x27;GarageArea&#x27;, &#x27;GarageCars&#x27;,\n","                                                   &#x27;Gara...\n","                                                                   SimpleImputer(strategy=&#x27;most_frequent&#x27;)),\n","                                                                  (&#x27;onehotencoder&#x27;,\n","                                                                   OneHotEncoder(handle_unknown=&#x27;ignore&#x27;))]),\n","                                                  [&#x27;Alley&#x27;, &#x27;BldgType&#x27;,\n","                                                   &#x27;CentralAir&#x27;, &#x27;Condition1&#x27;,\n","                                                   &#x27;Condition2&#x27;, &#x27;Exterior1st&#x27;,\n","                                                   &#x27;Exterior2nd&#x27;, &#x27;Foundation&#x27;,\n","                                                   &#x27;GarageType&#x27;, &#x27;Heating&#x27;,\n","                                                   &#x27;HouseStyle&#x27;, &#x27;LotConfig&#x27;,\n","                                                   &#x27;MSZoning&#x27;, &#x27;MasVnrType&#x27;,\n","                                                   &#x27;MiscFeature&#x27;,\n","                                                   &#x27;Neighborhood&#x27;, &#x27;RoofMatl&#x27;,\n","                                                   &#x27;RoofStyle&#x27;, &#x27;SaleCondition&#x27;,\n","                                                   &#x27;SaleType&#x27;, &#x27;Street&#x27;,\n","                                                   &#x27;Utilities&#x27;])]))])</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-50\" type=\"checkbox\" ><label for=\"sk-estimator-id-50\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">columntransformer: ColumnTransformer</label><div class=\"sk-toggleable__content\"><pre>ColumnTransformer(transformers=[(&#x27;pipeline-1&#x27;,\n","                                 Pipeline(steps=[(&#x27;knnimputer&#x27;, KNNImputer()),\n","                                                 (&#x27;minmaxscaler&#x27;,\n","                                                  MinMaxScaler())]),\n","                                 [&#x27;1stFlrSF&#x27;, &#x27;2ndFlrSF&#x27;, &#x27;3SsnPorch&#x27;,\n","                                  &#x27;BedroomAbvGr&#x27;, &#x27;BsmtFinSF1&#x27;, &#x27;BsmtFinSF2&#x27;,\n","                                  &#x27;BsmtFullBath&#x27;, &#x27;BsmtHalfBath&#x27;, &#x27;BsmtUnfSF&#x27;,\n","                                  &#x27;EnclosedPorch&#x27;, &#x27;Fireplaces&#x27;, &#x27;FullBath&#x27;,\n","                                  &#x27;GarageArea&#x27;, &#x27;GarageCars&#x27;, &#x27;GarageYrBlt&#x27;,\n","                                  &#x27;GrLivArea&#x27;, &#x27;HalfBath&#x27;, &#x27;Kitc...\n","                                                  SimpleImputer(strategy=&#x27;most_frequent&#x27;)),\n","                                                 (&#x27;onehotencoder&#x27;,\n","                                                  OneHotEncoder(handle_unknown=&#x27;ignore&#x27;))]),\n","                                 [&#x27;Alley&#x27;, &#x27;BldgType&#x27;, &#x27;CentralAir&#x27;,\n","                                  &#x27;Condition1&#x27;, &#x27;Condition2&#x27;, &#x27;Exterior1st&#x27;,\n","                                  &#x27;Exterior2nd&#x27;, &#x27;Foundation&#x27;, &#x27;GarageType&#x27;,\n","                                  &#x27;Heating&#x27;, &#x27;HouseStyle&#x27;, &#x27;LotConfig&#x27;,\n","                                  &#x27;MSZoning&#x27;, &#x27;MasVnrType&#x27;, &#x27;MiscFeature&#x27;,\n","                                  &#x27;Neighborhood&#x27;, &#x27;RoofMatl&#x27;, &#x27;RoofStyle&#x27;,\n","                                  &#x27;SaleCondition&#x27;, &#x27;SaleType&#x27;, &#x27;Street&#x27;,\n","                                  &#x27;Utilities&#x27;])])</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-51\" type=\"checkbox\" ><label for=\"sk-estimator-id-51\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">pipeline-1</label><div class=\"sk-toggleable__content\"><pre>[&#x27;1stFlrSF&#x27;, &#x27;2ndFlrSF&#x27;, &#x27;3SsnPorch&#x27;, &#x27;BedroomAbvGr&#x27;, &#x27;BsmtFinSF1&#x27;, &#x27;BsmtFinSF2&#x27;, &#x27;BsmtFullBath&#x27;, &#x27;BsmtHalfBath&#x27;, &#x27;BsmtUnfSF&#x27;, &#x27;EnclosedPorch&#x27;, &#x27;Fireplaces&#x27;, &#x27;FullBath&#x27;, &#x27;GarageArea&#x27;, &#x27;GarageCars&#x27;, &#x27;GarageYrBlt&#x27;, &#x27;GrLivArea&#x27;, &#x27;HalfBath&#x27;, &#x27;KitchenAbvGr&#x27;, &#x27;LotArea&#x27;, &#x27;LotFrontage&#x27;, &#x27;LowQualFinSF&#x27;, &#x27;MSSubClass&#x27;, &#x27;MasVnrArea&#x27;, &#x27;MiscVal&#x27;, &#x27;MoSold_cos&#x27;, &#x27;MoSold_sin&#x27;, &#x27;OpenPorchSF&#x27;, &#x27;OverallCond&#x27;, &#x27;OverallQual&#x27;, &#x27;PoolArea&#x27;, &#x27;ScreenPorch&#x27;, &#x27;TotRmsAbvGrd&#x27;, &#x27;TotalBsmtSF&#x27;, &#x27;WoodDeckSF&#x27;, &#x27;YearBuilt&#x27;, &#x27;YearRemodAdd&#x27;, &#x27;YrSold&#x27;]</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-52\" type=\"checkbox\" ><label for=\"sk-estimator-id-52\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">KNNImputer</label><div class=\"sk-toggleable__content\"><pre>KNNImputer()</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-53\" type=\"checkbox\" ><label for=\"sk-estimator-id-53\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">MinMaxScaler</label><div class=\"sk-toggleable__content\"><pre>MinMaxScaler()</pre></div></div></div></div></div></div></div></div><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-54\" type=\"checkbox\" ><label for=\"sk-estimator-id-54\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">pipeline-2</label><div class=\"sk-toggleable__content\"><pre>[&#x27;BsmtCond&#x27;, &#x27;BsmtExposure&#x27;, &#x27;BsmtFinType1&#x27;, &#x27;BsmtFinType2&#x27;, &#x27;BsmtQual&#x27;, &#x27;Electrical&#x27;, &#x27;ExterCond&#x27;, &#x27;ExterQual&#x27;, &#x27;Fence&#x27;, &#x27;FireplaceQu&#x27;, &#x27;Functional&#x27;, &#x27;GarageCond&#x27;, &#x27;GarageFinish&#x27;, &#x27;GarageQual&#x27;, &#x27;HeatingQC&#x27;, &#x27;KitchenQual&#x27;, &#x27;LandContour&#x27;, &#x27;LandSlope&#x27;, &#x27;LotShape&#x27;, &#x27;PavedDrive&#x27;, &#x27;PoolQC&#x27;]</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-55\" type=\"checkbox\" ><label for=\"sk-estimator-id-55\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">SimpleImputer</label><div class=\"sk-toggleable__content\"><pre>SimpleImputer(fill_value=&#x27;missing&#x27;, strategy=&#x27;constant&#x27;)</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-56\" type=\"checkbox\" ><label for=\"sk-estimator-id-56\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">OrdinalEncoder</label><div class=\"sk-toggleable__content\"><pre>OrdinalEncoder(categories=[[&#x27;missing&#x27;, &#x27;Po&#x27;, &#x27;Fa&#x27;, &#x27;TA&#x27;, &#x27;Gd&#x27;],\n","                           [&#x27;missing&#x27;, &#x27;No&#x27;, &#x27;Mn&#x27;, &#x27;Av&#x27;, &#x27;Gd&#x27;],\n","                           [&#x27;missing&#x27;, &#x27;Unf&#x27;, &#x27;LwQ&#x27;, &#x27;Rec&#x27;, &#x27;BLQ&#x27;, &#x27;ALQ&#x27;,\n","                            &#x27;GLQ&#x27;],\n","                           [&#x27;missing&#x27;, &#x27;Unf&#x27;, &#x27;LwQ&#x27;, &#x27;Rec&#x27;, &#x27;BLQ&#x27;, &#x27;ALQ&#x27;,\n","                            &#x27;GLQ&#x27;],\n","                           [&#x27;missing&#x27;, &#x27;Fa&#x27;, &#x27;TA&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;],\n","                           [&#x27;missing&#x27;, &#x27;Mix&#x27;, &#x27;FuseP&#x27;, &#x27;FuseF&#x27;, &#x27;FuseA&#x27;,\n","                            &#x27;SBrkr&#x27;],\n","                           [&#x27;missing&#x27;, &#x27;Po&#x27;, &#x27;Fa&#x27;, &#x27;TA&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;],\n","                           [&#x27;missing&#x27;, &#x27;Fa&#x27;, &#x27;TA&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;],\n","                           [&#x27;missing&#x27;, &#x27;...\n","                           [&#x27;missing&#x27;, &#x27;Unf&#x27;, &#x27;RFn&#x27;, &#x27;Fin&#x27;],\n","                           [&#x27;missing&#x27;, &#x27;Po&#x27;, &#x27;Fa&#x27;, &#x27;TA&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;],\n","                           [&#x27;missing&#x27;, &#x27;Po&#x27;, &#x27;Fa&#x27;, &#x27;TA&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;],\n","                           [&#x27;missing&#x27;, &#x27;Fa&#x27;, &#x27;TA&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;],\n","                           [&#x27;missing&#x27;, &#x27;Low&#x27;, &#x27;Bnk&#x27;, &#x27;HLS&#x27;, &#x27;Lvl&#x27;],\n","                           [&#x27;missing&#x27;, &#x27;Sev&#x27;, &#x27;Mod&#x27;, &#x27;Gtl&#x27;],\n","                           [&#x27;missing&#x27;, &#x27;IR3&#x27;, &#x27;IR2&#x27;, &#x27;IR1&#x27;, &#x27;Reg&#x27;],\n","                           [&#x27;missing&#x27;, &#x27;N&#x27;, &#x27;P&#x27;, &#x27;Y&#x27;],\n","                           [&#x27;missing&#x27;, &#x27;Fa&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;]],\n","               dtype=&lt;class &#x27;numpy.int64&#x27;&gt;, handle_unknown=&#x27;use_encoded_value&#x27;,\n","               unknown_value=-1)</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-57\" type=\"checkbox\" ><label for=\"sk-estimator-id-57\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">MinMaxScaler</label><div class=\"sk-toggleable__content\"><pre>MinMaxScaler()</pre></div></div></div></div></div></div></div></div><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-58\" type=\"checkbox\" ><label for=\"sk-estimator-id-58\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">pipeline-3</label><div class=\"sk-toggleable__content\"><pre>[&#x27;Alley&#x27;, &#x27;BldgType&#x27;, &#x27;CentralAir&#x27;, &#x27;Condition1&#x27;, &#x27;Condition2&#x27;, &#x27;Exterior1st&#x27;, &#x27;Exterior2nd&#x27;, &#x27;Foundation&#x27;, &#x27;GarageType&#x27;, &#x27;Heating&#x27;, &#x27;HouseStyle&#x27;, &#x27;LotConfig&#x27;, &#x27;MSZoning&#x27;, &#x27;MasVnrType&#x27;, &#x27;MiscFeature&#x27;, &#x27;Neighborhood&#x27;, &#x27;RoofMatl&#x27;, &#x27;RoofStyle&#x27;, &#x27;SaleCondition&#x27;, &#x27;SaleType&#x27;, &#x27;Street&#x27;, &#x27;Utilities&#x27;]</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-59\" type=\"checkbox\" ><label for=\"sk-estimator-id-59\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">SimpleImputer</label><div class=\"sk-toggleable__content\"><pre>SimpleImputer(strategy=&#x27;most_frequent&#x27;)</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-60\" type=\"checkbox\" ><label for=\"sk-estimator-id-60\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">OneHotEncoder</label><div class=\"sk-toggleable__content\"><pre>OneHotEncoder(handle_unknown=&#x27;ignore&#x27;)</pre></div></div></div></div></div></div></div></div></div></div></div></div></div></div>"],"text/plain":["Pipeline(steps=[('columntransformer',\n","                 ColumnTransformer(transformers=[('pipeline-1',\n","                                                  Pipeline(steps=[('knnimputer',\n","                                                                   KNNImputer()),\n","                                                                  ('minmaxscaler',\n","                                                                   MinMaxScaler())]),\n","                                                  ['1stFlrSF', '2ndFlrSF',\n","                                                   '3SsnPorch', 'BedroomAbvGr',\n","                                                   'BsmtFinSF1', 'BsmtFinSF2',\n","                                                   'BsmtFullBath',\n","                                                   'BsmtHalfBath', 'BsmtUnfSF',\n","                                                   'EnclosedPorch',\n","                                                   'Fireplaces', 'FullBath',\n","                                                   'GarageArea', 'GarageCars',\n","                                                   'Gara...\n","                                                                   SimpleImputer(strategy='most_frequent')),\n","                                                                  ('onehotencoder',\n","                                                                   OneHotEncoder(handle_unknown='ignore'))]),\n","                                                  ['Alley', 'BldgType',\n","                                                   'CentralAir', 'Condition1',\n","                                                   'Condition2', 'Exterior1st',\n","                                                   'Exterior2nd', 'Foundation',\n","                                                   'GarageType', 'Heating',\n","                                                   'HouseStyle', 'LotConfig',\n","                                                   'MSZoning', 'MasVnrType',\n","                                                   'MiscFeature',\n","                                                   'Neighborhood', 'RoofMatl',\n","                                                   'RoofStyle', 'SaleCondition',\n","                                                   'SaleType', 'Street',\n","                                                   'Utilities'])]))])"]},"execution_count":117,"metadata":{},"output_type":"execute_result"}],"source":["preproc"]},{"cell_type":"code","execution_count":null,"id":"43c14417-ca3e-44c2-b014-e6c94179e1d4","metadata":{"id":"43c14417-ca3e-44c2-b014-e6c94179e1d4"},"outputs":[],"source":[""]},{"cell_type":"markdown","id":"80e38591-4fbb-475a-8490-a1d5eca0122b","metadata":{"id":"80e38591-4fbb-475a-8490-a1d5eca0122b"},"source":["## Test Blending"]},{"cell_type":"code","execution_count":null,"id":"88a8c9e4-a594-4f82-a7fc-84cfe6c69869","metadata":{"id":"88a8c9e4-a594-4f82-a7fc-84cfe6c69869","outputId":"3209ace4-1333-4a8d-9366-a98d31edde6f"},"outputs":[{"data":{"text/plain":["(1460, 80)"]},"execution_count":118,"metadata":{},"output_type":"execute_result"}],"source":["X.shape"]},{"cell_type":"code","execution_count":null,"id":"cb33477f-3eec-474b-9ff8-63c82aa03268","metadata":{"id":"cb33477f-3eec-474b-9ff8-63c82aa03268","outputId":"9d52441c-cbd2-4edd-8d1b-7f9b59e7f6bf"},"outputs":[{"data":{"text/plain":["(1460, 217)"]},"execution_count":119,"metadata":{},"output_type":"execute_result"}],"source":["preproc.fit(X, y_log)\n","X_processed = preproc.transform(X)\n","X_processed.shape"]},{"cell_type":"code","execution_count":null,"id":"f5f37e93-a53e-48d0-b1a9-00aa7a3eecc7","metadata":{"id":"f5f37e93-a53e-48d0-b1a9-00aa7a3eecc7"},"outputs":[],"source":["train = X_processed\n","target = y_log"]},{"cell_type":"code","execution_count":null,"id":"b07496a1-a83f-47b4-9758-7c40a9fe8bd5","metadata":{"id":"b07496a1-a83f-47b4-9758-7c40a9fe8bd5"},"outputs":[],"source":["import warnings\n","warnings.filterwarnings('ignore')\n","warnings.simplefilter(action='ignore', category=FutureWarning)\n","\n","gboost = GradientBoostingRegressor(\n","    learning_rate = 0.092, \n","    loss = 'squared_error',\n","    max_depth = 4,\n","    max_features = 23,\n","    min_samples_split = 2,\n","    n_estimators = 274,\n","    subsample = 1).fit(train, target)\n","ridge = Ridge().fit(train, target)\n","svm = SVR(C=2, epsilon=0.05).fit(train, target)\n","adaboost = AdaBoostRegressor(base_estimator=DecisionTreeRegressor(max_depth=None)).fit(train, target)\n","knn = KNeighborsRegressor().fit(train, target)\n","xgb = XGBRegressor(max_depth=3, n_estimators=2000, learning_rate=0.05).fit(train, target)"]},{"cell_type":"code","execution_count":null,"id":"94c0b971-e713-404a-82c0-7f4700c5dd2f","metadata":{"id":"94c0b971-e713-404a-82c0-7f4700c5dd2f","outputId":"0e296cbc-e9f2-40a0-a568-de1846120979"},"outputs":[{"name":"stdout","output_type":"stream","text":["RMSE score (of log) on train data:\n","0.05031734421397502\n"]}],"source":["import warnings\n","warnings.filterwarnings('ignore')\n","warnings.simplefilter(action='ignore', category=FutureWarning)\n","\n","def blended_predict(X):\n","    return ((0.1 * ridge.predict(X)) + \\\n","            (0.2 * gboost.predict(X)) + \\\n","            (0.1 * adaboost.predict(X)) + \\\n","            (0.2 * xgb.predict(X)) + \\\n","            (0.1 * knn.predict(X)) + \\\n","            (0.3 * svm.predict(X)))\n","\n","def rmse_blend(y, y_pred):\n","    return np.sqrt(mean_squared_error(y, y_pred))\n","\n","# Get final precitions from the blended model\n","blended_score = rmse_blend(target, blended_predict(train))\n","print('RMSE score (of log) on train data:')\n","print(blended_score)"]},{"cell_type":"markdown","id":"8b45de0d-4cc9-4e57-9f49-285f3d12b369","metadata":{"id":"8b45de0d-4cc9-4e57-9f49-285f3d12b369"},"source":["# Kaggle Submission"]},{"cell_type":"code","execution_count":null,"id":"9dac9986-8286-44ed-b737-a216c113c2ec","metadata":{"id":"9dac9986-8286-44ed-b737-a216c113c2ec","outputId":"ec4224c6-6870-4adf-e681-97028e411455"},"outputs":[{"data":{"text/html":["<style>#sk-container-id-6 {color: black;background-color: white;}#sk-container-id-6 pre{padding: 0;}#sk-container-id-6 div.sk-toggleable {background-color: white;}#sk-container-id-6 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-6 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-6 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-6 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-6 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-6 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-6 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-6 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-6 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-6 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-6 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-6 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-6 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-6 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-6 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-6 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-6 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-6 div.sk-item {position: relative;z-index: 1;}#sk-container-id-6 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-6 div.sk-item::before, #sk-container-id-6 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-6 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-6 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-6 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-6 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-6 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-6 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-6 div.sk-label-container {text-align: center;}#sk-container-id-6 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-6 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-6\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>Pipeline(steps=[(&#x27;columntransformer&#x27;,\n","                 ColumnTransformer(transformers=[(&#x27;pipeline-1&#x27;,\n","                                                  Pipeline(steps=[(&#x27;knnimputer&#x27;,\n","                                                                   KNNImputer()),\n","                                                                  (&#x27;minmaxscaler&#x27;,\n","                                                                   MinMaxScaler())]),\n","                                                  [&#x27;1stFlrSF&#x27;, &#x27;2ndFlrSF&#x27;,\n","                                                   &#x27;3SsnPorch&#x27;, &#x27;BedroomAbvGr&#x27;,\n","                                                   &#x27;BsmtFinSF1&#x27;, &#x27;BsmtFinSF2&#x27;,\n","                                                   &#x27;BsmtFullBath&#x27;,\n","                                                   &#x27;BsmtHalfBath&#x27;, &#x27;BsmtUnfSF&#x27;,\n","                                                   &#x27;EnclosedPorch&#x27;,\n","                                                   &#x27;Fireplaces&#x27;, &#x27;FullBath&#x27;,\n","                                                   &#x27;GarageArea&#x27;, &#x27;GarageCars&#x27;,\n","                                                   &#x27;Gara...\n","                                                                   SimpleImputer(strategy=&#x27;most_frequent&#x27;)),\n","                                                                  (&#x27;onehotencoder&#x27;,\n","                                                                   OneHotEncoder(handle_unknown=&#x27;ignore&#x27;))]),\n","                                                  [&#x27;Alley&#x27;, &#x27;BldgType&#x27;,\n","                                                   &#x27;CentralAir&#x27;, &#x27;Condition1&#x27;,\n","                                                   &#x27;Condition2&#x27;, &#x27;Exterior1st&#x27;,\n","                                                   &#x27;Exterior2nd&#x27;, &#x27;Foundation&#x27;,\n","                                                   &#x27;GarageType&#x27;, &#x27;Heating&#x27;,\n","                                                   &#x27;HouseStyle&#x27;, &#x27;LotConfig&#x27;,\n","                                                   &#x27;MSZoning&#x27;, &#x27;MasVnrType&#x27;,\n","                                                   &#x27;MiscFeature&#x27;,\n","                                                   &#x27;Neighborhood&#x27;, &#x27;RoofMatl&#x27;,\n","                                                   &#x27;RoofStyle&#x27;, &#x27;SaleCondition&#x27;,\n","                                                   &#x27;SaleType&#x27;, &#x27;Street&#x27;,\n","                                                   &#x27;Utilities&#x27;])]))])</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-61\" type=\"checkbox\" ><label for=\"sk-estimator-id-61\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">Pipeline</label><div class=\"sk-toggleable__content\"><pre>Pipeline(steps=[(&#x27;columntransformer&#x27;,\n","                 ColumnTransformer(transformers=[(&#x27;pipeline-1&#x27;,\n","                                                  Pipeline(steps=[(&#x27;knnimputer&#x27;,\n","                                                                   KNNImputer()),\n","                                                                  (&#x27;minmaxscaler&#x27;,\n","                                                                   MinMaxScaler())]),\n","                                                  [&#x27;1stFlrSF&#x27;, &#x27;2ndFlrSF&#x27;,\n","                                                   &#x27;3SsnPorch&#x27;, &#x27;BedroomAbvGr&#x27;,\n","                                                   &#x27;BsmtFinSF1&#x27;, &#x27;BsmtFinSF2&#x27;,\n","                                                   &#x27;BsmtFullBath&#x27;,\n","                                                   &#x27;BsmtHalfBath&#x27;, &#x27;BsmtUnfSF&#x27;,\n","                                                   &#x27;EnclosedPorch&#x27;,\n","                                                   &#x27;Fireplaces&#x27;, &#x27;FullBath&#x27;,\n","                                                   &#x27;GarageArea&#x27;, &#x27;GarageCars&#x27;,\n","                                                   &#x27;Gara...\n","                                                                   SimpleImputer(strategy=&#x27;most_frequent&#x27;)),\n","                                                                  (&#x27;onehotencoder&#x27;,\n","                                                                   OneHotEncoder(handle_unknown=&#x27;ignore&#x27;))]),\n","                                                  [&#x27;Alley&#x27;, &#x27;BldgType&#x27;,\n","                                                   &#x27;CentralAir&#x27;, &#x27;Condition1&#x27;,\n","                                                   &#x27;Condition2&#x27;, &#x27;Exterior1st&#x27;,\n","                                                   &#x27;Exterior2nd&#x27;, &#x27;Foundation&#x27;,\n","                                                   &#x27;GarageType&#x27;, &#x27;Heating&#x27;,\n","                                                   &#x27;HouseStyle&#x27;, &#x27;LotConfig&#x27;,\n","                                                   &#x27;MSZoning&#x27;, &#x27;MasVnrType&#x27;,\n","                                                   &#x27;MiscFeature&#x27;,\n","                                                   &#x27;Neighborhood&#x27;, &#x27;RoofMatl&#x27;,\n","                                                   &#x27;RoofStyle&#x27;, &#x27;SaleCondition&#x27;,\n","                                                   &#x27;SaleType&#x27;, &#x27;Street&#x27;,\n","                                                   &#x27;Utilities&#x27;])]))])</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-62\" type=\"checkbox\" ><label for=\"sk-estimator-id-62\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">columntransformer: ColumnTransformer</label><div class=\"sk-toggleable__content\"><pre>ColumnTransformer(transformers=[(&#x27;pipeline-1&#x27;,\n","                                 Pipeline(steps=[(&#x27;knnimputer&#x27;, KNNImputer()),\n","                                                 (&#x27;minmaxscaler&#x27;,\n","                                                  MinMaxScaler())]),\n","                                 [&#x27;1stFlrSF&#x27;, &#x27;2ndFlrSF&#x27;, &#x27;3SsnPorch&#x27;,\n","                                  &#x27;BedroomAbvGr&#x27;, &#x27;BsmtFinSF1&#x27;, &#x27;BsmtFinSF2&#x27;,\n","                                  &#x27;BsmtFullBath&#x27;, &#x27;BsmtHalfBath&#x27;, &#x27;BsmtUnfSF&#x27;,\n","                                  &#x27;EnclosedPorch&#x27;, &#x27;Fireplaces&#x27;, &#x27;FullBath&#x27;,\n","                                  &#x27;GarageArea&#x27;, &#x27;GarageCars&#x27;, &#x27;GarageYrBlt&#x27;,\n","                                  &#x27;GrLivArea&#x27;, &#x27;HalfBath&#x27;, &#x27;Kitc...\n","                                                  SimpleImputer(strategy=&#x27;most_frequent&#x27;)),\n","                                                 (&#x27;onehotencoder&#x27;,\n","                                                  OneHotEncoder(handle_unknown=&#x27;ignore&#x27;))]),\n","                                 [&#x27;Alley&#x27;, &#x27;BldgType&#x27;, &#x27;CentralAir&#x27;,\n","                                  &#x27;Condition1&#x27;, &#x27;Condition2&#x27;, &#x27;Exterior1st&#x27;,\n","                                  &#x27;Exterior2nd&#x27;, &#x27;Foundation&#x27;, &#x27;GarageType&#x27;,\n","                                  &#x27;Heating&#x27;, &#x27;HouseStyle&#x27;, &#x27;LotConfig&#x27;,\n","                                  &#x27;MSZoning&#x27;, &#x27;MasVnrType&#x27;, &#x27;MiscFeature&#x27;,\n","                                  &#x27;Neighborhood&#x27;, &#x27;RoofMatl&#x27;, &#x27;RoofStyle&#x27;,\n","                                  &#x27;SaleCondition&#x27;, &#x27;SaleType&#x27;, &#x27;Street&#x27;,\n","                                  &#x27;Utilities&#x27;])])</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-63\" type=\"checkbox\" ><label for=\"sk-estimator-id-63\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">pipeline-1</label><div class=\"sk-toggleable__content\"><pre>[&#x27;1stFlrSF&#x27;, &#x27;2ndFlrSF&#x27;, &#x27;3SsnPorch&#x27;, &#x27;BedroomAbvGr&#x27;, &#x27;BsmtFinSF1&#x27;, &#x27;BsmtFinSF2&#x27;, &#x27;BsmtFullBath&#x27;, &#x27;BsmtHalfBath&#x27;, &#x27;BsmtUnfSF&#x27;, &#x27;EnclosedPorch&#x27;, &#x27;Fireplaces&#x27;, &#x27;FullBath&#x27;, &#x27;GarageArea&#x27;, &#x27;GarageCars&#x27;, &#x27;GarageYrBlt&#x27;, &#x27;GrLivArea&#x27;, &#x27;HalfBath&#x27;, &#x27;KitchenAbvGr&#x27;, &#x27;LotArea&#x27;, &#x27;LotFrontage&#x27;, &#x27;LowQualFinSF&#x27;, &#x27;MSSubClass&#x27;, &#x27;MasVnrArea&#x27;, &#x27;MiscVal&#x27;, &#x27;MoSold_cos&#x27;, &#x27;MoSold_sin&#x27;, &#x27;OpenPorchSF&#x27;, &#x27;OverallCond&#x27;, &#x27;OverallQual&#x27;, &#x27;PoolArea&#x27;, &#x27;ScreenPorch&#x27;, &#x27;TotRmsAbvGrd&#x27;, &#x27;TotalBsmtSF&#x27;, &#x27;WoodDeckSF&#x27;, &#x27;YearBuilt&#x27;, &#x27;YearRemodAdd&#x27;, &#x27;YrSold&#x27;]</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-64\" type=\"checkbox\" ><label for=\"sk-estimator-id-64\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">KNNImputer</label><div class=\"sk-toggleable__content\"><pre>KNNImputer()</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-65\" type=\"checkbox\" ><label for=\"sk-estimator-id-65\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">MinMaxScaler</label><div class=\"sk-toggleable__content\"><pre>MinMaxScaler()</pre></div></div></div></div></div></div></div></div><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-66\" type=\"checkbox\" ><label for=\"sk-estimator-id-66\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">pipeline-2</label><div class=\"sk-toggleable__content\"><pre>[&#x27;BsmtCond&#x27;, &#x27;BsmtExposure&#x27;, &#x27;BsmtFinType1&#x27;, &#x27;BsmtFinType2&#x27;, &#x27;BsmtQual&#x27;, &#x27;Electrical&#x27;, &#x27;ExterCond&#x27;, &#x27;ExterQual&#x27;, &#x27;Fence&#x27;, &#x27;FireplaceQu&#x27;, &#x27;Functional&#x27;, &#x27;GarageCond&#x27;, &#x27;GarageFinish&#x27;, &#x27;GarageQual&#x27;, &#x27;HeatingQC&#x27;, &#x27;KitchenQual&#x27;, &#x27;LandContour&#x27;, &#x27;LandSlope&#x27;, &#x27;LotShape&#x27;, &#x27;PavedDrive&#x27;, &#x27;PoolQC&#x27;]</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-67\" type=\"checkbox\" ><label for=\"sk-estimator-id-67\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">SimpleImputer</label><div class=\"sk-toggleable__content\"><pre>SimpleImputer(fill_value=&#x27;missing&#x27;, strategy=&#x27;constant&#x27;)</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-68\" type=\"checkbox\" ><label for=\"sk-estimator-id-68\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">OrdinalEncoder</label><div class=\"sk-toggleable__content\"><pre>OrdinalEncoder(categories=[[&#x27;missing&#x27;, &#x27;Po&#x27;, &#x27;Fa&#x27;, &#x27;TA&#x27;, &#x27;Gd&#x27;],\n","                           [&#x27;missing&#x27;, &#x27;No&#x27;, &#x27;Mn&#x27;, &#x27;Av&#x27;, &#x27;Gd&#x27;],\n","                           [&#x27;missing&#x27;, &#x27;Unf&#x27;, &#x27;LwQ&#x27;, &#x27;Rec&#x27;, &#x27;BLQ&#x27;, &#x27;ALQ&#x27;,\n","                            &#x27;GLQ&#x27;],\n","                           [&#x27;missing&#x27;, &#x27;Unf&#x27;, &#x27;LwQ&#x27;, &#x27;Rec&#x27;, &#x27;BLQ&#x27;, &#x27;ALQ&#x27;,\n","                            &#x27;GLQ&#x27;],\n","                           [&#x27;missing&#x27;, &#x27;Fa&#x27;, &#x27;TA&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;],\n","                           [&#x27;missing&#x27;, &#x27;Mix&#x27;, &#x27;FuseP&#x27;, &#x27;FuseF&#x27;, &#x27;FuseA&#x27;,\n","                            &#x27;SBrkr&#x27;],\n","                           [&#x27;missing&#x27;, &#x27;Po&#x27;, &#x27;Fa&#x27;, &#x27;TA&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;],\n","                           [&#x27;missing&#x27;, &#x27;Fa&#x27;, &#x27;TA&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;],\n","                           [&#x27;missing&#x27;, &#x27;...\n","                           [&#x27;missing&#x27;, &#x27;Unf&#x27;, &#x27;RFn&#x27;, &#x27;Fin&#x27;],\n","                           [&#x27;missing&#x27;, &#x27;Po&#x27;, &#x27;Fa&#x27;, &#x27;TA&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;],\n","                           [&#x27;missing&#x27;, &#x27;Po&#x27;, &#x27;Fa&#x27;, &#x27;TA&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;],\n","                           [&#x27;missing&#x27;, &#x27;Fa&#x27;, &#x27;TA&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;],\n","                           [&#x27;missing&#x27;, &#x27;Low&#x27;, &#x27;Bnk&#x27;, &#x27;HLS&#x27;, &#x27;Lvl&#x27;],\n","                           [&#x27;missing&#x27;, &#x27;Sev&#x27;, &#x27;Mod&#x27;, &#x27;Gtl&#x27;],\n","                           [&#x27;missing&#x27;, &#x27;IR3&#x27;, &#x27;IR2&#x27;, &#x27;IR1&#x27;, &#x27;Reg&#x27;],\n","                           [&#x27;missing&#x27;, &#x27;N&#x27;, &#x27;P&#x27;, &#x27;Y&#x27;],\n","                           [&#x27;missing&#x27;, &#x27;Fa&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;]],\n","               dtype=&lt;class &#x27;numpy.int64&#x27;&gt;, handle_unknown=&#x27;use_encoded_value&#x27;,\n","               unknown_value=-1)</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-69\" type=\"checkbox\" ><label for=\"sk-estimator-id-69\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">MinMaxScaler</label><div class=\"sk-toggleable__content\"><pre>MinMaxScaler()</pre></div></div></div></div></div></div></div></div><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-70\" type=\"checkbox\" ><label for=\"sk-estimator-id-70\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">pipeline-3</label><div class=\"sk-toggleable__content\"><pre>[&#x27;Alley&#x27;, &#x27;BldgType&#x27;, &#x27;CentralAir&#x27;, &#x27;Condition1&#x27;, &#x27;Condition2&#x27;, &#x27;Exterior1st&#x27;, &#x27;Exterior2nd&#x27;, &#x27;Foundation&#x27;, &#x27;GarageType&#x27;, &#x27;Heating&#x27;, &#x27;HouseStyle&#x27;, &#x27;LotConfig&#x27;, &#x27;MSZoning&#x27;, &#x27;MasVnrType&#x27;, &#x27;MiscFeature&#x27;, &#x27;Neighborhood&#x27;, &#x27;RoofMatl&#x27;, &#x27;RoofStyle&#x27;, &#x27;SaleCondition&#x27;, &#x27;SaleType&#x27;, &#x27;Street&#x27;, &#x27;Utilities&#x27;]</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-71\" type=\"checkbox\" ><label for=\"sk-estimator-id-71\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">SimpleImputer</label><div class=\"sk-toggleable__content\"><pre>SimpleImputer(strategy=&#x27;most_frequent&#x27;)</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-72\" type=\"checkbox\" ><label for=\"sk-estimator-id-72\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">OneHotEncoder</label><div class=\"sk-toggleable__content\"><pre>OneHotEncoder(handle_unknown=&#x27;ignore&#x27;)</pre></div></div></div></div></div></div></div></div></div></div></div></div></div></div>"],"text/plain":["Pipeline(steps=[('columntransformer',\n","                 ColumnTransformer(transformers=[('pipeline-1',\n","                                                  Pipeline(steps=[('knnimputer',\n","                                                                   KNNImputer()),\n","                                                                  ('minmaxscaler',\n","                                                                   MinMaxScaler())]),\n","                                                  ['1stFlrSF', '2ndFlrSF',\n","                                                   '3SsnPorch', 'BedroomAbvGr',\n","                                                   'BsmtFinSF1', 'BsmtFinSF2',\n","                                                   'BsmtFullBath',\n","                                                   'BsmtHalfBath', 'BsmtUnfSF',\n","                                                   'EnclosedPorch',\n","                                                   'Fireplaces', 'FullBath',\n","                                                   'GarageArea', 'GarageCars',\n","                                                   'Gara...\n","                                                                   SimpleImputer(strategy='most_frequent')),\n","                                                                  ('onehotencoder',\n","                                                                   OneHotEncoder(handle_unknown='ignore'))]),\n","                                                  ['Alley', 'BldgType',\n","                                                   'CentralAir', 'Condition1',\n","                                                   'Condition2', 'Exterior1st',\n","                                                   'Exterior2nd', 'Foundation',\n","                                                   'GarageType', 'Heating',\n","                                                   'HouseStyle', 'LotConfig',\n","                                                   'MSZoning', 'MasVnrType',\n","                                                   'MiscFeature',\n","                                                   'Neighborhood', 'RoofMatl',\n","                                                   'RoofStyle', 'SaleCondition',\n","                                                   'SaleType', 'Street',\n","                                                   'Utilities'])]))])"]},"execution_count":123,"metadata":{},"output_type":"execute_result"}],"source":["preproc"]},{"cell_type":"code","execution_count":null,"id":"7517f929-2cae-49f3-a9f7-0dec40c7f34b","metadata":{"id":"7517f929-2cae-49f3-a9f7-0dec40c7f34b","outputId":"a0b22f4d-71eb-4337-dfa1-1ec2ac1c1d1b"},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>MSSubClass</th>\n","      <th>MSZoning</th>\n","      <th>LotFrontage</th>\n","      <th>LotArea</th>\n","      <th>Street</th>\n","      <th>Alley</th>\n","      <th>LotShape</th>\n","      <th>LandContour</th>\n","      <th>Utilities</th>\n","      <th>LotConfig</th>\n","      <th>...</th>\n","      <th>ScreenPorch</th>\n","      <th>PoolArea</th>\n","      <th>PoolQC</th>\n","      <th>Fence</th>\n","      <th>MiscFeature</th>\n","      <th>MiscVal</th>\n","      <th>MoSold</th>\n","      <th>YrSold</th>\n","      <th>SaleType</th>\n","      <th>SaleCondition</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>20</td>\n","      <td>RH</td>\n","      <td>80.0</td>\n","      <td>11622</td>\n","      <td>Pave</td>\n","      <td>NaN</td>\n","      <td>Reg</td>\n","      <td>Lvl</td>\n","      <td>AllPub</td>\n","      <td>Inside</td>\n","      <td>...</td>\n","      <td>120</td>\n","      <td>0</td>\n","      <td>NaN</td>\n","      <td>MnPrv</td>\n","      <td>NaN</td>\n","      <td>0</td>\n","      <td>6</td>\n","      <td>2010</td>\n","      <td>WD</td>\n","      <td>Normal</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>20</td>\n","      <td>RL</td>\n","      <td>81.0</td>\n","      <td>14267</td>\n","      <td>Pave</td>\n","      <td>NaN</td>\n","      <td>IR1</td>\n","      <td>Lvl</td>\n","      <td>AllPub</td>\n","      <td>Corner</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>NaN</td>\n","      <td>NaN</td>\n","      <td>Gar2</td>\n","      <td>12500</td>\n","      <td>6</td>\n","      <td>2010</td>\n","      <td>WD</td>\n","      <td>Normal</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>60</td>\n","      <td>RL</td>\n","      <td>74.0</td>\n","      <td>13830</td>\n","      <td>Pave</td>\n","      <td>NaN</td>\n","      <td>IR1</td>\n","      <td>Lvl</td>\n","      <td>AllPub</td>\n","      <td>Inside</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>NaN</td>\n","      <td>MnPrv</td>\n","      <td>NaN</td>\n","      <td>0</td>\n","      <td>3</td>\n","      <td>2010</td>\n","      <td>WD</td>\n","      <td>Normal</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>60</td>\n","      <td>RL</td>\n","      <td>78.0</td>\n","      <td>9978</td>\n","      <td>Pave</td>\n","      <td>NaN</td>\n","      <td>IR1</td>\n","      <td>Lvl</td>\n","      <td>AllPub</td>\n","      <td>Inside</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>NaN</td>\n","      <td>NaN</td>\n","      <td>NaN</td>\n","      <td>0</td>\n","      <td>6</td>\n","      <td>2010</td>\n","      <td>WD</td>\n","      <td>Normal</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>120</td>\n","      <td>RL</td>\n","      <td>43.0</td>\n","      <td>5005</td>\n","      <td>Pave</td>\n","      <td>NaN</td>\n","      <td>IR1</td>\n","      <td>HLS</td>\n","      <td>AllPub</td>\n","      <td>Inside</td>\n","      <td>...</td>\n","      <td>144</td>\n","      <td>0</td>\n","      <td>NaN</td>\n","      <td>NaN</td>\n","      <td>NaN</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>2010</td>\n","      <td>WD</td>\n","      <td>Normal</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>1454</th>\n","      <td>160</td>\n","      <td>RM</td>\n","      <td>21.0</td>\n","      <td>1936</td>\n","      <td>Pave</td>\n","      <td>NaN</td>\n","      <td>Reg</td>\n","      <td>Lvl</td>\n","      <td>AllPub</td>\n","      <td>Inside</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>NaN</td>\n","      <td>NaN</td>\n","      <td>NaN</td>\n","      <td>0</td>\n","      <td>6</td>\n","      <td>2006</td>\n","      <td>WD</td>\n","      <td>Normal</td>\n","    </tr>\n","    <tr>\n","      <th>1455</th>\n","      <td>160</td>\n","      <td>RM</td>\n","      <td>21.0</td>\n","      <td>1894</td>\n","      <td>Pave</td>\n","      <td>NaN</td>\n","      <td>Reg</td>\n","      <td>Lvl</td>\n","      <td>AllPub</td>\n","      <td>Inside</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>NaN</td>\n","      <td>NaN</td>\n","      <td>NaN</td>\n","      <td>0</td>\n","      <td>4</td>\n","      <td>2006</td>\n","      <td>WD</td>\n","      <td>Abnorml</td>\n","    </tr>\n","    <tr>\n","      <th>1456</th>\n","      <td>20</td>\n","      <td>RL</td>\n","      <td>160.0</td>\n","      <td>20000</td>\n","      <td>Pave</td>\n","      <td>NaN</td>\n","      <td>Reg</td>\n","      <td>Lvl</td>\n","      <td>AllPub</td>\n","      <td>Inside</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>NaN</td>\n","      <td>NaN</td>\n","      <td>NaN</td>\n","      <td>0</td>\n","      <td>9</td>\n","      <td>2006</td>\n","      <td>WD</td>\n","      <td>Abnorml</td>\n","    </tr>\n","    <tr>\n","      <th>1457</th>\n","      <td>85</td>\n","      <td>RL</td>\n","      <td>62.0</td>\n","      <td>10441</td>\n","      <td>Pave</td>\n","      <td>NaN</td>\n","      <td>Reg</td>\n","      <td>Lvl</td>\n","      <td>AllPub</td>\n","      <td>Inside</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>NaN</td>\n","      <td>MnPrv</td>\n","      <td>Shed</td>\n","      <td>700</td>\n","      <td>7</td>\n","      <td>2006</td>\n","      <td>WD</td>\n","      <td>Normal</td>\n","    </tr>\n","    <tr>\n","      <th>1458</th>\n","      <td>60</td>\n","      <td>RL</td>\n","      <td>74.0</td>\n","      <td>9627</td>\n","      <td>Pave</td>\n","      <td>NaN</td>\n","      <td>Reg</td>\n","      <td>Lvl</td>\n","      <td>AllPub</td>\n","      <td>Inside</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>NaN</td>\n","      <td>NaN</td>\n","      <td>NaN</td>\n","      <td>0</td>\n","      <td>11</td>\n","      <td>2006</td>\n","      <td>WD</td>\n","      <td>Normal</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>1459 rows × 79 columns</p>\n","</div>"],"text/plain":["      MSSubClass MSZoning  LotFrontage  LotArea Street Alley LotShape  \\\n","0             20       RH         80.0    11622   Pave   NaN      Reg   \n","1             20       RL         81.0    14267   Pave   NaN      IR1   \n","2             60       RL         74.0    13830   Pave   NaN      IR1   \n","3             60       RL         78.0     9978   Pave   NaN      IR1   \n","4            120       RL         43.0     5005   Pave   NaN      IR1   \n","...          ...      ...          ...      ...    ...   ...      ...   \n","1454         160       RM         21.0     1936   Pave   NaN      Reg   \n","1455         160       RM         21.0     1894   Pave   NaN      Reg   \n","1456          20       RL        160.0    20000   Pave   NaN      Reg   \n","1457          85       RL         62.0    10441   Pave   NaN      Reg   \n","1458          60       RL         74.0     9627   Pave   NaN      Reg   \n","\n","     LandContour Utilities LotConfig  ... ScreenPorch PoolArea PoolQC  Fence  \\\n","0            Lvl    AllPub    Inside  ...         120        0    NaN  MnPrv   \n","1            Lvl    AllPub    Corner  ...           0        0    NaN    NaN   \n","2            Lvl    AllPub    Inside  ...           0        0    NaN  MnPrv   \n","3            Lvl    AllPub    Inside  ...           0        0    NaN    NaN   \n","4            HLS    AllPub    Inside  ...         144        0    NaN    NaN   \n","...          ...       ...       ...  ...         ...      ...    ...    ...   \n","1454         Lvl    AllPub    Inside  ...           0        0    NaN    NaN   \n","1455         Lvl    AllPub    Inside  ...           0        0    NaN    NaN   \n","1456         Lvl    AllPub    Inside  ...           0        0    NaN    NaN   \n","1457         Lvl    AllPub    Inside  ...           0        0    NaN  MnPrv   \n","1458         Lvl    AllPub    Inside  ...           0        0    NaN    NaN   \n","\n","     MiscFeature MiscVal  MoSold  YrSold  SaleType  SaleCondition  \n","0            NaN       0       6    2010        WD         Normal  \n","1           Gar2   12500       6    2010        WD         Normal  \n","2            NaN       0       3    2010        WD         Normal  \n","3            NaN       0       6    2010        WD         Normal  \n","4            NaN       0       1    2010        WD         Normal  \n","...          ...     ...     ...     ...       ...            ...  \n","1454         NaN       0       6    2006        WD         Normal  \n","1455         NaN       0       4    2006        WD        Abnorml  \n","1456         NaN       0       9    2006        WD        Abnorml  \n","1457        Shed     700       7    2006        WD         Normal  \n","1458         NaN       0      11    2006        WD         Normal  \n","\n","[1459 rows x 79 columns]"]},"execution_count":124,"metadata":{},"output_type":"execute_result"}],"source":["X_test = pd.read_csv(\"https://storage.googleapis.com/efcabinet/data/house-price-test.csv\")\n","X_test_ids = X_test['Id'] # Keep ids\n","X_test = X_test.drop(columns=['Id'])\n","X_test"]},{"cell_type":"code","execution_count":null,"id":"a01ab730-90dd-4f34-ab7b-d565d27d27eb","metadata":{"id":"a01ab730-90dd-4f34-ab7b-d565d27d27eb","outputId":"523dd0fa-8eec-41a6-ccc3-13e9d8129aa9"},"outputs":[{"data":{"text/plain":["(1459, 80)"]},"execution_count":125,"metadata":{},"output_type":"execute_result"}],"source":["X_test['MoSold_sin'] = np.sin((X_test.MoSold-1)*(2.*np.pi/12))\n","X_test['MoSold_cos'] = np.cos((X_test.MoSold-1)*(2.*np.pi/12))\n","X_test.drop(columns=['MoSold'], inplace=True)\n","X_test.shape"]},{"cell_type":"code","execution_count":null,"id":"a3c7f6a5-b5ee-4e3c-9e87-87cb945aefb5","metadata":{"id":"a3c7f6a5-b5ee-4e3c-9e87-87cb945aefb5","outputId":"4bf10a96-c74f-4a0a-c1c2-7528c7aeecac"},"outputs":[{"data":{"text/plain":["(1459, 217)"]},"execution_count":126,"metadata":{},"output_type":"execute_result"}],"source":["preproc.fit(X,y_log)\n","X_test_processed = preproc.transform(X_test)\n","X_test_processed.shape"]},{"cell_type":"markdown","id":"7fdd03b8-f40d-480b-a196-295bd3615f17","metadata":{"id":"7fdd03b8-f40d-480b-a196-295bd3615f17"},"source":["**Blending methods**"]},{"cell_type":"code","execution_count":null,"id":"8b7def07-8c3b-4944-ba37-dd8243a333eb","metadata":{"id":"8b7def07-8c3b-4944-ba37-dd8243a333eb"},"outputs":[],"source":["predictions_logs = blended_predict(X_test_processed)"]},{"cell_type":"code","execution_count":null,"id":"c998a0c9-2292-42d8-8b97-fcac2878b320","metadata":{"id":"c998a0c9-2292-42d8-8b97-fcac2878b320","outputId":"ff393d1d-d6cf-41e9-dc06-985cc7ab0819"},"outputs":[{"data":{"text/plain":["array([124104., 151482., 180579., ..., 155446., 113784., 218870.])"]},"execution_count":128,"metadata":{},"output_type":"execute_result"}],"source":["predictions = np.floor(np.exp(predictions_logs))\n","predictions"]},{"cell_type":"code","execution_count":null,"id":"e7386680-2588-401a-a208-3ca02a0a0370","metadata":{"id":"e7386680-2588-401a-a208-3ca02a0a0370","outputId":"5652c829-9857-4846-b5a2-7993547263e9"},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>Id</th>\n","      <th>SalePrice</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>1461</td>\n","      <td>124104.0</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>1462</td>\n","      <td>151482.0</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>1463</td>\n","      <td>180579.0</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>1464</td>\n","      <td>193454.0</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>1465</td>\n","      <td>187332.0</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>1454</th>\n","      <td>2915</td>\n","      <td>80986.0</td>\n","    </tr>\n","    <tr>\n","      <th>1455</th>\n","      <td>2916</td>\n","      <td>81803.0</td>\n","    </tr>\n","    <tr>\n","      <th>1456</th>\n","      <td>2917</td>\n","      <td>155446.0</td>\n","    </tr>\n","    <tr>\n","      <th>1457</th>\n","      <td>2918</td>\n","      <td>113784.0</td>\n","    </tr>\n","    <tr>\n","      <th>1458</th>\n","      <td>2919</td>\n","      <td>218870.0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>1459 rows × 2 columns</p>\n","</div>"],"text/plain":["        Id  SalePrice\n","0     1461   124104.0\n","1     1462   151482.0\n","2     1463   180579.0\n","3     1464   193454.0\n","4     1465   187332.0\n","...    ...        ...\n","1454  2915    80986.0\n","1455  2916    81803.0\n","1456  2917   155446.0\n","1457  2918   113784.0\n","1458  2919   218870.0\n","\n","[1459 rows x 2 columns]"]},"execution_count":129,"metadata":{},"output_type":"execute_result"}],"source":["results = pd.concat([X_test_ids, pd.Series(predictions, name=\"SalePrice\")], axis=1)\n","results"]},{"cell_type":"code","execution_count":null,"id":"b4c55fe5-632b-49d3-82aa-a6f8b0695112","metadata":{"id":"b4c55fe5-632b-49d3-82aa-a6f8b0695112"},"outputs":[],"source":["# Export to Kaggle format submission and submit it online!\n","results.to_csv(\"submission_blending.csv\", header=True, index=False)"]},{"cell_type":"markdown","id":"0dadf7c3-e1e3-43fe-aefe-7c5447b3cbbf","metadata":{"id":"0dadf7c3-e1e3-43fe-aefe-7c5447b3cbbf"},"source":["**Stacking methods**"]},{"cell_type":"code","execution_count":null,"id":"98ef28fe-e1c2-48af-9bb3-e7dbd2e3b748","metadata":{"id":"98ef28fe-e1c2-48af-9bb3-e7dbd2e3b748"},"outputs":[],"source":["import warnings\n","warnings.filterwarnings('ignore')\n","warnings.simplefilter(action='ignore', category=FutureWarning)\n","\n","pipe_stacking_2.fit(X,y_log)\n","predictions_log_2 = pipe_stacking_2.predict(X_test)\n","predictions_2 = np.floor(np.exp(predictions_log_2))"]},{"cell_type":"code","execution_count":null,"id":"df5bf75d-1a7d-44fe-a97a-af75561737cf","metadata":{"id":"df5bf75d-1a7d-44fe-a97a-af75561737cf","outputId":"86d29d4b-9bc3-4ab5-8085-0d7f28c554f5"},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>Id</th>\n","      <th>SalePrice</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>1461</td>\n","      <td>119899.0</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>1462</td>\n","      <td>155968.0</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>1463</td>\n","      <td>182115.0</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>1464</td>\n","      <td>193459.0</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>1465</td>\n","      <td>187727.0</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>1454</th>\n","      <td>2915</td>\n","      <td>78440.0</td>\n","    </tr>\n","    <tr>\n","      <th>1455</th>\n","      <td>2916</td>\n","      <td>79013.0</td>\n","    </tr>\n","    <tr>\n","      <th>1456</th>\n","      <td>2917</td>\n","      <td>159188.0</td>\n","    </tr>\n","    <tr>\n","      <th>1457</th>\n","      <td>2918</td>\n","      <td>111117.0</td>\n","    </tr>\n","    <tr>\n","      <th>1458</th>\n","      <td>2919</td>\n","      <td>217638.0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>1459 rows × 2 columns</p>\n","</div>"],"text/plain":["        Id  SalePrice\n","0     1461   119899.0\n","1     1462   155968.0\n","2     1463   182115.0\n","3     1464   193459.0\n","4     1465   187727.0\n","...    ...        ...\n","1454  2915    78440.0\n","1455  2916    79013.0\n","1456  2917   159188.0\n","1457  2918   111117.0\n","1458  2919   217638.0\n","\n","[1459 rows x 2 columns]"]},"execution_count":132,"metadata":{},"output_type":"execute_result"}],"source":["results_2 = pd.concat([X_test_ids, pd.Series(predictions_2, name=\"SalePrice\")], axis=1)\n","results_2"]},{"cell_type":"code","execution_count":null,"id":"acaa2192-0cf8-48e9-ac44-7ecf7850a91e","metadata":{"id":"acaa2192-0cf8-48e9-ac44-7ecf7850a91e"},"outputs":[],"source":["results.to_csv(\"submission_stacking.csv\", header=True, index=False)"]},{"cell_type":"code","execution_count":null,"id":"51d685d3-f65c-4908-ab1e-b81b067ed2c2","metadata":{"id":"51d685d3-f65c-4908-ab1e-b81b067ed2c2"},"outputs":[],"source":[""]},{"cell_type":"code","execution_count":null,"id":"a91282fe-23ef-4ac0-821b-8888f34c93e4","metadata":{"id":"a91282fe-23ef-4ac0-821b-8888f34c93e4"},"outputs":[],"source":["# Clear the cache directory at the end\n","rmtree(cachedir)"]},{"cell_type":"code","execution_count":null,"id":"a7b75c82-fd5a-417e-8d0c-34b7fd0252b9","metadata":{"id":"a7b75c82-fd5a-417e-8d0c-34b7fd0252b9"},"outputs":[],"source":[""]}],"metadata":{"kernelspec":{"display_name":"Python 3 (ipykernel)","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.8.12"},"toc-autonumbering":true,"colab":{"name":"NN-House-Price-Short.ipynb","provenance":[],"collapsed_sections":[],"toc_visible":true}},"nbformat":4,"nbformat_minor":5}