Random Forest.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Online News Popularity Dataset\n",
    "\n",
    "#### Using Scikit-Learn to build and tune a random forest.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 1: Import Standard Libraries and modules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn import preprocessing\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import mean_squared_error, r2_score\n",
    "from sklearn.externals import joblib\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 2 : Load and prepare data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.chdir(\"/Users/prassanajoshi/Documents/extras/github-projects/python notebooks/Online News Popularity\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "news_data = pd.read_csv(\"OnlineNewsPopularity.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>url</th>\n",
       "      <th>timedelta</th>\n",
       "      <th>n_tokens_title</th>\n",
       "      <th>n_tokens_content</th>\n",
       "      <th>n_unique_tokens</th>\n",
       "      <th>n_non_stop_words</th>\n",
       "      <th>n_non_stop_unique_tokens</th>\n",
       "      <th>num_hrefs</th>\n",
       "      <th>num_self_hrefs</th>\n",
       "      <th>num_imgs</th>\n",
       "      <th>...</th>\n",
       "      <th>min_positive_polarity</th>\n",
       "      <th>max_positive_polarity</th>\n",
       "      <th>avg_negative_polarity</th>\n",
       "      <th>min_negative_polarity</th>\n",
       "      <th>max_negative_polarity</th>\n",
       "      <th>title_subjectivity</th>\n",
       "      <th>title_sentiment_polarity</th>\n",
       "      <th>abs_title_subjectivity</th>\n",
       "      <th>abs_title_sentiment_polarity</th>\n",
       "      <th>shares</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>http://mashable.com/2013/01/07/amazon-instant-...</td>\n",
       "      <td>731.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>219.0</td>\n",
       "      <td>0.663594</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.815385</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>0.7</td>\n",
       "      <td>-0.350000</td>\n",
       "      <td>-0.600</td>\n",
       "      <td>-0.200000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>-0.187500</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.187500</td>\n",
       "      <td>593</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>http://mashable.com/2013/01/07/ap-samsung-spon...</td>\n",
       "      <td>731.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>255.0</td>\n",
       "      <td>0.604743</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.791946</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.033333</td>\n",
       "      <td>0.7</td>\n",
       "      <td>-0.118750</td>\n",
       "      <td>-0.125</td>\n",
       "      <td>-0.100000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>711</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>http://mashable.com/2013/01/07/apple-40-billio...</td>\n",
       "      <td>731.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>211.0</td>\n",
       "      <td>0.575130</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.663866</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-0.466667</td>\n",
       "      <td>-0.800</td>\n",
       "      <td>-0.133333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>http://mashable.com/2013/01/07/astronaut-notre...</td>\n",
       "      <td>731.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>531.0</td>\n",
       "      <td>0.503788</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.665635</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.136364</td>\n",
       "      <td>0.8</td>\n",
       "      <td>-0.369697</td>\n",
       "      <td>-0.600</td>\n",
       "      <td>-0.166667</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>http://mashable.com/2013/01/07/att-u-verse-apps/</td>\n",
       "      <td>731.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>1072.0</td>\n",
       "      <td>0.415646</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.540890</td>\n",
       "      <td>19.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.033333</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-0.220192</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.050000</td>\n",
       "      <td>0.454545</td>\n",
       "      <td>0.136364</td>\n",
       "      <td>0.045455</td>\n",
       "      <td>0.136364</td>\n",
       "      <td>505</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 61 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 url   timedelta  \\\n",
       "0  http://mashable.com/2013/01/07/amazon-instant-...       731.0   \n",
       "1  http://mashable.com/2013/01/07/ap-samsung-spon...       731.0   \n",
       "2  http://mashable.com/2013/01/07/apple-40-billio...       731.0   \n",
       "3  http://mashable.com/2013/01/07/astronaut-notre...       731.0   \n",
       "4   http://mashable.com/2013/01/07/att-u-verse-apps/       731.0   \n",
       "\n",
       "    n_tokens_title   n_tokens_content   n_unique_tokens   n_non_stop_words  \\\n",
       "0             12.0              219.0          0.663594                1.0   \n",
       "1              9.0              255.0          0.604743                1.0   \n",
       "2              9.0              211.0          0.575130                1.0   \n",
       "3              9.0              531.0          0.503788                1.0   \n",
       "4             13.0             1072.0          0.415646                1.0   \n",
       "\n",
       "    n_non_stop_unique_tokens   num_hrefs   num_self_hrefs   num_imgs   ...     \\\n",
       "0                   0.815385         4.0              2.0        1.0   ...      \n",
       "1                   0.791946         3.0              1.0        1.0   ...      \n",
       "2                   0.663866         3.0              1.0        1.0   ...      \n",
       "3                   0.665635         9.0              0.0        1.0   ...      \n",
       "4                   0.540890        19.0             19.0       20.0   ...      \n",
       "\n",
       "    min_positive_polarity   max_positive_polarity   avg_negative_polarity  \\\n",
       "0                0.100000                     0.7               -0.350000   \n",
       "1                0.033333                     0.7               -0.118750   \n",
       "2                0.100000                     1.0               -0.466667   \n",
       "3                0.136364                     0.8               -0.369697   \n",
       "4                0.033333                     1.0               -0.220192   \n",
       "\n",
       "    min_negative_polarity   max_negative_polarity   title_subjectivity  \\\n",
       "0                  -0.600               -0.200000             0.500000   \n",
       "1                  -0.125               -0.100000             0.000000   \n",
       "2                  -0.800               -0.133333             0.000000   \n",
       "3                  -0.600               -0.166667             0.000000   \n",
       "4                  -0.500               -0.050000             0.454545   \n",
       "\n",
       "    title_sentiment_polarity   abs_title_subjectivity  \\\n",
       "0                  -0.187500                 0.000000   \n",
       "1                   0.000000                 0.500000   \n",
       "2                   0.000000                 0.500000   \n",
       "3                   0.000000                 0.500000   \n",
       "4                   0.136364                 0.045455   \n",
       "\n",
       "    abs_title_sentiment_polarity   shares  \n",
       "0                       0.187500      593  \n",
       "1                       0.000000      711  \n",
       "2                       0.000000     1500  \n",
       "3                       0.000000     1200  \n",
       "4                       0.136364      505  \n",
       "\n",
       "[5 rows x 61 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "news_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(39644, 61)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# dimensions of the df\n",
    "news_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "          timedelta   n_tokens_title   n_tokens_content   n_unique_tokens  \\\n",
      "count  39644.000000     39644.000000       39644.000000      39644.000000   \n",
      "mean     354.530471        10.398749         546.514731          0.548216   \n",
      "std      214.163767         2.114037         471.107508          3.520708   \n",
      "min        8.000000         2.000000           0.000000          0.000000   \n",
      "25%      164.000000         9.000000         246.000000          0.470870   \n",
      "50%      339.000000        10.000000         409.000000          0.539226   \n",
      "75%      542.000000        12.000000         716.000000          0.608696   \n",
      "max      731.000000        23.000000        8474.000000        701.000000   \n",
      "\n",
      "        n_non_stop_words   n_non_stop_unique_tokens     num_hrefs  \\\n",
      "count       39644.000000               39644.000000  39644.000000   \n",
      "mean            0.996469                   0.689175     10.883690   \n",
      "std             5.231231                   3.264816     11.332017   \n",
      "min             0.000000                   0.000000      0.000000   \n",
      "25%             1.000000                   0.625739      4.000000   \n",
      "50%             1.000000                   0.690476      8.000000   \n",
      "75%             1.000000                   0.754630     14.000000   \n",
      "max          1042.000000                 650.000000    304.000000   \n",
      "\n",
      "        num_self_hrefs      num_imgs    num_videos      ...        \\\n",
      "count     39644.000000  39644.000000  39644.000000      ...         \n",
      "mean          3.293638      4.544143      1.249874      ...         \n",
      "std           3.855141      8.309434      4.107855      ...         \n",
      "min           0.000000      0.000000      0.000000      ...         \n",
      "25%           1.000000      1.000000      0.000000      ...         \n",
      "50%           3.000000      1.000000      0.000000      ...         \n",
      "75%           4.000000      4.000000      1.000000      ...         \n",
      "max         116.000000    128.000000     91.000000      ...         \n",
      "\n",
      "        min_positive_polarity   max_positive_polarity   avg_negative_polarity  \\\n",
      "count            39644.000000            39644.000000            39644.000000   \n",
      "mean                 0.095446                0.756728               -0.259524   \n",
      "std                  0.071315                0.247786                0.127726   \n",
      "min                  0.000000                0.000000               -1.000000   \n",
      "25%                  0.050000                0.600000               -0.328383   \n",
      "50%                  0.100000                0.800000               -0.253333   \n",
      "75%                  0.100000                1.000000               -0.186905   \n",
      "max                  1.000000                1.000000                0.000000   \n",
      "\n",
      "        min_negative_polarity   max_negative_polarity   title_subjectivity  \\\n",
      "count            39644.000000            39644.000000         39644.000000   \n",
      "mean                -0.521944               -0.107500             0.282353   \n",
      "std                  0.290290                0.095373             0.324247   \n",
      "min                 -1.000000               -1.000000             0.000000   \n",
      "25%                 -0.700000               -0.125000             0.000000   \n",
      "50%                 -0.500000               -0.100000             0.150000   \n",
      "75%                 -0.300000               -0.050000             0.500000   \n",
      "max                  0.000000                0.000000             1.000000   \n",
      "\n",
      "        title_sentiment_polarity   abs_title_subjectivity  \\\n",
      "count               39644.000000             39644.000000   \n",
      "mean                    0.071425                 0.341843   \n",
      "std                     0.265450                 0.188791   \n",
      "min                    -1.000000                 0.000000   \n",
      "25%                     0.000000                 0.166667   \n",
      "50%                     0.000000                 0.500000   \n",
      "75%                     0.150000                 0.500000   \n",
      "max                     1.000000                 0.500000   \n",
      "\n",
      "        abs_title_sentiment_polarity         shares  \n",
      "count                   39644.000000   39644.000000  \n",
      "mean                        0.156064    3395.380184  \n",
      "std                         0.226294   11626.950749  \n",
      "min                         0.000000       1.000000  \n",
      "25%                         0.000000     946.000000  \n",
      "50%                         0.000000    1400.000000  \n",
      "75%                         0.250000    2800.000000  \n",
      "max                         1.000000  843300.000000  \n",
      "\n",
      "[8 rows x 60 columns]\n"
     ]
    }
   ],
   "source": [
    "# data types of the variables\n",
    "print news_data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Seperate target and training features\n",
    "y = news_data[' shares']\n",
    "y = pd.Series(y)\n",
    "X = news_data.drop([' shares','url'],axis = 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 3: Split data into train and test sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, \n",
    "                                                   test_size = 0.2,\n",
    "                                                   random_state = 123)\n",
    "\n",
    "#It's good practice to stratify your sample by the target variable. \n",
    "#This will ensure your training set looks similar to your test set, making your evaluation metrics more reliable"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step4: Data pre-processing\n",
    "\n",
    "#### Since features are on different scales, standardizing them by subtracting the means and dividing by the standard deviations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[  9.58891032e-17   1.73406930e-16  -1.19189259e-16  -1.56827973e-18\n",
      "  -8.51351851e-18   1.72510770e-17   1.89313767e-17   2.24039961e-19\n",
      "   8.06543859e-18  -2.97973148e-17   1.16052700e-16  -1.49210614e-16\n",
      "   6.72119882e-18   8.06543859e-18  -2.10597563e-17  -5.24253508e-17\n",
      "  -1.88193567e-17   9.00640643e-17  -5.64580701e-17   4.03271929e-18\n",
      "   9.49929434e-17   2.77809551e-17  -2.29640960e-16   1.86401247e-16\n",
      "  -1.49658694e-16  -8.87198245e-17  -3.16344425e-16   2.28520760e-17\n",
      "   4.12233528e-17  -3.94310331e-17   4.61522319e-17  -3.09175146e-17\n",
      "  -3.58463937e-17  -1.43385575e-17   3.58463937e-17   3.02453947e-17\n",
      "  -4.07752729e-17   5.33215107e-17  -1.07987261e-16  -6.49715886e-17\n",
      "   3.80867933e-17   1.28822977e-16   6.27311890e-18   5.06554351e-16\n",
      "   6.69879483e-17   1.44281735e-16   1.79231969e-18  -2.81394191e-16\n",
      "  -1.25462378e-16  -4.23659566e-16  -1.09779581e-16   7.86380262e-17\n",
      "  -3.31467122e-16  -1.98051325e-16   6.63158284e-17   2.55405555e-17\n",
      "   1.79231969e-18   4.77205117e-17  -1.20981579e-17]\n",
      "[ 1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.\n",
      "  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.\n",
      "  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.\n",
      "  1.  1.  1.  1.  1.]\n"
     ]
    }
   ],
   "source": [
    "\n",
    "scaler = preprocessing.StandardScaler().fit(X_train)\n",
    "X_train_scaled = scaler.transform(X_train)\n",
    "print X_train_scaled.mean(axis=0)\n",
    "print X_train_scaled.std(axis=0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-0.02502243 -0.00421941  0.02015284 -0.00606497 -0.00579596 -0.00606456\n",
      " -0.00252232 -0.0099696   0.00613773 -0.00887598 -0.00408505 -0.01012717\n",
      "  0.00719427 -0.01907651 -0.00200188  0.0049685  -0.0082003   0.00445631\n",
      " -0.03258591  0.01586163  0.01943958  0.01265531  0.02886226  0.02615369\n",
      "  0.00435483  0.01281415  0.01780455 -0.01504554 -0.01935563 -0.02039538\n",
      " -0.00389071  0.00809178 -0.00041899 -0.00058531  0.00079592  0.00287401\n",
      " -0.00893594 -0.00468122  0.01474593 -0.01173055  0.00405238  0.0024373\n",
      " -0.01074112 -0.02574469 -0.01230275 -0.00083714 -0.00181537 -0.0060137\n",
      "  0.00146525 -0.0125299  -0.00874494 -0.00260079  0.0026241  -0.00700374\n",
      "  0.0009064  -0.00072235  0.01372437 -0.00720816 -0.00463801]\n",
      "[ 0.99515668  0.99117605  1.03714301  0.03528044  0.0294733   0.04282419\n",
      "  1.01449526  0.96967368  1.01558434  0.97883135  1.01728884  1.01592727\n",
      "  1.01428663  0.98375235  0.99811918  1.00930441  0.99332446  1.00312195\n",
      "  0.95998331  1.43791063  1.67849197  1.03689387  0.96086853  1.00591572\n",
      "  1.00752877  1.13924474  1.08898185  0.69737971  0.808462    0.74103938\n",
      "  0.99653721  1.00648369  0.99966453  0.99952075  1.00080756  1.00521569\n",
      "  0.9847151   0.99486324  1.01567655  0.9811974   0.99976141  0.99520876\n",
      "  0.99267417  1.00796172  0.98468156  1.00982546  0.98851879  1.00803201\n",
      "  1.00539565  1.00128828  1.02986111  1.00420163  0.99922962  1.00332186\n",
      "  1.01809596  0.99608155  0.9901665   1.00301598  0.99484526]\n"
     ]
    }
   ],
   "source": [
    "# Tranforming the test set using means from the training set\n",
    "\n",
    "X_test_scaled = scaler.transform(X_test)\n",
    "\n",
    "print X_test_scaled.mean(axis = 0)\n",
    "print X_test_scaled.std(axis = 0)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# pipeline with preprocessing and model\n",
    "pipeline = make_pipeline(preprocessing.StandardScaler(), \n",
    "                         RandomForestRegressor(n_estimators=100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 5: Declare hyperparameters to tune\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'randomforestregressor__random_state': None, 'randomforestregressor__min_weight_fraction_leaf': 0.0, 'standardscaler__with_mean': True, 'randomforestregressor__n_estimators': 100, 'randomforestregressor__min_samples_leaf': 1, 'standardscaler__copy': True, 'randomforestregressor__warm_start': False, 'randomforestregressor__criterion': 'mse', 'randomforestregressor__n_jobs': 1, 'randomforestregressor__max_leaf_nodes': None, 'randomforestregressor__oob_score': False, 'randomforestregressor__verbose': 0, 'randomforestregressor__min_impurity_split': 1e-07, 'steps': [('standardscaler', StandardScaler(copy=True, with_mean=True, with_std=True)), ('randomforestregressor', RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=None,\n",
      "           max_features='auto', max_leaf_nodes=None,\n",
      "           min_impurity_split=1e-07, min_samples_leaf=1,\n",
      "           min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
      "           n_estimators=100, n_jobs=1, oob_score=False, random_state=None,\n",
      "           verbose=0, warm_start=False))], 'randomforestregressor__min_samples_split': 2, 'randomforestregressor': RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=None,\n",
      "           max_features='auto', max_leaf_nodes=None,\n",
      "           min_impurity_split=1e-07, min_samples_leaf=1,\n",
      "           min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
      "           n_estimators=100, n_jobs=1, oob_score=False, random_state=None,\n",
      "           verbose=0, warm_start=False), 'randomforestregressor__max_depth': None, 'standardscaler__with_std': True, 'randomforestregressor__max_features': 'auto', 'standardscaler': StandardScaler(copy=True, with_mean=True, with_std=True), 'randomforestregressor__bootstrap': True}\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# List tunable hyperparameters:\n",
    "print pipeline.get_params()\n",
    "\n",
    "# Declare hyperparameters to tune:\n",
    "\n",
    "hyperparameters = { 'randomforestregressor__max_features' : ['auto', 'sqrt', 'log2'],\n",
    "                  'randomforestregressor__max_depth': [None, 5, 3, 1]}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 6: Tune model using a cross-validation pipeline\n",
    "\n",
    "#### 1. Cross Validation Pipeline:\n",
    "#### 2. Split data into k equal parts, or \"folds\" (typically k=10).\n",
    "#### 3. Preprocess k-1 training folds.\n",
    "#### 4. Train the model on the same k-1 folds.\n",
    "#### 5. Preprocess the hold-out fold using the same transformations from step (2).\n",
    "#### 6. Evaluate the model on the same hold-out fold.\n",
    "#### 7. Perform steps (2) - (5) k times, each time holding out a different fold.\n",
    "#### 8. Aggregate the performance across all k folds. This is your performance metric.\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'randomforestregressor__max_depth': 3, 'randomforestregressor__max_features': 'sqrt'}\n"
     ]
    }
   ],
   "source": [
    "# GridSearchCV() preforms cross-validation across the entire \"grid\" (all possible permuataions) of hyperparameters\n",
    "\n",
    "clf = GridSearchCV(pipeline, hyperparameters, cv=10)\n",
    "\n",
    "# Fit and tune model\n",
    "clf.fit(X_train, y_train)\n",
    "\n",
    "# See the best parameters found using CV\n",
    "print clf.best_params_\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 7: After tuning, refit on the entire training set\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n"
     ]
    }
   ],
   "source": [
    "print clf.refit"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 8: Evaluate model pipeline on test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_pred = clf.predict(X_test)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0129403759342\n",
      "151183401.696\n"
     ]
    }
   ],
   "source": [
    "# Evaluate model performance on testing data:\n",
    "\n",
    "print r2_score(y_test, y_pred)\n",
    "\n",
    "print mean_squared_error(y_test, y_pred)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 9: Save model for future use"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['rf_regressor.pkl']"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "joblib.dump(clf, 'rf_regressor.pkl')\n",
    "\n",
    "# To load again, use:\n",
    "\n",
    "#clf2 = joblib.load('rf_regressor.pkl')\n",
    "\n",
    "# To predict data using a loaded model:\n",
    "\n",
    "#clf2.predict(X_test)\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}