src/task-one-sample.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First, we need to import the necessary packages.\n",
    "\n",
    "**Note:** You will most likely have a more expansive import cell, I'm just importing what I need for the sample."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import datasets, linear_model, metrics, model_selection\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Next, we need to load the data set and store it into a pandas DataFrame. Feel free to copy and paste the code below directly into your own Jupyter notebook (.ipynb) source file(s)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = datasets.fetch_california_housing(as_frame=True).frame"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, we went to see the data in a tabular representation. Executing the following cell will do just that."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MedInc</th>\n",
       "      <th>HouseAge</th>\n",
       "      <th>AveRooms</th>\n",
       "      <th>AveBedrms</th>\n",
       "      <th>Population</th>\n",
       "      <th>AveOccup</th>\n",
       "      <th>Latitude</th>\n",
       "      <th>Longitude</th>\n",
       "      <th>MedHouseVal</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8.3252</td>\n",
       "      <td>41.0</td>\n",
       "      <td>6.984127</td>\n",
       "      <td>1.023810</td>\n",
       "      <td>322.0</td>\n",
       "      <td>2.555556</td>\n",
       "      <td>37.88</td>\n",
       "      <td>-122.23</td>\n",
       "      <td>4.526</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>8.3014</td>\n",
       "      <td>21.0</td>\n",
       "      <td>6.238137</td>\n",
       "      <td>0.971880</td>\n",
       "      <td>2401.0</td>\n",
       "      <td>2.109842</td>\n",
       "      <td>37.86</td>\n",
       "      <td>-122.22</td>\n",
       "      <td>3.585</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7.2574</td>\n",
       "      <td>52.0</td>\n",
       "      <td>8.288136</td>\n",
       "      <td>1.073446</td>\n",
       "      <td>496.0</td>\n",
       "      <td>2.802260</td>\n",
       "      <td>37.85</td>\n",
       "      <td>-122.24</td>\n",
       "      <td>3.521</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5.6431</td>\n",
       "      <td>52.0</td>\n",
       "      <td>5.817352</td>\n",
       "      <td>1.073059</td>\n",
       "      <td>558.0</td>\n",
       "      <td>2.547945</td>\n",
       "      <td>37.85</td>\n",
       "      <td>-122.25</td>\n",
       "      <td>3.413</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3.8462</td>\n",
       "      <td>52.0</td>\n",
       "      <td>6.281853</td>\n",
       "      <td>1.081081</td>\n",
       "      <td>565.0</td>\n",
       "      <td>2.181467</td>\n",
       "      <td>37.85</td>\n",
       "      <td>-122.25</td>\n",
       "      <td>3.422</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20635</th>\n",
       "      <td>1.5603</td>\n",
       "      <td>25.0</td>\n",
       "      <td>5.045455</td>\n",
       "      <td>1.133333</td>\n",
       "      <td>845.0</td>\n",
       "      <td>2.560606</td>\n",
       "      <td>39.48</td>\n",
       "      <td>-121.09</td>\n",
       "      <td>0.781</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20636</th>\n",
       "      <td>2.5568</td>\n",
       "      <td>18.0</td>\n",
       "      <td>6.114035</td>\n",
       "      <td>1.315789</td>\n",
       "      <td>356.0</td>\n",
       "      <td>3.122807</td>\n",
       "      <td>39.49</td>\n",
       "      <td>-121.21</td>\n",
       "      <td>0.771</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20637</th>\n",
       "      <td>1.7000</td>\n",
       "      <td>17.0</td>\n",
       "      <td>5.205543</td>\n",
       "      <td>1.120092</td>\n",
       "      <td>1007.0</td>\n",
       "      <td>2.325635</td>\n",
       "      <td>39.43</td>\n",
       "      <td>-121.22</td>\n",
       "      <td>0.923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20638</th>\n",
       "      <td>1.8672</td>\n",
       "      <td>18.0</td>\n",
       "      <td>5.329513</td>\n",
       "      <td>1.171920</td>\n",
       "      <td>741.0</td>\n",
       "      <td>2.123209</td>\n",
       "      <td>39.43</td>\n",
       "      <td>-121.32</td>\n",
       "      <td>0.847</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20639</th>\n",
       "      <td>2.3886</td>\n",
       "      <td>16.0</td>\n",
       "      <td>5.254717</td>\n",
       "      <td>1.162264</td>\n",
       "      <td>1387.0</td>\n",
       "      <td>2.616981</td>\n",
       "      <td>39.37</td>\n",
       "      <td>-121.24</td>\n",
       "      <td>0.894</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20640 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       MedInc  HouseAge  AveRooms  AveBedrms  Population  AveOccup  Latitude  \\\n",
       "0      8.3252      41.0  6.984127   1.023810       322.0  2.555556     37.88   \n",
       "1      8.3014      21.0  6.238137   0.971880      2401.0  2.109842     37.86   \n",
       "2      7.2574      52.0  8.288136   1.073446       496.0  2.802260     37.85   \n",
       "3      5.6431      52.0  5.817352   1.073059       558.0  2.547945     37.85   \n",
       "4      3.8462      52.0  6.281853   1.081081       565.0  2.181467     37.85   \n",
       "...       ...       ...       ...        ...         ...       ...       ...   \n",
       "20635  1.5603      25.0  5.045455   1.133333       845.0  2.560606     39.48   \n",
       "20636  2.5568      18.0  6.114035   1.315789       356.0  3.122807     39.49   \n",
       "20637  1.7000      17.0  5.205543   1.120092      1007.0  2.325635     39.43   \n",
       "20638  1.8672      18.0  5.329513   1.171920       741.0  2.123209     39.43   \n",
       "20639  2.3886      16.0  5.254717   1.162264      1387.0  2.616981     39.37   \n",
       "\n",
       "       Longitude  MedHouseVal  \n",
       "0        -122.23        4.526  \n",
       "1        -122.22        3.585  \n",
       "2        -122.24        3.521  \n",
       "3        -122.25        3.413  \n",
       "4        -122.25        3.422  \n",
       "...          ...          ...  \n",
       "20635    -121.09        0.781  \n",
       "20636    -121.21        0.771  \n",
       "20637    -121.22        0.923  \n",
       "20638    -121.32        0.847  \n",
       "20639    -121.24        0.894  \n",
       "\n",
       "[20640 rows x 9 columns]"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Next, we want to process the data. I will only process it in the following ways:\n",
    "1. Deal with missing (null or NA) data, if any such records exist.\n",
    "1. Dealing with duplicate data, if any such records exist.\n",
    "\n",
    "**Note:** You should do more than what I'm doing (variable transformation if its distribution is skewed, outlier removal if you deem it necessary, et cetera, in addition to the above steps)... I'm only doing this much to make the baseline model you'll be using as simple as possible."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MedInc         False\n",
       "HouseAge       False\n",
       "AveRooms       False\n",
       "AveBedrms      False\n",
       "Population     False\n",
       "AveOccup       False\n",
       "Latitude       False\n",
       "Longitude      False\n",
       "MedHouseVal    False\n",
       "dtype: bool"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check if any data is missing...\n",
    "data.isna().any()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MedInc         False\n",
       "HouseAge       False\n",
       "AveRooms       False\n",
       "AveBedrms      False\n",
       "Population     False\n",
       "AveOccup       False\n",
       "Latitude       False\n",
       "Longitude      False\n",
       "MedHouseVal    False\n",
       "dtype: bool"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check if any data is duplicated...\n",
    "data[data.duplicated()].any()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It appears that there are no missing (null or NA) data entries, nor are there any duplicate data records. \n",
    "\n",
    "Next, we should split our data into $X$ and $y$ variables, and then split them into training and testing data sets.\n",
    "\n",
    "**Note:** the \"train_test_split\" function uses seeds for randomization. If you want to stabilize the splitting so your model is the same each time you train it or test it, hard code the seed by setting the \"random_state\" parameter manually."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "# split into X and y...\n",
    "# make sure to only include in X variables that you want to use in the model as explanatory/independent variables...\n",
    "# here, I'm only using MedInc and HouseAge...\n",
    "# Note: every time you want to add or remove a variable, you'll need to re-run this cell and all the ones after it...\n",
    "X = data[[\"MedInc\", \"HouseAge\"]]\n",
    "y = data.MedHouseVal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MedInc</th>\n",
       "      <th>HouseAge</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8.3252</td>\n",
       "      <td>41.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>8.3014</td>\n",
       "      <td>21.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7.2574</td>\n",
       "      <td>52.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5.6431</td>\n",
       "      <td>52.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3.8462</td>\n",
       "      <td>52.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20635</th>\n",
       "      <td>1.5603</td>\n",
       "      <td>25.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20636</th>\n",
       "      <td>2.5568</td>\n",
       "      <td>18.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20637</th>\n",
       "      <td>1.7000</td>\n",
       "      <td>17.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20638</th>\n",
       "      <td>1.8672</td>\n",
       "      <td>18.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20639</th>\n",
       "      <td>2.3886</td>\n",
       "      <td>16.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20640 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       MedInc  HouseAge\n",
       "0      8.3252      41.0\n",
       "1      8.3014      21.0\n",
       "2      7.2574      52.0\n",
       "3      5.6431      52.0\n",
       "4      3.8462      52.0\n",
       "...       ...       ...\n",
       "20635  1.5603      25.0\n",
       "20636  2.5568      18.0\n",
       "20637  1.7000      17.0\n",
       "20638  1.8672      18.0\n",
       "20639  2.3886      16.0\n",
       "\n",
       "[20640 rows x 2 columns]"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# view X variable...\n",
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MedHouseVal</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4.526</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3.585</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3.521</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3.413</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3.422</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20635</th>\n",
       "      <td>0.781</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20636</th>\n",
       "      <td>0.771</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20637</th>\n",
       "      <td>0.923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20638</th>\n",
       "      <td>0.847</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20639</th>\n",
       "      <td>0.894</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20640 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       MedHouseVal\n",
       "0            4.526\n",
       "1            3.585\n",
       "2            3.521\n",
       "3            3.413\n",
       "4            3.422\n",
       "...            ...\n",
       "20635        0.781\n",
       "20636        0.771\n",
       "20637        0.923\n",
       "20638        0.847\n",
       "20639        0.894\n",
       "\n",
       "[20640 rows x 1 columns]"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# view y variable...\n",
    "y = y.to_frame()\n",
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "# split X, y into training and testing versions...\n",
    "X_train, X_test, y_train, y_test = model_selection.train_test_split(X, y, train_size=0.8, random_state=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "At this point, we can create out baseline linear regression model. Please try copy and paste the code in every cell from below labelled\n",
    "\n",
    "```\n",
    "# COPY THIS CELL!!!\n",
    "```\n",
    "\n",
    "directly into each of your own Jupyter notebook (.ipynb) source file(s) in which you create a model for task one. \n",
    "\n",
    "Also, make sure to label the cell where you define the model you have chosen as your accepted model as follows:\n",
    "\n",
    "```\n",
    "# THIS IS MY ACCEPTED MODEL!!!\n",
    "```\n",
    "\n",
    "Remember, you want to choose your accepted model based on how much better your models perform compared to this baseline model: the best model you create, compared to this baseline, should be your accepted model, using the adjusted-$R^2$ score as your measure of comparison."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "# COPY THIS CELL!!!\n",
    "\n",
    "# define a linear regression model...\n",
    "baseline_lin_reg = linear_model.LinearRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LinearRegression()"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# COPY THIS CELL!!!\n",
    "\n",
    "# train the linear regression model using the \"fit\" function...\n",
    "baseline_lin_reg.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "# COPY THIS CELL!!!\n",
    "\n",
    "# use the trained linear regression model to make predictions for y_test based on the X_test data...\n",
    "y_pred = baseline_lin_reg.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "# COPY THIS CELL!!!\n",
    "\n",
    "# determine the non-adjusted r2 score of the baseline model...\n",
    "r2 = metrics.r2_score(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "# COPY THIS CELL!!!\n",
    "\n",
    "# determine the number of observations in the test set (n) and the number of explanatory/independent variables used (k)...\n",
    "n, k = X_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Baseline Adjusted-R^2 : 0.4938153753759118\n"
     ]
    }
   ],
   "source": [
    "# COPY THIS CELL!!!\n",
    "\n",
    "# calculate the adjusted-r2 score for the baseline linear regression model...\n",
    "adj_r2 = 1 - (1 - r2) * ((n - 1) / (n - k - 1))\n",
    "\n",
    "# print the calculated adjusted-r2 score (this is the value you want to beat!!!)\n",
    "print(f'Baseline Adjusted-R^2 : {adj_r2}')"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "5f1757bcd5b2b135bb28c0f46c3cd840729cbc634a36c28aa70e8d620d258ca7"
  },
  "kernelspec": {
   "display_name": "Python 3.9.7 ('base')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}