04-deployment/homework_answers.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "da2523e1-2706-4d00-875b-7bec93f88e2d",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mDEPRECATION: Configuring installation scheme with distutils config files is deprecated and will no longer work in the near future. If you are using a Homebrew or Linuxbrew Python, please see discussion at https://github.com/Homebrew/homebrew-core/issues/76621\u001b[0m\u001b[33m\n",
      "\u001b[0mCollecting scikit-learn==1.0.2\n",
      "  Downloading scikit_learn-1.0.2-cp39-cp39-macosx_10_13_x86_64.whl (8.0 MB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m8.0/8.0 MB\u001b[0m \u001b[31m6.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m[36m0:00:01\u001b[0m\n",
      "\u001b[?25hCollecting scipy>=1.1.0\n",
      "  Downloading scipy-1.8.1-cp39-cp39-macosx_12_0_universal2.macosx_10_9_x86_64.whl (55.6 MB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m55.6/55.6 MB\u001b[0m \u001b[31m6.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m[36m0:00:01\u001b[0mm\n",
      "\u001b[?25hRequirement already satisfied: numpy>=1.14.6 in /usr/local/lib/python3.9/site-packages (from scikit-learn==1.0.2) (1.22.2)\n",
      "Collecting joblib>=0.11\n",
      "  Using cached joblib-1.1.0-py2.py3-none-any.whl (306 kB)\n",
      "Collecting threadpoolctl>=2.0.0\n",
      "  Using cached threadpoolctl-3.1.0-py3-none-any.whl (14 kB)\n",
      "Installing collected packages: threadpoolctl, scipy, joblib, scikit-learn\n",
      "\u001b[33m  DEPRECATION: Configuring installation scheme with distutils config files is deprecated and will no longer work in the near future. If you are using a Homebrew or Linuxbrew Python, please see discussion at https://github.com/Homebrew/homebrew-core/issues/76621\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[33m  DEPRECATION: Configuring installation scheme with distutils config files is deprecated and will no longer work in the near future. If you are using a Homebrew or Linuxbrew Python, please see discussion at https://github.com/Homebrew/homebrew-core/issues/76621\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[33m  DEPRECATION: Configuring installation scheme with distutils config files is deprecated and will no longer work in the near future. If you are using a Homebrew or Linuxbrew Python, please see discussion at https://github.com/Homebrew/homebrew-core/issues/76621\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[33m  DEPRECATION: Configuring installation scheme with distutils config files is deprecated and will no longer work in the near future. If you are using a Homebrew or Linuxbrew Python, please see discussion at https://github.com/Homebrew/homebrew-core/issues/76621\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[33mDEPRECATION: Configuring installation scheme with distutils config files is deprecated and will no longer work in the near future. If you are using a Homebrew or Linuxbrew Python, please see discussion at https://github.com/Homebrew/homebrew-core/issues/76621\u001b[0m\u001b[33m\n",
      "\u001b[0mSuccessfully installed joblib-1.1.0 scikit-learn-1.0.2 scipy-1.8.1 threadpoolctl-3.1.0\n",
      "\u001b[33mWARNING: You are using pip version 22.0.3; however, version 22.1.2 is available.\n",
      "You should consider upgrading via the '/usr/local/opt/python@3.9/bin/python3.9 -m pip install --upgrade pip' command.\u001b[0m\u001b[33m\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!pip install scikit-learn==1.0.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "d3680bcf-5309-4a73-8aef-72e10fdf9de0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "8766592e-9f51-447e-ab73-3080d4c4234c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "with open('model.bin', 'rb') as f_in:\n",
    "    dv, lr = pickle.load(f_in)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "369170d2-f155-4b91-ba55-be6dcb30ede9",
   "metadata": {},
   "outputs": [],
   "source": [
    "categorical = ['PUlocationID', 'DOlocationID']\n",
    "\n",
    "def read_data(filename):\n",
    "    df = pd.read_parquet(filename)\n",
    "    \n",
    "    df['duration'] = df.dropOff_datetime - df.pickup_datetime\n",
    "    df['duration'] = df.duration.dt.total_seconds() / 60\n",
    "\n",
    "    df = df[(df.duration >= 1) & (df.duration <= 60)].copy()\n",
    "\n",
    "    df[categorical] = df[categorical].fillna(-1).astype('int').astype('str')\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "e18caa97-6ae7-4b78-8884-54a0657072e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = read_data('../data/fhv_tripdata_2021-02.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "f5ee329b-1a6e-4b8c-a0e0-c75065a468b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "dicts = df[categorical].to_dict(orient='records')\n",
    "X_val = dv.transform(dicts)\n",
    "y_pred = lr.predict(X_val)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "889f539f-3d48-47a3-955c-690c566b0c34",
   "metadata": {},
   "source": [
    "## Q1. Notebook\n",
    "Run this notebook for the February 2021 FVH data.\n",
    "\n",
    "What's the mean predicted duration for this dataset?\n",
    "\n",
    "- 11.19\n",
    "- 16.19\n",
    "- 21.19\n",
    "- 26.19"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "9c2ea8bb-c206-46bc-b02a-013198fcb7d7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The mean predicted duration for this dataset is 16.191691679979066\n"
     ]
    }
   ],
   "source": [
    "print(f\"The mean predicted duration for this dataset is {np.mean(y_pred)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5a87e84f-0eaa-4cb1-bdf2-5a9a79d32677",
   "metadata": {},
   "source": [
    "## Q2. Preparing the output\n",
    "\n",
    "Like in the course videos, we want to prepare the dataframe with the output.\n",
    "\n",
    "First, let's create an artificial ride_id column:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "32ed6e85-0e4a-4642-8bc7-93e3b0e7c21b",
   "metadata": {},
   "outputs": [],
   "source": [
    "year = 2021\n",
    "month = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "69b04508-38ad-4a8e-9fe6-792279cc9847",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['ride_id'] = f'{year:04d}/{month:02d}_' + df.index.astype('str')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dc745a82-317a-4101-b77a-5afc23c254c0",
   "metadata": {},
   "source": [
    "Next, write the ride id and the predictions to a dataframe with results."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "2fb28cce-22bd-44bc-8f78-5241f93b21f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_result = df.copy(deep=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "3ccebe5a-83a9-4918-8ac7-2317ec783aac",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/0s/qfqb94xj5pg6xz3d_l7xfsqm0000gn/T/ipykernel_15431/1765440389.py:1: FutureWarning: In a future version of pandas all arguments of DataFrame.drop except for the argument 'labels' will be keyword-only.\n",
      "  df_result.drop(df_result.columns.difference(['ride_id']), 1, inplace=True)\n"
     ]
    }
   ],
   "source": [
    "df_result.drop(df_result.columns.difference(['ride_id']), 1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "d7005700-489c-4cf9-9335-0320bd98ab3c",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_result[\"predictions\"] = y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "1bfb5701-2871-48a5-8c5f-e5f25e14e441",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_result.to_parquet(\n",
    "    \"../data/results.parquet\",\n",
    "    engine='pyarrow',\n",
    "    compression=None,\n",
    "    index=False\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "df973ec2-1817-4db6-ba66-16fb7948418b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The size of the output file is 19.7MB\n"
     ]
    }
   ],
   "source": [
    "print(\"The size of the output file is 19.7MB\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3c0cc050-6124-4da3-ba19-44d016120284",
   "metadata": {},
   "source": [
    "## Q3. Creating the scoring script\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4b599f91-3623-430c-af03-da1ad68de50a",
   "metadata": {},
   "source": [
    "Now let's turn the notebook into a script.\n",
    "\n",
    "Which command you need to execute for that?"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a8e3626d-62b6-4fa3-a67d-ee0561374203",
   "metadata": {},
   "source": [
    "**Answer**: "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9f7467f2-130b-41c6-bd10-3d7423f1bea1",
   "metadata": {},
   "source": [
    "```\n",
    "jupyter nbconvert --to script homework_answrs.ipynb\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3fb002d5-62ec-440d-bcc1-b0817da2f113",
   "metadata": {},
   "source": [
    "## Q4. Virtual environment\n",
    "Now let's put everything into a virtual environment. We'll use pipenv for that.\n",
    "\n",
    "Install all the required libraries. Pay attention to the Scikit-Learn version: check the starter notebook for details.\n",
    "\n",
    "After installing the libraries, pipenv creates two files: Pipfile and Pipfile.lock. The Pipfile.lock file keeps the hashes of the dependencies we use for the virtual env.\n",
    "\n",
    "What's the first hash for the Scikit-Learn dependency?"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f49c8e48-a7bb-414f-acd3-a8257479c784",
   "metadata": {},
   "source": [
    "**Answers**"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a4185c10-30fb-41ef-8745-600502b9253b",
   "metadata": {},
   "source": [
    "\"sha256:08ef968f6b72033c16c479c966bf37ccd49b06ea91b765e1cc27afefe723920b\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8a82c141-fa77-42f3-9242-3fbdf062aaba",
   "metadata": {},
   "source": [
    "## Q5. Parametrize the script\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d2a7be7b-1a0b-40a2-ab7d-d16eb8bc60a5",
   "metadata": {},
   "source": [
    "Let's now make the script configurable via CLI. We'll create two parameters: year and month.\n",
    "\n",
    "Run the script for March 2021.\n",
    "\n",
    "What's the mean predicted duration?\n",
    "\n",
    "- 11.29\n",
    "- 16.29\n",
    "- 21.29\n",
    "- 26.29"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "44536ff2-7257-4714-924e-3d1cf42bd97c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}