Part 1 - Web Scraping.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Segmenting and Clustering Neighborhoods in Toronto - part1.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyOKZttKxe9ef2yjba2DeVS2"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eb3oM4Tf4C2t"
      },
      "source": [
        "Import libraries\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b8SqO_IN4ImP"
      },
      "source": [
        "import pandas as pd\n",
        "import requests\n",
        "from bs4 import BeautifulSoup"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AmNXK6Bb4dGR"
      },
      "source": [
        "Scrape list of postal codes on Wikipedia"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q2zRkhZk4g9C"
      },
      "source": [
        "url = \"https://en.wikipedia.org/w/index.php?title=List_of_postal_codes_of_Canada:_M&oldid=1012118802\"\n",
        "extracting_data = requests.get(url)\n",
        "soup = BeautifulSoup(extracting_data.content, 'html.parser')"
      ],
      "execution_count": 62,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "o84148ed4j1I"
      },
      "source": [
        "Create dataframe from web content"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 173
        },
        "id": "WOXh-fYR4xIn",
        "outputId": "be635ea6-7364-4816-ee55-6963d9b160ff"
      },
      "source": [
        "table = soup.find('tbody')\n",
        "rows = table.select('tr')\n",
        "row = [r.get_text() for r in rows]\n",
        "\n",
        "df = pd.DataFrame(row)\n",
        "df = df[0].str.split('\\n', expand=True)\n",
        "df = df.rename(columns=df.iloc[0])\n",
        "df = df.drop(df.index[0])\n",
        "df = df[['Postal Code', 'Borough', 'Neighbourhood']]\n",
        "df = df.rename(columns={'Postal Code': 'Postalcode'})"
      ],
      "execution_count": 87,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Postalcode</th>\n",
              "      <th>Borough</th>\n",
              "      <th>Neighbourhood</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>180</td>\n",
              "      <td>180</td>\n",
              "      <td>180</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>unique</th>\n",
              "      <td>180</td>\n",
              "      <td>12</td>\n",
              "      <td>100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>top</th>\n",
              "      <td>M2H</td>\n",
              "      <td>Not assigned</td>\n",
              "      <td>Not assigned</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>freq</th>\n",
              "      <td>1</td>\n",
              "      <td>77</td>\n",
              "      <td>77</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "       Postalcode       Borough Neighbourhood\n",
              "count         180           180           180\n",
              "unique        180            12           100\n",
              "top           M2H  Not assigned  Not assigned\n",
              "freq            1            77            77"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 87
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xfa7OP0oI-7q"
      },
      "source": [
        "**Clean data**\n",
        "\n",
        "*   Remove rows with Borough = Not assigned\n",
        "*   If Neighborhood is not assigned replace it with Borough\n",
        "*   Combine similar postal code areas"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gkMK6BHZJAFG"
      },
      "source": [
        "df = df[df.Borough != 'Not assigned']\n",
        "i = 0\n",
        "for i in range(0,df.shape[0]):\n",
        "    if df.iloc[i][2] == 'Not assigned':\n",
        "        df.iloc[i][2] = df.iloc[i][1]\n",
        "        i = i+1\n",
        "df = df.groupby(['Postalcode', 'Borough'], sort = False).agg(','.join)\n",
        "df.reset_index(inplace = True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5PY22fuKU1sW"
      },
      "source": [
        "Shape of the dataframe"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GKQCKiGCU3cu",
        "outputId": "f8e50abc-ca19-487b-f9d4-f3ebdbd1fc0a"
      },
      "source": [
        "df.shape"
      ],
      "execution_count": 96,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(103, 3)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 96
        }
      ]
    }
  ]
}