ke_media_scraper.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "ke-media_scraper.ipynb",
      "provenance": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NmDzqRecunMQ",
        "colab_type": "text"
      },
      "source": [
        "# Set up Essentials"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oljWmv2fuvfN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# !pip install pygithub"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MX6bRjOdu35E",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from google.colab import drive , files\n",
        "import os, os.path\n",
        "from github import Github\n",
        "import requests\n",
        "from urllib.request import urlopen"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8CP1uNiiu7uq",
        "colab_type": "text"
      },
      "source": [
        "## **Mount Drive and Operate from within said folder**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BF3sZ1xlvJL4",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "1b9b4d7e-c8d7-495a-e8b2-0e1e1f3891a8"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cl6MU0c6vMJ_",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "ecec9c2c-587f-4fbe-c3b3-cc6e5359b2a6"
      },
      "source": [
        "%cd /content/drive/'My Drive'/Colab_Notebooks/news_tools"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/drive/My Drive/Colab_Notebooks/news_tools\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OzOwOGllvsEA",
        "colab_type": "text"
      },
      "source": [
        "## **Get the toolbox out**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5e1LAbCvwkAO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "links = []\n",
        "articles = []\n",
        "\n",
        "Daily_nation = ['newstools/2010-daily-nation','newstools/2012-daily-nation','newstools/2013-daily-nation', 'newstools/2016-daily-nation',\n",
        "                'newstools/2017-daily-nation','newstools/2018-daily-nation','newstools/2019-daily-nation']\n",
        "\n",
        "BD = [\"2017-business-daily-kenya\",\"2018-business-daily-kenya\",\"2019-business-daily-kenya\"]\n",
        "Standard_digital=[\"2015-standard-digital\",\"2016-standard-digital\",\"2017-standard-digital\",\n",
        "                  \"2018-standard-digital\",\"2019-standard-digital\"]\n",
        "Star=[\"2013-the-star\",\"2014-the-star\",\"2015-the-star\",\"2016-the-star\",\n",
        "        \"2017-the-star\",\"2018-the-star\",\"2019-the-star\"]\n",
        "EA=[\"1970-the-east-african\",\"2015-the-east-african\",\"2016-the-east-african\",\n",
        "      \"2017-the-east-african\",\"2018-the-east-african\",\"2019-the-east-african\"]\n",
        "\n",
        "kenya_media = [Daily_nation, BD,Standard_digital, Star, EA]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yeY56yaHwoVU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "base_url = \"https://github.com/\"\n",
        "raw_url = \"https://raw.githubusercontent.com/\"\n",
        "tree = \"/master/\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "s0zksXntwOI8",
        "colab_type": "text"
      },
      "source": [
        "# Fire the hose"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JwtgiynpwjRs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# get_links(media)\n",
        "\n",
        "# collect_repos(links)\n",
        "\n",
        "i = 0\n",
        "for media in kenya_media:\n",
        "  get_links(media)\n",
        "  for proj in media:\n",
        "    proj = \"newstools/\"+proj\n",
        "    repo = g.get_repo(proj)\n",
        "    contents = repo.get_contents(\"\") \n",
        "    while contents:\n",
        "        file_content = contents.pop(0)\n",
        "        if file_content.type == \"dir\":\n",
        "            contents.extend(repo.get_contents(file_content.path))\n",
        "        else:\n",
        "            article_path = file_content.path\n",
        "\n",
        "            # print(article_path)\n",
        "            # articles.append(article_path)\n",
        "\n",
        "            # collect_and_save(articles)\n",
        "            article_url = raw_url+ proj + tree + str(file_content.path)\n",
        "            print(article_url)\n",
        "\n",
        "            data = urlopen(article_url).read() #.decode(\"utf-8\")\n",
        "          #     save to file\n",
        "            year = article_url.split('/')[4]\n",
        "            month = article_url.split('/')[6]\n",
        "\n",
        "            # print(year +\":>><<:\"+month)\n",
        "\n",
        "            Dir = str(year)+\"/\"+str(month)\n",
        "\n",
        "            file_path = './media/'+ Dir # Change path to suit local dir\n",
        "            # file_name = data[:100].title()+'.txt'\n",
        "            file_name = str(article_path.split('/')[1])+'.txt'\n",
        "\n",
        "            # print(file_name)\n",
        "            \n",
        "            if not os.path.exists(file_path):\n",
        "                os.makedirs(file_path)\n",
        "\n",
        "            # if not os.path.isfile(file_path+file_name):\n",
        "            with open(file_path+\"/\"+file_name,'wb') as f:\n",
        "              f.write(data)\n",
        "              f.close()\n",
        "              i += 1\n",
        "  print ( str(i) +\" : no of files saved\")"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}