_BlogTagPredictorProject[Completed].ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from bs4 import BeautifulSoup\n",
    "import urllib3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "http=urllib3.PoolManager()\n",
    "from pandas import DataFrame\n",
    "column=['Title','Body']\n",
    "dfBA=DataFrame(columns=column)\n",
    "dfT=DataFrame(columns=[0,1,2,3,4])\n",
    "def CreateDataFrame(someList):\n",
    "    t={}\n",
    "    d={'Title':[someList[0]],'Body':[someList[1]]}\n",
    "    for n in range(5):\n",
    "        if len(someList[2])>n:\n",
    "            t[n]=[someList[2][n]]\n",
    "        else:\n",
    "            t[n]=['0']\n",
    "    toDf=DataFrame(data=d)\n",
    "    global dfBA,dfT\n",
    "    #print(dfBA)\n",
    "    dfBA=dfBA.append(toDf)\n",
    "    #print(dfBA)\n",
    "    dfT=dfT.append(DataFrame(data=t))\n",
    "def CrawlAndFrame(link):\n",
    "    try:\n",
    "        print(link)\n",
    "        blogData=http.request('GET',link)\n",
    "        soup=BeautifulSoup(blogData.data,'html.parser')\n",
    "        article=''\n",
    "        tags=[]\n",
    "        heading=soup.find('h1').text\n",
    "        for para in soup.find_all('p'):\n",
    "            p=para.text\n",
    "            #p=p.strip('\\u')\n",
    "            article=article+' '+p\n",
    "        for mtags in soup.find_all('a',{'class':'link u-baseColor--link'}):\n",
    "            tags.append(mtags.text)\n",
    "        #CreateDataFrame(list())\n",
    "        someList=[heading,article,tuple(tags)]\n",
    "        #print(someList[0])\n",
    "        CreateDataFrame(someList)\n",
    "    except:\n",
    "        pass\n",
    "def spider(link):\n",
    "    print(link)\n",
    "    blogData=http.request('GET',link)\n",
    "    soup=BeautifulSoup(blogData.data,'html.parser')\n",
    "    for links in soup.find_all('div',{'class':'postArticle-readMore'}):\n",
    "        link=links.find('a').get('href')\n",
    "        CrawlAndFrame(link)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://medium.com/search?q=machine%20learning\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://medium.com/@ageitgey/machine-learning-is-fun-80ea3ec3c471?source=search_post---------0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n",
      "/home/indresh/.local/lib/python3.5/site-packages/pandas/core/frame.py:6211: FutureWarning: Sorting because non-concatenation axis is not aligned. A future version\n",
      "of pandas will change to not sort by default.\n",
      "\n",
      "To accept the future behavior, pass 'sort=False'.\n",
      "\n",
      "To retain the current behavior and silence the warning, pass 'sort=True'.\n",
      "\n",
      "  sort=sort)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://medium.freecodecamp.org/every-single-machine-learning-course-on-the-internet-ranked-by-your-reviews-3c4a7b8026c0?source=search_post---------1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n",
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n",
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://towardsdatascience.com/a-tour-of-the-top-10-algorithms-for-machine-learning-newbies-dde4edffae11?source=search_post---------2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n",
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n",
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://becominghuman.ai/cheat-sheets-for-ai-neural-networks-machine-learning-deep-learning-big-data-678c51b4b463?source=search_post---------3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n",
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n",
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://medium.com/@ageitgey/machine-learning-is-fun-part-4-modern-face-recognition-with-deep-learning-c3cffc121d78?source=search_post---------4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://medium.com/@ageitgey/machine-learning-is-fun-part-3-deep-learning-and-convolutional-neural-networks-f40359318721?source=search_post---------5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://machinelearnings.co/a-humans-guide-to-machine-learning-e179f43b67a0?source=search_post---------6\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n",
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n",
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://startupsventurecapital.com/essential-cheat-sheets-for-machine-learning-and-deep-learning-researchers-efb6a8ebd2e5?source=search_post---------7\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n",
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n",
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://medium.com/@ageitgey/machine-learning-is-fun-part-2-a26a10b68df3?source=search_post---------8\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://medium.mybridge.co/30-amazing-machine-learning-projects-for-the-past-year-v-2018-b853b8621ac7?source=search_post---------9\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n",
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n",
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://medium.com/search?q=deep%20learning\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://medium.freecodecamp.org/want-to-know-how-deep-learning-works-heres-a-quick-guide-for-everyone-1aedeca88076?source=search_post---------0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n",
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n",
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://becominghuman.ai/cheat-sheets-for-ai-neural-networks-machine-learning-deep-learning-big-data-678c51b4b463?source=search_post---------1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n",
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n",
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://medium.com/basic-income/deep-learning-is-going-to-teach-us-all-the-lesson-of-our-lives-jobs-are-for-machines-7c6442e37a49?source=search_post---------2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://medium.com/@ageitgey/machine-learning-is-fun-part-4-modern-face-recognition-with-deep-learning-c3cffc121d78?source=search_post---------3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://medium.com/@ageitgey/machine-learning-is-fun-part-3-deep-learning-and-convolutional-neural-networks-f40359318721?source=search_post---------4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://startupsventurecapital.com/essential-cheat-sheets-for-machine-learning-and-deep-learning-researchers-efb6a8ebd2e5?source=search_post---------5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n",
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n",
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://medium.com/@andrewng/deeplearning-ai-announcing-new-deep-learning-courses-on-coursera-43af0a368116?source=search_post---------6\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://towardsdatascience.com/thoughts-after-taking-the-deeplearning-ai-courses-8568f132153?source=search_post---------7\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n",
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n",
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://towardsdatascience.com/how-to-learn-deep-learning-in-6-months-e45e40ef7d48?source=search_post---------8\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n",
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n",
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://medium.freecodecamp.org/recognizing-traffic-lights-with-deep-learning-23dae23287cc?source=search_post---------9\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n",
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n",
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://medium.com/search?q=artificial%20intelligence\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://medium.com/@mijordan3/artificial-intelligence-the-revolution-hasnt-happened-yet-5e1d5812e1e7?source=search_post---------0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://machinelearnings.co/a-humans-guide-to-machine-learning-e179f43b67a0?source=search_post---------1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n",
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n",
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://medium.com/@LiamHanel/a-list-of-artificial-intelligence-tools-you-can-use-today-for-personal-use-1-3-7f1b60b6c94f?source=search_post---------2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://medium.com/startup-grind/i-reverse-engineered-a-500m-artificial-intelligence-company-in-one-week-heres-the-full-story-d067cef99e1c?source=search_post---------3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://medium.com/mmc-writes/the-fourth-industrial-revolution-a-primer-on-artificial-intelligence-ai-ff5e7fffcae1?source=search_post---------4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://medium.com/iotforall/the-difference-between-artificial-intelligence-machine-learning-and-deep-learning-3aa67bff5991?source=search_post---------5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://42hire.com/how-artificial-intelligence-can-revolutionize-recruitment-de0936cca38c?source=search_post---------6\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n",
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n",
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://medium.com/@ow/google-is-redefining-mobile-with-artificial-intelligence-76c7160b04f9?source=search_post---------7\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://medium.com/numerai/a-new-cryptocurrency-for-coordinating-artificial-intelligence-on-numerai-9251a131419a?source=search_post---------8\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://bothsidesofthetable.com/mark-cuban-on-why-you-need-to-study-artificial-intelligence-or-youll-be-a-dinosaur-in-3-years-db3447bea1b4?source=search_post---------9\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n",
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n",
      "/home/indresh/.local/lib/python3.5/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n"
     ]
    }
   ],
   "source": [
    "url=['https://medium.com/search?q=machine%20learning','https://medium.com/search?q=deep%20learning','https://medium.com/search?q=artificial%20intelligence']\n",
    "for u in url:\n",
    "    spider(u)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Machine Learning</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Machine Learning</td>\n",
       "      <td>Artificial Intelligence</td>\n",
       "      <td>Tech</td>\n",
       "      <td>Programming</td>\n",
       "      <td>Technology</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Machine Learning</td>\n",
       "      <td>Algorithms</td>\n",
       "      <td>Data Science</td>\n",
       "      <td>Towards Data Science</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Machine Learning</td>\n",
       "      <td>Deep Learning</td>\n",
       "      <td>Artificial Intelligence</td>\n",
       "      <td>Neural Networks</td>\n",
       "      <td>Big Data</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Machine Learning</td>\n",
       "      <td>Artificial Intelligence</td>\n",
       "      <td>Deep Learning</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Machine Learning</td>\n",
       "      <td>Artificial Intelligence</td>\n",
       "      <td>Deep Learning</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Artificial Intelligence</td>\n",
       "      <td>Machine Learning</td>\n",
       "      <td>Startups</td>\n",
       "      <td>Tech</td>\n",
       "      <td>Technology</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Machine Learning</td>\n",
       "      <td>Artificial Intelligence</td>\n",
       "      <td>Deep Learning</td>\n",
       "      <td>Technology</td>\n",
       "      <td>Computer Science</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Artificial Intelligence</td>\n",
       "      <td>Machine Learning</td>\n",
       "      <td>Nintendo</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Machine Learning</td>\n",
       "      <td>Data Science</td>\n",
       "      <td>Artificial Intelligence</td>\n",
       "      <td>Programming</td>\n",
       "      <td>Tech</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Jonathan Sugumar</td>\n",
       "      <td>Technology</td>\n",
       "      <td>Startup</td>\n",
       "      <td>Business</td>\n",
       "      <td>Artificial Intelligence</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Machine Learning</td>\n",
       "      <td>Deep Learning</td>\n",
       "      <td>Artificial Intelligence</td>\n",
       "      <td>Neural Networks</td>\n",
       "      <td>Big Data</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Some rights reserved</td>\n",
       "      <td>Artificial Intelligence</td>\n",
       "      <td>Machine Learning</td>\n",
       "      <td>Basic Income</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Machine Learning</td>\n",
       "      <td>Artificial Intelligence</td>\n",
       "      <td>Deep Learning</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Machine Learning</td>\n",
       "      <td>Artificial Intelligence</td>\n",
       "      <td>Deep Learning</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Machine Learning</td>\n",
       "      <td>Artificial Intelligence</td>\n",
       "      <td>Deep Learning</td>\n",
       "      <td>Technology</td>\n",
       "      <td>Computer Science</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Machine Learning</td>\n",
       "      <td>Deep Learning</td>\n",
       "      <td>AI</td>\n",
       "      <td>Andrew Ng</td>\n",
       "      <td>Towards Data Science</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Machine Learning</td>\n",
       "      <td>Deep Learning</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Data Science</td>\n",
       "      <td>Self Driving Cars</td>\n",
       "      <td>Artificial Intelligence</td>\n",
       "      <td>Machine Learning</td>\n",
       "      <td>Programming</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Artificial Intelligence</td>\n",
       "      <td>Machine Learning</td>\n",
       "      <td>Statistics</td>\n",
       "      <td>Data Science</td>\n",
       "      <td>Technology</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Artificial Intelligence</td>\n",
       "      <td>Machine Learning</td>\n",
       "      <td>Startups</td>\n",
       "      <td>Tech</td>\n",
       "      <td>Technology</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Artificial Intelligence</td>\n",
       "      <td>Machine Learning</td>\n",
       "      <td>Deep Learning</td>\n",
       "      <td>IoT</td>\n",
       "      <td>Resources</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Machine Learning</td>\n",
       "      <td>Artificial Intelligence</td>\n",
       "      <td>Energy</td>\n",
       "      <td>Data Science</td>\n",
       "      <td>Startup</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Artificial Intelligence</td>\n",
       "      <td>Machine Learning</td>\n",
       "      <td>Venture Capital</td>\n",
       "      <td>Research</td>\n",
       "      <td>Insights</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Artificial Intelligence</td>\n",
       "      <td>Machine Learning</td>\n",
       "      <td>Deep Learning</td>\n",
       "      <td>IoT</td>\n",
       "      <td>Education</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Artificial Intelligence</td>\n",
       "      <td>Robots</td>\n",
       "      <td>Recruitment</td>\n",
       "      <td>Management</td>\n",
       "      <td>HR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Android</td>\n",
       "      <td>GoogleIO</td>\n",
       "      <td>Developer</td>\n",
       "      <td>Technology</td>\n",
       "      <td>Google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Bitcoin</td>\n",
       "      <td>Blockchain</td>\n",
       "      <td>Ethereum</td>\n",
       "      <td>Artificial Intelligence</td>\n",
       "      <td>Machine Learning</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Startup</td>\n",
       "      <td>Artificial Intelligence</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         0                        1                        2  \\\n",
       "0         Machine Learning                        0                        0   \n",
       "0         Machine Learning  Artificial Intelligence                     Tech   \n",
       "0         Machine Learning               Algorithms             Data Science   \n",
       "0         Machine Learning            Deep Learning  Artificial Intelligence   \n",
       "0         Machine Learning  Artificial Intelligence            Deep Learning   \n",
       "0         Machine Learning  Artificial Intelligence            Deep Learning   \n",
       "0  Artificial Intelligence         Machine Learning                 Startups   \n",
       "0         Machine Learning  Artificial Intelligence            Deep Learning   \n",
       "0  Artificial Intelligence         Machine Learning                 Nintendo   \n",
       "0         Machine Learning             Data Science  Artificial Intelligence   \n",
       "0         Jonathan Sugumar               Technology                  Startup   \n",
       "0         Machine Learning            Deep Learning  Artificial Intelligence   \n",
       "0     Some rights reserved  Artificial Intelligence         Machine Learning   \n",
       "0         Machine Learning  Artificial Intelligence            Deep Learning   \n",
       "0         Machine Learning  Artificial Intelligence            Deep Learning   \n",
       "0         Machine Learning  Artificial Intelligence            Deep Learning   \n",
       "0         Machine Learning            Deep Learning                       AI   \n",
       "0         Machine Learning            Deep Learning                        0   \n",
       "0             Data Science        Self Driving Cars  Artificial Intelligence   \n",
       "0  Artificial Intelligence         Machine Learning               Statistics   \n",
       "0  Artificial Intelligence         Machine Learning                 Startups   \n",
       "0  Artificial Intelligence         Machine Learning            Deep Learning   \n",
       "0         Machine Learning  Artificial Intelligence                   Energy   \n",
       "0  Artificial Intelligence         Machine Learning          Venture Capital   \n",
       "0  Artificial Intelligence         Machine Learning            Deep Learning   \n",
       "0  Artificial Intelligence                   Robots              Recruitment   \n",
       "0                  Android                 GoogleIO                Developer   \n",
       "0                  Bitcoin               Blockchain                 Ethereum   \n",
       "0                  Startup  Artificial Intelligence                        0   \n",
       "\n",
       "                         3                        4  \n",
       "0                        0                        0  \n",
       "0              Programming               Technology  \n",
       "0     Towards Data Science                        0  \n",
       "0          Neural Networks                 Big Data  \n",
       "0                        0                        0  \n",
       "0                        0                        0  \n",
       "0                     Tech               Technology  \n",
       "0               Technology         Computer Science  \n",
       "0                        0                        0  \n",
       "0              Programming                     Tech  \n",
       "0                 Business  Artificial Intelligence  \n",
       "0          Neural Networks                 Big Data  \n",
       "0             Basic Income                        0  \n",
       "0                        0                        0  \n",
       "0                        0                        0  \n",
       "0               Technology         Computer Science  \n",
       "0                Andrew Ng     Towards Data Science  \n",
       "0                        0                        0  \n",
       "0         Machine Learning              Programming  \n",
       "0             Data Science               Technology  \n",
       "0                     Tech               Technology  \n",
       "0                      IoT                Resources  \n",
       "0             Data Science                  Startup  \n",
       "0                 Research                 Insights  \n",
       "0                      IoT                Education  \n",
       "0               Management                       HR  \n",
       "0               Technology                   Google  \n",
       "0  Artificial Intelligence         Machine Learning  \n",
       "0                        0                        0  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "okList=[]\n",
    "for cl in dfT.columns:\n",
    "     for n in dfT[cl]:\n",
    "            okList.append(n)\n",
    "okList=list(set(okList))\n",
    "del(okList[okList.index('0')])\n",
    "newDF=DataFrame(columns=okList)\n",
    "for x in range(dfT.count()[0]):\n",
    "    someDict={}\n",
    "    for d in okList:\n",
    "        rowdata=list(dfT.iloc[x])\n",
    "        if d in rowdata:\n",
    "            someDict[d]=1\n",
    "        else:\n",
    "            someDict[d]=0\n",
    "    newDF=newDF.append(someDict,ignore_index=True)\n",
    "newDF.to_csv('blogData.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "categories = list(newDF.columns.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "counts=[]\n",
    "for i in categories:\n",
    "    counts.append((i, newDF[i].sum()))\n",
    "df_stats = pd.DataFrame(counts, columns=['category', 'number_of_articles'])\n",
    "df_stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "df_stats.plot(x='category', y='number_of_articles', kind='bar', legend=False, grid=True, figsize=(8, 5))\n",
    "plt.title(\"Number of comments per category\")\n",
    "plt.ylabel('# of Occurrences', fontsize=12)\n",
    "plt.xlabel('category', fontsize=12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfBA.index=range(dfBA.count()[0])\n",
    "df=dfBA.join(newDF)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "rowsums = df.iloc[:,2:].sum(axis=1)\n",
    "x=rowsums.value_counts()\n",
    "#plot\n",
    "plt.figure(figsize=(8,5))\n",
    "ax = sns.barplot(x.index, x.values)\n",
    "plt.title(\"Multiple categories per article\")\n",
    "plt.ylabel('# of Occurrences', fontsize=12)\n",
    "plt.xlabel('# of categories', fontsize=12)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "lens = df.Body.str.len()\n",
    "lens.hist(bins = np.arange(900,20000,1000))\n",
    "print(lens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.corpus import stopwords\n",
    "stopWordList=stopwords.words('english')\n",
    "stopWordList.remove('no')\n",
    "stopWordList.remove('not')\n",
    "def removeTags(data):\n",
    "    soup=BeautifulSoup(data,'html.parser')\n",
    "    text=soup.get_text()\n",
    "    return text\n",
    "import unicodedata\n",
    "import spacy\n",
    "#nlp = spacy.load('en_core', parse=True, tag=True, entity=True)\n",
    "def removeAscendingChar(data):\n",
    "    data=unicodedata.normalize('NFKD', data).encode('ascii', 'ignore').decode('utf-8', 'ignore')\n",
    "    return data\n",
    "def removeCharDigit(text):\n",
    "    str='`1234567890-=~@#$%^&*()_+[!{;\":\\'><.,/?\"}]'\n",
    "    for w in text:\n",
    "        if w in str:\n",
    "            text=text.replace(w,'')\n",
    "    return text\n",
    "from nltk.stem.wordnet import WordNetLemmatizer\n",
    "from nltk.tokenize import ToktokTokenizer\n",
    "lemma=WordNetLemmatizer()\n",
    "token=ToktokTokenizer()\n",
    "def lemitizeWords(text):\n",
    "    words=token.tokenize(text)\n",
    "    listLemma=[]\n",
    "    for w in words:\n",
    "        x=lemma.lemmatize(w,'v')\n",
    "        listLemma.append(x)\n",
    "    return text\n",
    "def stopWordsRemove(text):\n",
    "    wordList=[x.lower().strip() for x in token.tokenize(text)]\n",
    "    removedList=[x for x in wordList if not x in stopWordList]\n",
    "    text=' '.join(removedList)\n",
    "    return text\n",
    "def PreProcessing(text):\n",
    "    text=removeTags(text)\n",
    "    text=removeCharDigit(text)\n",
    "    text=removeAscendingChar(text)\n",
    "    text=lemitizeWords(text)\n",
    "    text=stopWordsRemove(text)\n",
    "    return(text)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "totalText=''\n",
    "for x in df['Body']:\n",
    "    ps=PreProcessing(x)\n",
    "    totalText=totalText+\" \"+ps\n",
    "from wordcloud import WordCloud\n",
    "wc=WordCloud(background_color='black',max_font_size=50).generate(totalText)\n",
    "plt.figure(figsize=(16,12))\n",
    "plt.imshow(wc, interpolation=\"bilinear\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "from nltk.tokenize import ToktokTokenizer\n",
    "x=nltk.FreqDist(ToktokTokenizer().tokenize(totalText))\n",
    "plt.figure(figsize=(16,5))\n",
    "x.plot(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "def clean_text(text):\n",
    "    text = text.lower()\n",
    "    text = re.sub(r\"what's\", \"what is \", text)\n",
    "    text = re.sub(r\"\\'s\", \" \", text)\n",
    "    text = re.sub(r\"\\'ve\", \" have \", text)\n",
    "    text = re.sub(r\"can't\", \"can not \", text)\n",
    "    text = re.sub(r\"n't\", \" not \", text)\n",
    "    text = re.sub(r\"i'm\", \"i am \", text)\n",
    "    text = re.sub(r\"\\'re\", \" are \", text)\n",
    "    text = re.sub(r\"\\'d\", \" would \", text)\n",
    "    text = re.sub(r\"\\'ll\", \" will \", text)\n",
    "    text = re.sub(r\"\\'scuse\", \" excuse \", text)\n",
    "    text = re.sub('\\W', ' ', text)\n",
    "    text = re.sub('\\s+', ' ', text)\n",
    "    text = text.strip(' ')\n",
    "    return text\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['Body'] = df['Body'].map(lambda com : clean_text(com))\n",
    "df['Body'] = df['Body'].map(lambda com : PreProcessing(com))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "train,test=train_test_split(df,random_state=42,shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "xtrain = train.Body\n",
    "xtest = test.Body\n",
    "train['Neural Networks']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "from sklearn.pipeline import Pipeline\n",
    "# Define a pipeline combining a text feature extractor with multi lable classifier\n",
    "NB_pipeline = Pipeline([\n",
    "                ('tfidf', TfidfVectorizer()),\n",
    "                ('clf', OneVsRestClassifier(MultinomialNB(\n",
    "                    fit_prior=True, class_prior=None))),\n",
    "            ])\n",
    "for category in categories:\n",
    "    print('... Processing {}'.format(category))\n",
    "    # train the model using X_dtm & y\n",
    "    NB_pipeline.fit(xtrain, train[category].values.astype(int))\n",
    "    # compute the testing accuracy\n",
    "    prediction = NB_pipeline.predict(xtest)\n",
    "    print('Test accuracy is {}'.format(accuracy_score(test[category].values.astype(int), prediction)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "LogReg_pipeline = Pipeline([\n",
    "                ('tfidf', TfidfVectorizer()),\n",
    "                ('clf', OneVsRestClassifier(LogisticRegression(solver='sag'), n_jobs=1)),\n",
    "            ])\n",
    "for category in categories:\n",
    "    print('... Processing {}'.format(category))\n",
    "    # train the model using X_dtm & y\n",
    "    LogReg_pipeline.fit(xtrain, train[category].values.astype(int))\n",
    "    # compute the testing accuracy\n",
    "    prediction = LogReg_pipeline.predict(xtest)\n",
    "    print('Test accuracy is {}'.format(accuracy_score(test[category].values.astype(int), prediction)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}