Bi Lingual Fake News Detection.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/alanherprovat/ai-project/blob/main/Bi%20Lingual%20Fake%20News%20Detection.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "oEiExkf99Dre"
      },
      "outputs": [],
      "source": [
        "#importing libraries\n",
        "\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.metrics import classification_report,accuracy_score\n",
        "import nltk\n",
        "from nltk.corpus import stopwords\n",
        "import re\n",
        "import string"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "wx1N0YmG9FlK"
      },
      "outputs": [],
      "source": [
        "#load dataset \n",
        "from google.colab import files\n",
        "#u1= files.upload()\n",
        "#u2= files.upload()\n",
        "url1='https://raw.githubusercontent.com/alanherprovat/ai-project/main/eglishdata.csv'\n",
        "url2='https://raw.githubusercontent.com/alanherprovat/ai-project/main/spanishdata.csv'\n",
        "df1=pd.read_csv(url1)\n",
        "df2=pd.read_csv(url2)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "wqGHD10p9IAn",
        "outputId": "1e1fa66a-2565-453d-b73e-f70f9b7d2269"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-4fa22374-428a-4c6b-927e-f5c803086753\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>title</th>\n",
              "      <th>body</th>\n",
              "      <th>label</th>\n",
              "      <th>link</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Four ways Bob Corker skewered Donald Trump</td>\n",
              "      <td>Image copyright Getty Images\\nOn Sunday mornin...</td>\n",
              "      <td>1</td>\n",
              "      <td>http://www.bbc.com/news/world-us-canada-414191...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Linklater's war veteran comedy speaks to moder...</td>\n",
              "      <td>LONDON (Reuters) - “Last Flag Flying”, a comed...</td>\n",
              "      <td>1</td>\n",
              "      <td>https://www.reuters.com/article/us-filmfestiva...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Trump’s Fight With Corker Jeopardizes His Legi...</td>\n",
              "      <td>The feud broke into public view last week when...</td>\n",
              "      <td>1</td>\n",
              "      <td>https://www.nytimes.com/2017/10/09/us/politics...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Egypt's Cheiron wins tie-up with Pemex for Mex...</td>\n",
              "      <td>MEXICO CITY (Reuters) - Egypt’s Cheiron Holdin...</td>\n",
              "      <td>1</td>\n",
              "      <td>https://www.reuters.com/article/us-mexico-oil-...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Jason Aldean opens 'SNL' with Vegas tribute</td>\n",
              "      <td>Country singer Jason Aldean, who was performin...</td>\n",
              "      <td>1</td>\n",
              "      <td>http://www.cnn.com/videos/cnnmoney/2017/10/08/...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-4fa22374-428a-4c6b-927e-f5c803086753')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-4fa22374-428a-4c6b-927e-f5c803086753 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-4fa22374-428a-4c6b-927e-f5c803086753');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                                               title  \\\n",
              "0         Four ways Bob Corker skewered Donald Trump   \n",
              "1  Linklater's war veteran comedy speaks to moder...   \n",
              "2  Trump’s Fight With Corker Jeopardizes His Legi...   \n",
              "3  Egypt's Cheiron wins tie-up with Pemex for Mex...   \n",
              "4        Jason Aldean opens 'SNL' with Vegas tribute   \n",
              "\n",
              "                                                body  label  \\\n",
              "0  Image copyright Getty Images\\nOn Sunday mornin...      1   \n",
              "1  LONDON (Reuters) - “Last Flag Flying”, a comed...      1   \n",
              "2  The feud broke into public view last week when...      1   \n",
              "3  MEXICO CITY (Reuters) - Egypt’s Cheiron Holdin...      1   \n",
              "4  Country singer Jason Aldean, who was performin...      1   \n",
              "\n",
              "                                                link  \n",
              "0  http://www.bbc.com/news/world-us-canada-414191...  \n",
              "1  https://www.reuters.com/article/us-filmfestiva...  \n",
              "2  https://www.nytimes.com/2017/10/09/us/politics...  \n",
              "3  https://www.reuters.com/article/us-mexico-oil-...  \n",
              "4  http://www.cnn.com/videos/cnnmoney/2017/10/08/...  "
            ]
          },
          "execution_count": 3,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#print first & last 5 rows of datasets\n",
        "df1.head(5)\n",
        "#df1.tail(5)#,df2.head(5),df2.tail(5)\n",
        "#print last 5 rows of datasets\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "V-IQzNcn9Txb",
        "outputId": "e9eb0af3-b5b9-4036-f004-515b7a472347"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "((4009, 4), (3003, 6))"
            ]
          },
          "execution_count": 4,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#PRINT Shapes(no of rows and collmn)\n",
        "df1.shape,df2.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "95kFL4HS9Ygv",
        "outputId": "0b340f98-b516-49bf-a41d-92e5636f8f33"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "((3919, 4), (3002, 6))"
            ]
          },
          "execution_count": 5,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#check for duplicate and drop them\n",
        "df1.drop_duplicates(inplace= True)\n",
        "df2.drop_duplicates(inplace= True)\n",
        "\n",
        "#after duplicate checking\n",
        "df1.shape,df2.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hxbqloWnReHK",
        "outputId": "3fc95706-18b2-4bb2-d4e7-a704a2c2627a"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(                                                  title  \\\n",
              " 3999    Red Sox fire manager Farrell after five seasons   \n",
              " 4000  35 False Matches and 1 Erroneous Arrest As Pol...   \n",
              " 4001  Hervé Leroux, Creator of the Hervé Léger Banda...   \n",
              " 4002     The price tag for Pence's trip to Indianapolis   \n",
              " 4003  CNN and Globalist Exposed - Steve Quayle and A...   \n",
              " 4004                                    Trends to Watch   \n",
              " 4005  Trump Jr. Is Soon To Give A 30-Minute Speech F...   \n",
              " 4006        Ron Paul on Trump, Anarchism & the AltRight   \n",
              " 4007  China to accept overseas trial data in bid to ...   \n",
              " 4008  Vice President Mike Pence Leaves NFL Game Beca...   \n",
              " \n",
              "                                                    body  label  \\\n",
              " 3999  FILE PHOTO: Oct 25, 2013; St. Louis, MO, USA; ...      1   \n",
              " 4000  By Nicholas West\\nAll things biometric are swe...      0   \n",
              " 4001  Mr. Leroux was born Hervé Peugnet on May 30, 1...      1   \n",
              " 4002  Ethical questions loom after Vice President Mi...      1   \n",
              " 4003  Vietnam Is in Great Danger, You Must Publish a...      0   \n",
              " 4004  Trends to Watch\\n% of readers think this story...      0   \n",
              " 4005  Trump Jr. Is Soon To Give A 30-Minute Speech F...      0   \n",
              " 4006                                                NaN      0   \n",
              " 4007  SHANGHAI (Reuters) - China said it plans to ac...      1   \n",
              " 4008  Vice President Mike Pence Leaves NFL Game Beca...      0   \n",
              " \n",
              "                                                    link  \n",
              " 3999  https://www.reuters.com/article/us-baseball-ml...  \n",
              " 4000  https://www.activistpost.com/2017/09/false-arr...  \n",
              " 4001  https://www.nytimes.com/2017/10/09/obituaries/...  \n",
              " 4002  http://www.cnn.com/videos/politics/2017/10/09/...  \n",
              " 4003  http://beforeitsnews.com/u-s-politics/2017/10/...  \n",
              " 4004  http://beforeitsnews.com/sports/2017/09/trends...  \n",
              " 4005  http://beforeitsnews.com/u-s-politics/2017/10/...  \n",
              " 4006  https://www.activistpost.com/2017/09/ron-paul-...  \n",
              " 4007  https://www.reuters.com/article/us-china-pharm...  \n",
              " 4008  http://beforeitsnews.com/u-s-politics/2017/10/...  ,\n",
              "                                                   title  \\\n",
              " 2993           Corte prohíbe expresión “perros en misa”   \n",
              " 2994  Servihabitat: En 2016 el alquiler se encarecer...   \n",
              " 2995  Guindos dice que el cálculo del déficit de 201...   \n",
              " 2996  Consigue puesto en TM y de la felicidad comien...   \n",
              " 2997  Trump anuncia que aves migratorias no podrán r...   \n",
              " 2998  Simeone: \"No hay mejor camino para mejorar que...   \n",
              " 2999     Romper Huevos para Hacer Tortillas Libertarias   \n",
              " 3000  INPEC presenta brazaletes de oro para narcotra...   \n",
              " 3001  Interior cifra en 190 los yihadistas que viaja...   \n",
              " 3002  Lagrimón gana premio Emmy a mejor actor de rep...   \n",
              " \n",
              "                                                    body label  \\\n",
              " 2993  Al resolver en última instancia una tutela a t...  Fake   \n",
              " 2994  El alquiler de vivienda se encarecerá más de u...  True   \n",
              " 2995  El ministro de Economía y Competitividad en fu...  True   \n",
              " 2996  Un hecho curioso tuvo lugar ayer en la ruta B1...  Fake   \n",
              " 2997  “Por traidoras, antipatriotas y perdedoras” mi...  Fake   \n",
              " 2998  El entrenador del Atlético de Madrid, Diego Pa...  True   \n",
              " 2999  Romper Huevos para Hacer Tortillas Libertarias...  Fake   \n",
              " 3000  Las múltiples solicitudes de narcotraficantes,...  Fake   \n",
              " 3001  Política Interior cifra en 190 los yihadistas ...  True   \n",
              " 3002  El reconocido actor colombiano “Lagrimón” fue ...  Fake   \n",
              " \n",
              "                                                    link          Topic  \\\n",
              " 2993  http://www.actualidadpanamericana.com/corte-pr...       Politics   \n",
              " 2994  http://www.elmundo.es/economia/2016/10/25/580f...        Economy   \n",
              " 2995  http://www.europapress.es/economia/macroeconom...        Economy   \n",
              " 2996  http://www.actualidadpanamericana.com/consigue...        Society   \n",
              " 2997  http://www.actualidadpanamericana.com/trump-an...       Politics   \n",
              " 2998  http://www.europapress.es/deportes/futbol-0016...          Sport   \n",
              " 2999  http://beforeitsnews.com/politics/2012/08/romp...        Society   \n",
              " 3000  http://www.actualidadpanamericana.com/inpec-pr...       Politics   \n",
              " 3001  http://www.eldiario.es/politica/Interior-yihad...       Politics   \n",
              " 3002  http://www.actualidadpanamericana.com/lagrimon...  Entertainment   \n",
              " \n",
              "                           Source  \n",
              " 2993  actualidadpanamericana.com  \n",
              " 2994                Europa Press  \n",
              " 2995                Europa Press  \n",
              " 2996  actualidadpanamericana.com  \n",
              " 2997  actualidadpanamericana.com  \n",
              " 2998                Europa Press  \n",
              " 2999           beforeitsnews.com  \n",
              " 3000  actualidadpanamericana.com  \n",
              " 3001                Europa Press  \n",
              " 3002  actualidadpanamericana.com  )"
            ]
          },
          "execution_count": 6,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df1.tail(10),df2.tail(10)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "RCYcc9zWBGow",
        "outputId": "6e49e44d-f629-443f-aa82-f2054482820c"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-85acba16-4db7-45b9-be53-98148c24f3c1\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>title</th>\n",
              "      <th>body</th>\n",
              "      <th>label</th>\n",
              "      <th>link</th>\n",
              "      <th>Topic</th>\n",
              "      <th>Source</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Goirigolzarri aplaza la fusión de BMN y Bankia...</td>\n",
              "      <td>BANCA Tras el mandato del Frob Goirigolzarri a...</td>\n",
              "      <td>1</td>\n",
              "      <td>http://www.elmundo.es/economia/2016/10/04/57f3...</td>\n",
              "      <td>Economy</td>\n",
              "      <td>Europa Press</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Evo Morales critica a Almagro por su \"conducta...</td>\n",
              "      <td>El presidente de Bolivia, Evo Morales, ha crit...</td>\n",
              "      <td>1</td>\n",
              "      <td>http://www.europapress.es/internacional/notici...</td>\n",
              "      <td>Politics</td>\n",
              "      <td>Europa Press</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Errejón cree que PP y PSOE deben de estar nego...</td>\n",
              "      <td>Política Errejón cree que PP y PSOE deben de e...</td>\n",
              "      <td>1</td>\n",
              "      <td>http://www.eldiario.es/politica/Errejon-PP-PSO...</td>\n",
              "      <td>Politics</td>\n",
              "      <td>Europa Press</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>¿Adivinas cuántos países no tienen un Banco Ce...</td>\n",
              "      <td>¿Adivinas cuántos países no tienen un Banco Ce...</td>\n",
              "      <td>0</td>\n",
              "      <td>http://beforeitsnews.com/alternative/2015/06/a...</td>\n",
              "      <td>Economy</td>\n",
              "      <td>beforeitsnews.com</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Marchamalo entra en las bibliotecas de escrito...</td>\n",
              "      <td>El periodista Jesús Marchamalo ha entrado de n...</td>\n",
              "      <td>1</td>\n",
              "      <td>http://www.europapress.es/cultura/libros-00132...</td>\n",
              "      <td>Entertainment</td>\n",
              "      <td>Europa Press</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>995</th>\n",
              "      <td>Bruselas abre la puerta a \"explorar\" el coste ...</td>\n",
              "      <td>La Comisión Europea ha abierto la puerta este ...</td>\n",
              "      <td>1</td>\n",
              "      <td>http://www.europapress.es/economia/transportes...</td>\n",
              "      <td>Economy</td>\n",
              "      <td>Europa Press</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>996</th>\n",
              "      <td>Todas las solicitudes de nacionalidad por resi...</td>\n",
              "      <td>Todas las solicitudes de nacionalidad por resi...</td>\n",
              "      <td>1</td>\n",
              "      <td>http://www.europapress.es/sociedad/noticia-tod...</td>\n",
              "      <td>Society</td>\n",
              "      <td>Europa Press</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>997</th>\n",
              "      <td>Héctor Alterio experimenta \"el tierno viaje de...</td>\n",
              "      <td>El Teatro Bellas Artes de Madrid ha presentado...</td>\n",
              "      <td>1</td>\n",
              "      <td>http://www.europapress.es/cultura/exposiciones...</td>\n",
              "      <td>Entertainment</td>\n",
              "      <td>Europa Press</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>998</th>\n",
              "      <td>El Real Madrid busca crecer ante un Athletic m...</td>\n",
              "      <td>El Real Madrid buscará este domingo en el esta...</td>\n",
              "      <td>1</td>\n",
              "      <td>http://www.europapress.es/deportes/futbol-0016...</td>\n",
              "      <td>Sport</td>\n",
              "      <td>Europa Press</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>999</th>\n",
              "      <td>Norte: Feminicidios se agravan en Ciudad Juare...</td>\n",
              "      <td>Norte: Feminicidios se agravan en Ciudad Juare...</td>\n",
              "      <td>0</td>\n",
              "      <td>http://beforeitsnews.com/immigration/2012/03/n...</td>\n",
              "      <td>Politics</td>\n",
              "      <td>beforeitsnews.com</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>1000 rows × 6 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-85acba16-4db7-45b9-be53-98148c24f3c1')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-85acba16-4db7-45b9-be53-98148c24f3c1 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-85acba16-4db7-45b9-be53-98148c24f3c1');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                                                 title  \\\n",
              "0    Goirigolzarri aplaza la fusión de BMN y Bankia...   \n",
              "1    Evo Morales critica a Almagro por su \"conducta...   \n",
              "2    Errejón cree que PP y PSOE deben de estar nego...   \n",
              "3    ¿Adivinas cuántos países no tienen un Banco Ce...   \n",
              "4    Marchamalo entra en las bibliotecas de escrito...   \n",
              "..                                                 ...   \n",
              "995  Bruselas abre la puerta a \"explorar\" el coste ...   \n",
              "996  Todas las solicitudes de nacionalidad por resi...   \n",
              "997  Héctor Alterio experimenta \"el tierno viaje de...   \n",
              "998  El Real Madrid busca crecer ante un Athletic m...   \n",
              "999  Norte: Feminicidios se agravan en Ciudad Juare...   \n",
              "\n",
              "                                                  body  label  \\\n",
              "0    BANCA Tras el mandato del Frob Goirigolzarri a...      1   \n",
              "1    El presidente de Bolivia, Evo Morales, ha crit...      1   \n",
              "2    Política Errejón cree que PP y PSOE deben de e...      1   \n",
              "3    ¿Adivinas cuántos países no tienen un Banco Ce...      0   \n",
              "4    El periodista Jesús Marchamalo ha entrado de n...      1   \n",
              "..                                                 ...    ...   \n",
              "995  La Comisión Europea ha abierto la puerta este ...      1   \n",
              "996  Todas las solicitudes de nacionalidad por resi...      1   \n",
              "997  El Teatro Bellas Artes de Madrid ha presentado...      1   \n",
              "998  El Real Madrid buscará este domingo en el esta...      1   \n",
              "999  Norte: Feminicidios se agravan en Ciudad Juare...      0   \n",
              "\n",
              "                                                  link          Topic  \\\n",
              "0    http://www.elmundo.es/economia/2016/10/04/57f3...        Economy   \n",
              "1    http://www.europapress.es/internacional/notici...       Politics   \n",
              "2    http://www.eldiario.es/politica/Errejon-PP-PSO...       Politics   \n",
              "3    http://beforeitsnews.com/alternative/2015/06/a...        Economy   \n",
              "4    http://www.europapress.es/cultura/libros-00132...  Entertainment   \n",
              "..                                                 ...            ...   \n",
              "995  http://www.europapress.es/economia/transportes...        Economy   \n",
              "996  http://www.europapress.es/sociedad/noticia-tod...        Society   \n",
              "997  http://www.europapress.es/cultura/exposiciones...  Entertainment   \n",
              "998  http://www.europapress.es/deportes/futbol-0016...          Sport   \n",
              "999  http://beforeitsnews.com/immigration/2012/03/n...       Politics   \n",
              "\n",
              "                Source  \n",
              "0         Europa Press  \n",
              "1         Europa Press  \n",
              "2         Europa Press  \n",
              "3    beforeitsnews.com  \n",
              "4         Europa Press  \n",
              "..                 ...  \n",
              "995       Europa Press  \n",
              "996       Europa Press  \n",
              "997       Europa Press  \n",
              "998       Europa Press  \n",
              "999  beforeitsnews.com  \n",
              "\n",
              "[1000 rows x 6 columns]"
            ]
          },
          "execution_count": 7,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#converting text  value of spanish label collumn to integer\n",
        "\n",
        "for i in range(len(df2)):\n",
        "  p=df2['label'].values\n",
        "  if(p[i]==\"True\"):\n",
        "    df2['label'].replace({\"True\": 1},inplace=True)\n",
        "  elif(p[i]==\"Fake\"):\n",
        "    df2['label'].replace({\"Fake\": 0},inplace=True)\n",
        "\n",
        "df2.head(1000)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "id": "a3-DQGEuGco1",
        "outputId": "712c5ad0-b682-4479-d689-89a707f7a2b8"
      },
      "outputs": [
        {
          "data": {
            "application/javascript": [
              "\n",
              "    async function download(id, filename, size) {\n",
              "      if (!google.colab.kernel.accessAllowed) {\n",
              "        return;\n",
              "      }\n",
              "      const div = document.createElement('div');\n",
              "      const label = document.createElement('label');\n",
              "      label.textContent = `Downloading \"${filename}\": `;\n",
              "      div.appendChild(label);\n",
              "      const progress = document.createElement('progress');\n",
              "      progress.max = size;\n",
              "      div.appendChild(progress);\n",
              "      document.body.appendChild(div);\n",
              "\n",
              "      const buffers = [];\n",
              "      let downloaded = 0;\n",
              "\n",
              "      const channel = await google.colab.kernel.comms.open(id);\n",
              "      // Send a message to notify the kernel that we're ready.\n",
              "      channel.send({})\n",
              "\n",
              "      for await (const message of channel.messages) {\n",
              "        // Send a message to notify the kernel that we're ready.\n",
              "        channel.send({})\n",
              "        if (message.buffers) {\n",
              "          for (const buffer of message.buffers) {\n",
              "            buffers.push(buffer);\n",
              "            downloaded += buffer.byteLength;\n",
              "            progress.value = downloaded;\n",
              "          }\n",
              "        }\n",
              "      }\n",
              "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
              "      const a = document.createElement('a');\n",
              "      a.href = window.URL.createObjectURL(blob);\n",
              "      a.download = filename;\n",
              "      div.appendChild(a);\n",
              "      a.click();\n",
              "      div.remove();\n",
              "    }\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "download(\"download_6634354f-9267-4edf-b08f-9ef82784f1ad\", \"manual_testing.csv\", 52600)"
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "#### creating the manual testing csv file\n",
        "from google.colab import files\n",
        "df1_manual=df1.tail(10)\n",
        "df2_manual=df2.tail(10)\n",
        "\n",
        "manual_test=pd.concat([df1_manual,df2_manual],axis=0)\n",
        "manual_test.to_csv('manual_testing.csv')\n",
        "files.download('manual_testing.csv')\n",
        "\n",
        "for i in range(4008,3998,-1):\n",
        "  df1.drop([i],axis=0,inplace=True)\n",
        "for i in range(3002,2992,-1):\n",
        "  df2.drop([i],axis=0,inplace=True)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kEfgzFgQOCR9",
        "outputId": "c19e506a-19a8-4e8a-ab3e-ec21d9b23523"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(                                                  title  \\\n",
              " 3989  Margo Price, Nashville Outsider, Tells It Like...   \n",
              " 3990  Saturday: Two To Go - Jamo v Max Scherzer, Lin...   \n",
              " 3991                          Matchup: Bulls v Mud Hens   \n",
              " 3992  Iraq War Vet Gave Exact Location of Vegas Shoo...   \n",
              " 3993  10/2 Happenings: HBD Ernie,Honest Eddie, Rex, ...   \n",
              " 3994  Smart Meters Create A “Honeypot” Of Personal I...   \n",
              " 3995  Harvey Weinstein scandal: What next for Hollyw...   \n",
              " 3996                                 The Spy in the IRA   \n",
              " 3997  Trump Takes Bold Action to Save U.S. From Euro...   \n",
              " 3998  9/28 Through the 40s: The Gloaming; HBD Bill, ...   \n",
              " \n",
              "                                                    body  label  \\\n",
              " 3989  “I think all that strife, it kind of helped me...      1   \n",
              " 3990  Saturday: Two To Go – Jamo v Max Scherzer, Lin...      0   \n",
              " 3991  Matchup: Bulls v Mud Hens\\n% of readers think ...      0   \n",
              " 3992  By Jack Burns\\n \\nThe response time of the Las...      0   \n",
              " 3993  An Embattled Pharmaceutical Company That Sells...      0   \n",
              " 3994  By Catherine J. Frompovich\\nOver a year ago, r...      0   \n",
              " 3995  Image copyright Getty Images Image caption Har...      1   \n",
              " 3996  The Spy in the IRA\\n(Before It's News)\\nIn the...      0   \n",
              " 3997  Trump Takes Bold Action to Save U.S. From Euro...      0   \n",
              " 3998  No Getting Around it, The War Is Coming! Trump...      0   \n",
              " \n",
              "                                                    link  \n",
              " 3989  https://www.nytimes.com/2017/10/12/arts/music/...  \n",
              " 3990  http://beforeitsnews.com/sports/2017/09/saturd...  \n",
              " 3991  http://beforeitsnews.com/sports/2017/10/matchu...  \n",
              " 3992  https://www.activistpost.com/2017/10/iraq-war-...  \n",
              " 3993  http://beforeitsnews.com/sports/2017/10/102-ha...  \n",
              " 3994  https://www.activistpost.com/2017/10/smart-met...  \n",
              " 3995  http://www.bbc.com/news/entertainment-arts-415...  \n",
              " 3996  http://beforeitsnews.com/entertainment/2017/10...  \n",
              " 3997  http://beforeitsnews.com/u-s-politics/2017/10/...  \n",
              " 3998  http://beforeitsnews.com/sports/2017/09/928-th...  ,\n",
              "                                                   title  \\\n",
              " 2983    Máquinas de spinning ayudarán a generar energía   \n",
              " 2984  El juez excluye a la denunciante del caso Imel...   \n",
              " 2985  Maribel Verdú y Jorge Bosch protagonizan 'Inve...   \n",
              " 2986  Colegio de Abogados de Granada entrega distinc...   \n",
              " 2987  Una campaña municipal conciencia contra el aba...   \n",
              " 2988             Alicia Koplowitz vuelve a salir de FCC   \n",
              " 2989  La Roma empata en Empoli y hace más líder a la...   \n",
              " 2990  El presidente afgano afea ante Mogherini que l...   \n",
              " 2991  El Parlamento asturiano libera a Fernández (PS...   \n",
              " 2992  Los alevines de España recuperan el espíritu d...   \n",
              " \n",
              "                                                    body  label  \\\n",
              " 2983  Un convenio firmado hoy entre el Ministerio de...      0   \n",
              " 2984  Comunidad Valenciana El juez excluye a la denu...      1   \n",
              " 2985  Los actores Maribel Verdú y Jorge Bosch protag...      1   \n",
              " 2986  El Colegio de Abogados ha entregado en la maña...      1   \n",
              " 2987    \\nPresentada este viernes por el edil de Med...      1   \n",
              " 2988  Alicia Koplowitz ha vuelto a salir del capital...      1   \n",
              " 2989  La Roma cedió dos puntos en una visita a Empol...      1   \n",
              " 2990  El presidente afgano, Ashraf Ghani, ha afeado ...      1   \n",
              " 2991  Política El Parlamento asturiano libera a Fern...      1   \n",
              " 2992  Los alevines de la selección española de fútbo...      1   \n",
              " \n",
              "                                                    link          Topic  \\\n",
              " 2983  http://www.actualidadpanamericana.com/maquinas...        Science   \n",
              " 2984  http://www.eldiario.es/cv/imelsa-cultura-diput...  Entertainment   \n",
              " 2985  http://www.europapress.es/cultura/exposiciones...  Entertainment   \n",
              " 2986  http://www.europapress.es/sociedad/noticia-col...        Society   \n",
              " 2987  http://www.20minutos.es/noticia/2857594/0/camp...        Science   \n",
              " 2988  http://www.europapress.es/economia/noticia-ali...        Economy   \n",
              " 2989  http://www.europapress.es/deportes/futbol-0016...          Sport   \n",
              " 2990  http://www.europapress.es/internacional/notici...       Politics   \n",
              " 2991  http://www.eldiario.es/politica/Parlamento-Fer...       Politics   \n",
              " 2992  http://www.europapress.es/deportes/futbol-0016...          Sport   \n",
              " \n",
              "                           Source  \n",
              " 2983  actualidadpanamericana.com  \n",
              " 2984                Europa Press  \n",
              " 2985                Europa Press  \n",
              " 2986                Europa Press  \n",
              " 2987                Europa Press  \n",
              " 2988                Europa Press  \n",
              " 2989                Europa Press  \n",
              " 2990                Europa Press  \n",
              " 2991                Europa Press  \n",
              " 2992                Europa Press  )"
            ]
          },
          "execution_count": 9,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#after dropping the tails data \n",
        "df1.tail(10),df2.tail(10)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6EsIrv6pTI83",
        "outputId": "b448351b-8416-4fd9-e96b-62a8c0675e8d"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "((3909, 4), (2992, 6))"
            ]
          },
          "execution_count": 10,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df1.shape,df2.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "s9uaYQ_p9dwb",
        "outputId": "e117a314-f158-4f67-be9a-1f90030dc854"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(6901, 6)"
            ]
          },
          "execution_count": 11,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "merged_data = pd.concat([df1,df2])\n",
        "#shape of merged csv files\n",
        "merged_data.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "pFyawJPU9hLk",
        "outputId": "db4f0b05-c207-4a42-de92-36c2f5e329e9"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-5c1a0ac1-f7fd-492b-9f5b-0e0f7d6be217\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>title</th>\n",
              "      <th>body</th>\n",
              "      <th>label</th>\n",
              "      <th>link</th>\n",
              "      <th>Topic</th>\n",
              "      <th>Source</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Four ways Bob Corker skewered Donald Trump</td>\n",
              "      <td>Image copyright Getty Images\\nOn Sunday mornin...</td>\n",
              "      <td>1</td>\n",
              "      <td>http://www.bbc.com/news/world-us-canada-414191...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Linklater's war veteran comedy speaks to moder...</td>\n",
              "      <td>LONDON (Reuters) - “Last Flag Flying”, a comed...</td>\n",
              "      <td>1</td>\n",
              "      <td>https://www.reuters.com/article/us-filmfestiva...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Trump’s Fight With Corker Jeopardizes His Legi...</td>\n",
              "      <td>The feud broke into public view last week when...</td>\n",
              "      <td>1</td>\n",
              "      <td>https://www.nytimes.com/2017/10/09/us/politics...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Egypt's Cheiron wins tie-up with Pemex for Mex...</td>\n",
              "      <td>MEXICO CITY (Reuters) - Egypt’s Cheiron Holdin...</td>\n",
              "      <td>1</td>\n",
              "      <td>https://www.reuters.com/article/us-mexico-oil-...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Jason Aldean opens 'SNL' with Vegas tribute</td>\n",
              "      <td>Country singer Jason Aldean, who was performin...</td>\n",
              "      <td>1</td>\n",
              "      <td>http://www.cnn.com/videos/cnnmoney/2017/10/08/...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2087</th>\n",
              "      <td>Draghi dice que el BCE no ha discutido el fina...</td>\n",
              "      <td>El Consejo de Gobierno del Banco Central Europ...</td>\n",
              "      <td>1</td>\n",
              "      <td>http://www.europapress.es/economia/finanzas-00...</td>\n",
              "      <td>Economy</td>\n",
              "      <td>Europa Press</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2088</th>\n",
              "      <td>PP pide a PNV que sea \"serio\" y no \"maree a la...</td>\n",
              "      <td>Política PP pide a PNV que sea \"serio\" y no \"m...</td>\n",
              "      <td>1</td>\n",
              "      <td>http://www.eldiario.es/politica/PP-PNV-PSE-EH-...</td>\n",
              "      <td>Politics</td>\n",
              "      <td>Europa Press</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2089</th>\n",
              "      <td>Viajes: En Ecuador, joyas escondidas en la cos...</td>\n",
              "      <td>Mi cabaña en Playa Escondida era rústica pero ...</td>\n",
              "      <td>1</td>\n",
              "      <td>https://www.nytimes.com/2015/03/08/universal/e...</td>\n",
              "      <td>Society</td>\n",
              "      <td>nytimes.com</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2090</th>\n",
              "      <td>Nicolás Maduro también asume control de selecc...</td>\n",
              "      <td>Ya no es solo el poder judicial y la Asamblea....</td>\n",
              "      <td>0</td>\n",
              "      <td>http://www.actualidadpanamericana.com/nicolas-...</td>\n",
              "      <td>Sport</td>\n",
              "      <td>actualidadpanamericana.com</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2091</th>\n",
              "      <td>PNV cree que Aiete puede ser \"un paso\" para lo...</td>\n",
              "      <td>Política PNV cree que Aiete puede ser \"un paso...</td>\n",
              "      <td>1</td>\n",
              "      <td>http://www.eldiario.es/politica/PNV-Aiete-viol...</td>\n",
              "      <td>Politics</td>\n",
              "      <td>Europa Press</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>6000 rows × 6 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-5c1a0ac1-f7fd-492b-9f5b-0e0f7d6be217')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-5c1a0ac1-f7fd-492b-9f5b-0e0f7d6be217 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-5c1a0ac1-f7fd-492b-9f5b-0e0f7d6be217');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                                                  title  \\\n",
              "0            Four ways Bob Corker skewered Donald Trump   \n",
              "1     Linklater's war veteran comedy speaks to moder...   \n",
              "2     Trump’s Fight With Corker Jeopardizes His Legi...   \n",
              "3     Egypt's Cheiron wins tie-up with Pemex for Mex...   \n",
              "4           Jason Aldean opens 'SNL' with Vegas tribute   \n",
              "...                                                 ...   \n",
              "2087  Draghi dice que el BCE no ha discutido el fina...   \n",
              "2088  PP pide a PNV que sea \"serio\" y no \"maree a la...   \n",
              "2089  Viajes: En Ecuador, joyas escondidas en la cos...   \n",
              "2090  Nicolás Maduro también asume control de selecc...   \n",
              "2091  PNV cree que Aiete puede ser \"un paso\" para lo...   \n",
              "\n",
              "                                                   body  label  \\\n",
              "0     Image copyright Getty Images\\nOn Sunday mornin...      1   \n",
              "1     LONDON (Reuters) - “Last Flag Flying”, a comed...      1   \n",
              "2     The feud broke into public view last week when...      1   \n",
              "3     MEXICO CITY (Reuters) - Egypt’s Cheiron Holdin...      1   \n",
              "4     Country singer Jason Aldean, who was performin...      1   \n",
              "...                                                 ...    ...   \n",
              "2087  El Consejo de Gobierno del Banco Central Europ...      1   \n",
              "2088  Política PP pide a PNV que sea \"serio\" y no \"m...      1   \n",
              "2089  Mi cabaña en Playa Escondida era rústica pero ...      1   \n",
              "2090  Ya no es solo el poder judicial y la Asamblea....      0   \n",
              "2091  Política PNV cree que Aiete puede ser \"un paso...      1   \n",
              "\n",
              "                                                   link     Topic  \\\n",
              "0     http://www.bbc.com/news/world-us-canada-414191...       NaN   \n",
              "1     https://www.reuters.com/article/us-filmfestiva...       NaN   \n",
              "2     https://www.nytimes.com/2017/10/09/us/politics...       NaN   \n",
              "3     https://www.reuters.com/article/us-mexico-oil-...       NaN   \n",
              "4     http://www.cnn.com/videos/cnnmoney/2017/10/08/...       NaN   \n",
              "...                                                 ...       ...   \n",
              "2087  http://www.europapress.es/economia/finanzas-00...   Economy   \n",
              "2088  http://www.eldiario.es/politica/PP-PNV-PSE-EH-...  Politics   \n",
              "2089  https://www.nytimes.com/2015/03/08/universal/e...   Society   \n",
              "2090  http://www.actualidadpanamericana.com/nicolas-...     Sport   \n",
              "2091  http://www.eldiario.es/politica/PNV-Aiete-viol...  Politics   \n",
              "\n",
              "                          Source  \n",
              "0                            NaN  \n",
              "1                            NaN  \n",
              "2                            NaN  \n",
              "3                            NaN  \n",
              "4                            NaN  \n",
              "...                          ...  \n",
              "2087                Europa Press  \n",
              "2088                Europa Press  \n",
              "2089                 nytimes.com  \n",
              "2090  actualidadpanamericana.com  \n",
              "2091                Europa Press  \n",
              "\n",
              "[6000 rows x 6 columns]"
            ]
          },
          "execution_count": 12,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#**merged inforamtion**\n",
        "merged_data.head(6000)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 677
        },
        "id": "QRtvr9GP9nnO",
        "outputId": "a78b9862-4f50-4590-9719-764811745c25"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-d111bce3-b3a4-4b21-af86-a64e7901c735\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>title</th>\n",
              "      <th>body</th>\n",
              "      <th>label</th>\n",
              "      <th>link</th>\n",
              "      <th>Topic</th>\n",
              "      <th>Source</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>2617</th>\n",
              "      <td>U.S. Helicopter Crashes on Okinawa, Adding to ...</td>\n",
              "      <td>TOKYO — A transport helicopter made an emergen...</td>\n",
              "      <td>1</td>\n",
              "      <td>https://www.nytimes.com/2017/10/11/world/asia/...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1585</th>\n",
              "      <td>Euskadi, entre las 7 comunidades que crecerán ...</td>\n",
              "      <td>Euskadi, Baleares, Castilla-La Mancha, Cataluñ...</td>\n",
              "      <td>1</td>\n",
              "      <td>http://www.europapress.es/euskadi/noticia-eusk...</td>\n",
              "      <td>Economy</td>\n",
              "      <td>Europa Press</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2608</th>\n",
              "      <td>Navarro rescata al Barcelona ante el Unics Kazan</td>\n",
              "      <td>Navarro rescata al Barcelona ante el Unics Kaz...</td>\n",
              "      <td>1</td>\n",
              "      <td>http://www.europapress.es/deportes/baloncesto-...</td>\n",
              "      <td>Sport</td>\n",
              "      <td>Europa Press</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1111</th>\n",
              "      <td>Refugees - and forests - benefit as Kenya's he...</td>\n",
              "      <td>KAKUMA, Kenya (Thomson Reuters Foundation) - T...</td>\n",
              "      <td>1</td>\n",
              "      <td>https://www.reuters.com/article/us-kenya-energ...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1599</th>\n",
              "      <td>‘Dynasty’ or ‘Dallas’? Squaring Off in the Lil...</td>\n",
              "      <td>JACOBS A fig stuffed with goat cheese and wrap...</td>\n",
              "      <td>1</td>\n",
              "      <td>https://www.nytimes.com/2017/10/12/arts/televi...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1103</th>\n",
              "      <td>Desmienten que Santos haya usado el papamóvil ...</td>\n",
              "      <td>Punto final a los rumores sobre un supuesto us...</td>\n",
              "      <td>0</td>\n",
              "      <td>http://www.actualidadpanamericana.com/desmient...</td>\n",
              "      <td>Politics</td>\n",
              "      <td>actualidadpanamericana.com</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1785</th>\n",
              "      <td>Devuelven transacción de Isagen porque gobiern...</td>\n",
              "      <td>Amargo sinsabor deja en el gobierno nacional u...</td>\n",
              "      <td>0</td>\n",
              "      <td>http://www.actualidadpanamericana.com/devuelve...</td>\n",
              "      <td>Politics</td>\n",
              "      <td>actualidadpanamericana.com</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1536</th>\n",
              "      <td>How Mel Brooks' lady of dance made it in a man...</td>\n",
              "      <td>Image copyright Target Live Image caption Mel ...</td>\n",
              "      <td>1</td>\n",
              "      <td>http://www.bbc.com/news/entertainment-arts-415...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2959</th>\n",
              "      <td>El Real Madrid busca puntos obligados y con bu...</td>\n",
              "      <td>El Real Madrid buscará este martes en el estad...</td>\n",
              "      <td>1</td>\n",
              "      <td>http://www.europapress.es/deportes/futbol-0016...</td>\n",
              "      <td>Sport</td>\n",
              "      <td>Europa Press</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2732</th>\n",
              "      <td>Casado, sobre el 'powerpoint' para instruir en...</td>\n",
              "      <td>Política Casado, sobre el 'powerpoint' para in...</td>\n",
              "      <td>1</td>\n",
              "      <td>http://www.eldiario.es/politica/Casado-powerpo...</td>\n",
              "      <td>Politics</td>\n",
              "      <td>Europa Press</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>70</th>\n",
              "      <td>FEAFES-ARFES Pro Salud Mental aboga por \"rompe...</td>\n",
              "      <td>\" Romper las barreras, los estigmas y los prej...</td>\n",
              "      <td>1</td>\n",
              "      <td>http://www.europapress.es/la-rioja/noticia-fea...</td>\n",
              "      <td>Health</td>\n",
              "      <td>Europa Press</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2507</th>\n",
              "      <td>Federación le paga deuda a selección femenina ...</td>\n",
              "      <td>Al correo de las integrantes de la selección f...</td>\n",
              "      <td>0</td>\n",
              "      <td>http://www.actualidadpanamericana.com/federaci...</td>\n",
              "      <td>Sport</td>\n",
              "      <td>actualidadpanamericana.com</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2986</th>\n",
              "      <td>Colegio de Abogados de Granada entrega distinc...</td>\n",
              "      <td>El Colegio de Abogados ha entregado en la maña...</td>\n",
              "      <td>1</td>\n",
              "      <td>http://www.europapress.es/sociedad/noticia-col...</td>\n",
              "      <td>Society</td>\n",
              "      <td>Europa Press</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>140</th>\n",
              "      <td>Acción poética estaría tras cuenta de Twitter ...</td>\n",
              "      <td>La denuncia la hizo hoy el Centro Democrático....</td>\n",
              "      <td>0</td>\n",
              "      <td>http://www.actualidadpanamericana.com/accion-p...</td>\n",
              "      <td>Politics</td>\n",
              "      <td>actualidadpanamericana.com</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1313</th>\n",
              "      <td>Morata salva el liderato y otro discreto día e...</td>\n",
              "      <td>El Real Madrid se convirtió en el nuevo líder ...</td>\n",
              "      <td>1</td>\n",
              "      <td>http://www.europapress.es/deportes/futbol-0016...</td>\n",
              "      <td>Sport</td>\n",
              "      <td>Europa Press</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3021</th>\n",
              "      <td>Chicago Blackhawks defeat the salary cap menace</td>\n",
              "      <td>Chicago Blackhawks defeat the salary cap menac...</td>\n",
              "      <td>0</td>\n",
              "      <td>http://beforeitsnews.com/sports/2017/10/chicag...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2184</th>\n",
              "      <td>Piqué: \"Ganando al City casi te aseguras la pr...</td>\n",
              "      <td>Piqué: \"Ganando al City casi te aseguras la pr...</td>\n",
              "      <td>1</td>\n",
              "      <td>http://www.europapress.es/deportes/futbol-0016...</td>\n",
              "      <td>Sport</td>\n",
              "      <td>Europa Press</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2783</th>\n",
              "      <td>They Put Crackers On A Baking Sheet – When It ...</td>\n",
              "      <td>They Put Crackers On A Baking Sheet – When It ...</td>\n",
              "      <td>0</td>\n",
              "      <td>http://dailybuzzlive.com/they-put-crackers-on-...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3419</th>\n",
              "      <td>Indian court bans firecracker sales in Delhi a...</td>\n",
              "      <td>FILE PHOTO: A worker removes paper rolls after...</td>\n",
              "      <td>1</td>\n",
              "      <td>https://www.reuters.com/article/us-india-pollu...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1667</th>\n",
              "      <td>Con el árbol de navidad más grande del país, l...</td>\n",
              "      <td>A la presentación de sociedad de su gran árbol...</td>\n",
              "      <td>0</td>\n",
              "      <td>http://www.actualidadpanamericana.com/con-el-a...</td>\n",
              "      <td>Politics</td>\n",
              "      <td>actualidadpanamericana.com</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-d111bce3-b3a4-4b21-af86-a64e7901c735')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-d111bce3-b3a4-4b21-af86-a64e7901c735 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-d111bce3-b3a4-4b21-af86-a64e7901c735');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                                                  title  \\\n",
              "2617  U.S. Helicopter Crashes on Okinawa, Adding to ...   \n",
              "1585  Euskadi, entre las 7 comunidades que crecerán ...   \n",
              "2608   Navarro rescata al Barcelona ante el Unics Kazan   \n",
              "1111  Refugees - and forests - benefit as Kenya's he...   \n",
              "1599  ‘Dynasty’ or ‘Dallas’? Squaring Off in the Lil...   \n",
              "1103  Desmienten que Santos haya usado el papamóvil ...   \n",
              "1785  Devuelven transacción de Isagen porque gobiern...   \n",
              "1536  How Mel Brooks' lady of dance made it in a man...   \n",
              "2959  El Real Madrid busca puntos obligados y con bu...   \n",
              "2732  Casado, sobre el 'powerpoint' para instruir en...   \n",
              "70    FEAFES-ARFES Pro Salud Mental aboga por \"rompe...   \n",
              "2507  Federación le paga deuda a selección femenina ...   \n",
              "2986  Colegio de Abogados de Granada entrega distinc...   \n",
              "140   Acción poética estaría tras cuenta de Twitter ...   \n",
              "1313  Morata salva el liderato y otro discreto día e...   \n",
              "3021    Chicago Blackhawks defeat the salary cap menace   \n",
              "2184  Piqué: \"Ganando al City casi te aseguras la pr...   \n",
              "2783  They Put Crackers On A Baking Sheet – When It ...   \n",
              "3419  Indian court bans firecracker sales in Delhi a...   \n",
              "1667  Con el árbol de navidad más grande del país, l...   \n",
              "\n",
              "                                                   body  label  \\\n",
              "2617  TOKYO — A transport helicopter made an emergen...      1   \n",
              "1585  Euskadi, Baleares, Castilla-La Mancha, Cataluñ...      1   \n",
              "2608  Navarro rescata al Barcelona ante el Unics Kaz...      1   \n",
              "1111  KAKUMA, Kenya (Thomson Reuters Foundation) - T...      1   \n",
              "1599  JACOBS A fig stuffed with goat cheese and wrap...      1   \n",
              "1103  Punto final a los rumores sobre un supuesto us...      0   \n",
              "1785  Amargo sinsabor deja en el gobierno nacional u...      0   \n",
              "1536  Image copyright Target Live Image caption Mel ...      1   \n",
              "2959  El Real Madrid buscará este martes en el estad...      1   \n",
              "2732  Política Casado, sobre el 'powerpoint' para in...      1   \n",
              "70    \" Romper las barreras, los estigmas y los prej...      1   \n",
              "2507  Al correo de las integrantes de la selección f...      0   \n",
              "2986  El Colegio de Abogados ha entregado en la maña...      1   \n",
              "140   La denuncia la hizo hoy el Centro Democrático....      0   \n",
              "1313  El Real Madrid se convirtió en el nuevo líder ...      1   \n",
              "3021  Chicago Blackhawks defeat the salary cap menac...      0   \n",
              "2184  Piqué: \"Ganando al City casi te aseguras la pr...      1   \n",
              "2783  They Put Crackers On A Baking Sheet – When It ...      0   \n",
              "3419  FILE PHOTO: A worker removes paper rolls after...      1   \n",
              "1667  A la presentación de sociedad de su gran árbol...      0   \n",
              "\n",
              "                                                   link     Topic  \\\n",
              "2617  https://www.nytimes.com/2017/10/11/world/asia/...       NaN   \n",
              "1585  http://www.europapress.es/euskadi/noticia-eusk...   Economy   \n",
              "2608  http://www.europapress.es/deportes/baloncesto-...     Sport   \n",
              "1111  https://www.reuters.com/article/us-kenya-energ...       NaN   \n",
              "1599  https://www.nytimes.com/2017/10/12/arts/televi...       NaN   \n",
              "1103  http://www.actualidadpanamericana.com/desmient...  Politics   \n",
              "1785  http://www.actualidadpanamericana.com/devuelve...  Politics   \n",
              "1536  http://www.bbc.com/news/entertainment-arts-415...       NaN   \n",
              "2959  http://www.europapress.es/deportes/futbol-0016...     Sport   \n",
              "2732  http://www.eldiario.es/politica/Casado-powerpo...  Politics   \n",
              "70    http://www.europapress.es/la-rioja/noticia-fea...    Health   \n",
              "2507  http://www.actualidadpanamericana.com/federaci...     Sport   \n",
              "2986  http://www.europapress.es/sociedad/noticia-col...   Society   \n",
              "140   http://www.actualidadpanamericana.com/accion-p...  Politics   \n",
              "1313  http://www.europapress.es/deportes/futbol-0016...     Sport   \n",
              "3021  http://beforeitsnews.com/sports/2017/10/chicag...       NaN   \n",
              "2184  http://www.europapress.es/deportes/futbol-0016...     Sport   \n",
              "2783  http://dailybuzzlive.com/they-put-crackers-on-...       NaN   \n",
              "3419  https://www.reuters.com/article/us-india-pollu...       NaN   \n",
              "1667  http://www.actualidadpanamericana.com/con-el-a...  Politics   \n",
              "\n",
              "                          Source  \n",
              "2617                         NaN  \n",
              "1585                Europa Press  \n",
              "2608                Europa Press  \n",
              "1111                         NaN  \n",
              "1599                         NaN  \n",
              "1103  actualidadpanamericana.com  \n",
              "1785  actualidadpanamericana.com  \n",
              "1536                         NaN  \n",
              "2959                Europa Press  \n",
              "2732                Europa Press  \n",
              "70                  Europa Press  \n",
              "2507  actualidadpanamericana.com  \n",
              "2986                Europa Press  \n",
              "140   actualidadpanamericana.com  \n",
              "1313                Europa Press  \n",
              "3021                         NaN  \n",
              "2184                Europa Press  \n",
              "2783                         NaN  \n",
              "3419                         NaN  \n",
              "1667  actualidadpanamericana.com  "
            ]
          },
          "execution_count": 13,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "ds = merged_data.sample(frac=1)\n",
        "ds.head(20)\n",
        "#ds.head(200)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "x9j9hoNjUmsj"
      },
      "source": [
        "**dropping the title,link,topic,source collumns**\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 363
        },
        "id": "dHpp8XcOUlXl",
        "outputId": "0c17e654-b60f-4990-ced1-a731fa12527b"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-df96e83f-6815-4df6-9aca-38408cd0b185\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>body</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>2617</th>\n",
              "      <td>TOKYO — A transport helicopter made an emergen...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1585</th>\n",
              "      <td>Euskadi, Baleares, Castilla-La Mancha, Cataluñ...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2608</th>\n",
              "      <td>Navarro rescata al Barcelona ante el Unics Kaz...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1111</th>\n",
              "      <td>KAKUMA, Kenya (Thomson Reuters Foundation) - T...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1599</th>\n",
              "      <td>JACOBS A fig stuffed with goat cheese and wrap...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1103</th>\n",
              "      <td>Punto final a los rumores sobre un supuesto us...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1785</th>\n",
              "      <td>Amargo sinsabor deja en el gobierno nacional u...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1536</th>\n",
              "      <td>Image copyright Target Live Image caption Mel ...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2959</th>\n",
              "      <td>El Real Madrid buscará este martes en el estad...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2732</th>\n",
              "      <td>Política Casado, sobre el 'powerpoint' para in...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-df96e83f-6815-4df6-9aca-38408cd0b185')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-df96e83f-6815-4df6-9aca-38408cd0b185 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-df96e83f-6815-4df6-9aca-38408cd0b185');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                                                   body  label\n",
              "2617  TOKYO — A transport helicopter made an emergen...      1\n",
              "1585  Euskadi, Baleares, Castilla-La Mancha, Cataluñ...      1\n",
              "2608  Navarro rescata al Barcelona ante el Unics Kaz...      1\n",
              "1111  KAKUMA, Kenya (Thomson Reuters Foundation) - T...      1\n",
              "1599  JACOBS A fig stuffed with goat cheese and wrap...      1\n",
              "1103  Punto final a los rumores sobre un supuesto us...      0\n",
              "1785  Amargo sinsabor deja en el gobierno nacional u...      0\n",
              "1536  Image copyright Target Live Image caption Mel ...      1\n",
              "2959  El Real Madrid buscará este martes en el estad...      1\n",
              "2732  Política Casado, sobre el 'powerpoint' para in...      1"
            ]
          },
          "execution_count": 14,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "data=ds.drop(['title','link','Topic','Source'],axis=1)\n",
        "data.head(10)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WFKmXFZGV6BT",
        "outputId": "70f29e0d-b88a-481d-cde8-96a3851b368c"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(body     26\n",
              " label     0\n",
              " dtype: int64, (6901, 2))"
            ]
          },
          "execution_count": 15,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#cheking for null contents\n",
        "data.isnull().sum(),data.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BnkZBUeSXB1o",
        "outputId": "f872d271-6785-4675-e6f2-e4696e0f36e0"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(body     0\n",
              " label    0\n",
              " dtype: int64, (6875, 2))"
            ]
          },
          "execution_count": 16,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#removing the rows with null contents\n",
        "data.dropna(subset=['body'], inplace=True)\n",
        "data.isnull().sum(),data.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2X0ouoMCoTXB",
        "outputId": "118b2249-786b-4732-9b97-c4caf37eca59"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
          ]
        }
      ],
      "source": [
        "\n",
        "nltk.download('stopwords')\n",
        "from nltk.stem import PorterStemmer\n",
        "port_stem=PorterStemmer()\n",
        "#Stemming is the process of producing morphological variants of a root/base word. Stemming programs are commonly referred to as stemming algorithms or stemmers.\n",
        "# A stemming algorithm reduces the words “chocolates”, “chocolatey”, “choco” to the root word, “chocolate”\n",
        "#stopwords doesnot add much meaning to a sentence (is,a, the etc)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JRbj4x-BolPJ"
      },
      "outputs": [],
      "source": [
        "def process(body):\n",
        "  #remove puntuation\n",
        "  nopun=re.sub('[^a-zA-Zá-ü]',' ',body)\n",
        "  nopun=nopun.lower()\n",
        "  nopun=nopun.split()\n",
        "  nopun=[port_stem.stem(word) for word in nopun if word  not in stopwords.words(['english','spanish'])]\n",
        "  nopun=' '.join(nopun)\n",
        "\n",
        "  return nopun"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "dRhExS-tsJs8"
      },
      "outputs": [],
      "source": [
        "#removing all kinds of un necessary characters and stop words\n",
        "data['body']=data['body'].apply(process)\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "akXMP70yFF6e",
        "outputId": "09133f31-202a-4e6c-d2ee-1f580e5a42d7"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2617    tokyo transport helicopt made emerg land catch...\n",
            "1585    euskadi balear castilla mancha cataluña comuni...\n",
            "2608    navarro rescata barcelona unic kazan publicado...\n",
            "1111    kakuma kenya thomson reuter foundat turkana co...\n",
            "1599    jacob fig stuf goat chees wrap prosciutto play...\n",
            "                              ...                        \n",
            "3940    cnn massacr vega nation tragedi famili one ste...\n",
            "1578    get done black hole news offens throw outsid n...\n",
            "2387    obama deep state appointe burrow govern news g...\n",
            "3855    isaac davi cannabi legal gain ground unit stat...\n",
            "1551    tra secuencia captada partido equipo empató do...\n",
            "Name: body, Length: 6875, dtype: object\n"
          ]
        }
      ],
      "source": [
        "print(data['body'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "YqMZnYOAGDWM",
        "outputId": "d4d4b940-2ede-4cfe-c5e8-e6eed860db39"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "['tokyo transport helicopt made emerg land catch fire wednesday unit state militari train area northern part okinawa japan stir concern deploy craft near civilian area statement third marin expeditionari forc said helicopt take part routin train none crew member civilian injur accord japanes defens ministri helicopt ch made land taka district higashi villag japanes govern build helicopt land pad american militari project small fierc group resid protest year nhk public broadcast quot unit state militari say helicopt station futenma air base okinawa advertis continu read main stori itsunori onodera japan defens minist told report govern investig whether civilian hurt mr onodera said japan ask unit state immedi investig caus prevent recurr thoroughli implement safeti control suppli detail inform'\n",
            " 'euskadi balear castilla mancha cataluña comunitad valenciana galicia madrid siet comunidad autónoma crecerán encima media año seguida region aragón navarra mayoría superará media próximo año según pronosticado mart funca presentación prevision comunidad autónoma funca presentado zaragoza prevision comunidad autónoma advirtiendo prevé desaceleración conjunto próximo año si bien crecimiento alcanzará ciento mantendrá region eje mediterráneo vall ebro galicia madrid extremadura repunt superior media así explicado rueda prensa director gener funca carlo ocaña director coyuntura estadística fundación raymond torr acompañado director gener adjunto ibercaja francisco serrano asistir acto ofici presentación análisi desarrollado patio infanta presencia president aragón javier lambán president ibercaja amado franco paí vasco siet comunidad autónoma crecerá encima media gracia empuj construcción ocurrió indicado presentación sector ralentizarán reduciendo ritmo crecimiento pib término pib per cápita euskadi ampliará distancia respecto media española periodo tasa desempleo reducirá año próximo respectivament según prevision funca año senda recuperación sólida mantien do primero trimestr gracia diversificación economía siet comunidad balear castilla mancha cataluña comunitad valenciana galicia madrid paí vasco crecerán encima media ciento seguida aragón navarra mayoría superará media especialment aquella zona benefician turismo diversificación estructura productiva zona meno dinámica asturia cantabria castilla león extremadura logran aprovechar aug exportador ademá tienden perder población director gener funca indicado caso toda comunidad participan proceso recuperación economía aunqu hacen forma desigu recalcado trata recuperación diversificada apoya sector especi industria construcción recupera sector servicio desigualdad término pib per cápita mantendrán punto ocaña precisado año crecimiento diferencia tendían converg reducían crisi agrandaron ahora aumento parado va manten van reducir tasa desempleo destacado navarra única región paro bajará ciento mientra andalucía castilla mancha canaria extremadura mantendrán paro encima incluso ciento mientra cinco region aragón balear navarra paí vasco rioja sitúa debajo ciento'\n",
            " 'navarro rescata barcelona unic kazan publicado cet navarro rescata barcelona unic kazan ocho punto catalán último minuto permiten bartzoka estrenars victoria euroliga madrid oct europa press fc barcelona lassa consiguió estrenars victoria viern nueva edición euroliga unic kazan partido conjunto catalán despertó tiempo último cuarto mano gran juan carlo navarro apuntaba marcharían casa primera derrota equipo ruso máxima competición europea georgio bartzoka comenzaron partido errático defensa ademá perdieron balon ataqu part ruso aprovecharon venirs arriba rebot ofensivo latavi william encargado aprovechar máximo situación hacer daño conjunto azulgrana pese catalan conseguían lograr defensa férrea mantuvieron nivel contrincant dejaron escapars marcador unic kazan aumentó ritmo partido vino bien bartzoka pedía mucha insistencia mayor agresividad defensa segundo cuarto ambo conjunto mostraron fallon cara aro período surgió jugada espectacular encuentro keith langford vio perímetro william realizó gran mate alley oop incluido barça insistió línea tre do encontró fino topó varia vece metal justin doellman máximo anotador mvp partido punto tiró carro mejor primera part paso vestuario barcelones supieron reaccionar embestida tripl local recortaron distancia tra charla respectivo técnico partido dio giro amba escuadra dieron bajón cuanto intensidad barcelona mostró blando tercer cuarto supo aprovechar error rival lograr transicion ataqu si supo hacer unic kazan aumentando diferencia punto nueva máxima partido finalment intensa charla bartzoka surgieron efecto gran navarro echó equipo espalda lograr ocho punto seguido sirvieron poner nuevo barcelona delant minuto final conjunto ruso dio arreón suficient lograr victoria aparentement bolsillo ficha cnica resultado unic kazan fc barcelona lassa descanso equipo unic kazan antipov clark langford ponkrashov parakhouski quinteto inici colom voronov kaimakogl william fc barcelona lassa rice doellman claver oleson tomic quinteto inici navarro peno eriksson perperogl dorsey parcial rbitro pukl slo pastusiak pol vyklicki cze sede basket hall arena espectador'\n",
            " ...\n",
            " 'obama deep state appointe burrow govern news great articl like visit http www fixthisn com new report govern account offic watchdog organ meant keep tab feder govern show presid obama polit appointe burrow way deep state hold career posit trump administr accord report obtain washington time week appointe includ seven fail obtain permiss job switch offic personnel manag subsequ either deni job left accord nonetheless still leav obama polit appointe work trump administr mani reckon support trump agenda environment regul foreign polici illeg immigr spend cut forth guess none sure job best abil even align polit arch motiv administr peopl leak press thwart goal trump agenda intent sabotag peopl noth illeg presid allow polit appointe transfer career posit cours look proper light administr histori besieg much outright mutini one leak constantli flow press agenc head openli discuss ideolog differ within depart even presid made mention feder govern conspiraci frankli need mani peopl script caus hell lot troubl peopl across number differ agenc enough probabl scratch surfac come obama loyalist washington time point congress concern practic burrow time congression research servic report last year said burrow frequent occur transit period one administr prepar leav offic new administr come cr said practic foster belief individu convert career posit may seek undermin work new administr whose polici may odd espous serv appoint capac whether true realiti percept say know trump face extraordinari resist insid administr could help explain might great articl like visit http www fixthisn com sourc http www fixthisn com conserv break news obama deep state appointe burrow govern'\n",
            " 'isaac davi cannabi legal gain ground unit state nation evid emerg confirm societi benefit outlaw plant legal state peopl turn cannabi natur altern danger pharmaceut painkil judici system longer burden frivol possess case swat team longer dispatch raid pothead big money made tax revenu old habit die hard outdat mode think recent hit piec cannabi daili mail make claim smoke marijuana make peopl violent journalist steve doughti end clip new york citi polic commission bill bratton speak podcast natur drug violenc big appl bratton comment quot new york violenc see associ drug vast major around issu marijuana iron consid explos use heroin citi interestingli enough new york citi violenc see violenc around drug traffick involv marijuana scratch head see mani state want legal marijuana liber polici sourc nypd largest polic forc world better resembl stand nation armi almost employe foot soldier commission comment echo u attorney gener jeff session earlier year believ unhealthi practic current level thc marijuana high compar year ago see real violenc around expert tell violenc around marijuana one would think big money involv regard bratton comment new york post note comment came two year mayor blasio introduc nypd polici mandat person possess less gram marijuana get ticket instead arrest long id outstand warrant sourc word cannabi outlaw forc manag mitig predict neg side effect govern prohibit kind violenc associ alcohol industri commission bratton miss biggest point cannabi legal move product trade drug black market legitim market industri wherebi natur particip crimin becom economi creativ entrepreneurship fair trade practic deal longer made back alley gang lose whole lot turf drug relat violent crime involv marijuana replac crimin element sector legit busi peopl may actual end stop drug relat violent crime read articl isaac davi isaac davi staff writer wakingtim com offgridoutpost com surviv tip blog outspoken advoc liberti voluntari societi avid reader histori passion becom self suffici break free control matrix follow facebook articl nypd commission say drug violenc due cannabi entir miss point legal origin creat publish wake time publish creativ common licens attribut isaac davi wakingtim com may post freeli proper attribut author bio copyright statement'\n",
            " 'tra secuencia captada partido equipo empató do gole ghana loew introduc dedo índice derecho fosa nasal mismo costado encargada cartera salud alemania diana vöeller reaccionó fuertement creo sido suficient sabemo lejo ser primera vez vemo señor dedicado menester delant millon televident si nadi dice pue haga digo joachim loew buena vez abandon repugnant costumbr si quier tener pronto mal rato tribunal funcionaria allá aseguró trata asunto estético sino salud pública usted pued parec anécdotico respond bienestar población así crean hábito facilita aparición propagación número alto infeccion extraído vuelv ser ingerido organismo hecho caballero sabemo nariz pequeña metrópoli gérmene bacteria microbio vöeller inquieta particular menor sigan ejemplo duda ídolo referent señor pued haberl dado paí mucha alegría dado mucha preocupacion seguir ejemplo millon niño alemania decidan limpiars forma naric implica retroceso gravísimo abr ventana gigant reaparición incluso pandemia último dio dimensión histórica reclamo usted quizá recuerden costó generación abuelo lograr tipo costumbr superaran lograr gent dejara sacars mucosidad lavara mano despué hacer necesidad cubriera boca estornudar determinant aumentar significativament expectativa vida nacen nación añadió visiblement alterada cuestionado misma razón eurocopa adiestrador hizo quit tema afirmando veía inapropiado costumbr mundo aug orinoterapia terapia involucran líquido mucosidad corporal comentario comentario']\n",
            "[1 1 1 ... 0 0 0]\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "(None, None, (6875,), (6875,))"
            ]
          },
          "execution_count": null,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "x=data['body'].values\n",
        "y=data['label'].values\n",
        "print(x),print(y),x.shape,y.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "rsJfaP-1sqki",
        "outputId": "e1881320-9179-4ca9-b257-4b8808873a36"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "  (0, 38118)\t0.05042427582208952\n",
            "  (0, 21952)\t0.059532615043427335\n",
            "  (0, 69646)\t0.07092703487874205\n",
            "  (0, 17109)\t0.04857429677081841\n",
            "  (0, 64152)\t0.0653791202391223\n",
            "  (0, 37098)\t0.07437644147946958\n",
            "  (0, 71418)\t0.09738435635123042\n",
            "  (0, 60494)\t0.10016121655081357\n",
            "  (0, 57193)\t0.0653791202391223\n",
            "  (0, 12577)\t0.05635494820270157\n",
            "  (0, 36965)\t0.061749743839117334\n",
            "  (0, 6072)\t0.05032472521522529\n",
            "  (0, 48639)\t0.05677405364825163\n",
            "  (0, 36337)\t0.068745164009828\n",
            "  (0, 77036)\t0.053655037052886406\n",
            "  (0, 39400)\t0.1127851064436381\n",
            "  (0, 61763)\t0.04013910871927332\n",
            "  (0, 71895)\t0.044664915184492275\n",
            "  (0, 47456)\t0.06158734738113671\n",
            "  (0, 51713)\t0.24561561478417876\n",
            "  (0, 39850)\t0.12280780739208938\n",
            "  (0, 68719)\t0.03858808647318072\n",
            "  (0, 44840)\t0.052372202302952635\n",
            "  (0, 59769)\t0.04673832918516042\n",
            "  (0, 16915)\t0.04342548888467841\n",
            "  :\t:\n",
            "  (6874, 37381)\t0.08293840210003882\n",
            "  (6874, 39267)\t0.0569627281927473\n",
            "  (6874, 22255)\t0.07334648149086237\n",
            "  (6874, 59099)\t0.04842486511709925\n",
            "  (6874, 69251)\t0.05214929630193709\n",
            "  (6874, 20083)\t0.05647993475799328\n",
            "  (6874, 22644)\t0.044712337174737174\n",
            "  (6874, 72298)\t0.033304344734252464\n",
            "  (6874, 48684)\t0.08422566280509265\n",
            "  (6874, 43961)\t0.10086078998132136\n",
            "  (6874, 34431)\t0.034687406220442814\n",
            "  (6874, 26817)\t0.03933656160483848\n",
            "  (6874, 57265)\t0.03568079729688961\n",
            "  (6874, 45301)\t0.043706514434979506\n",
            "  (6874, 53396)\t0.036390701110072886\n",
            "  (6874, 78506)\t0.035180036406466095\n",
            "  (6874, 37538)\t0.042673827581710184\n",
            "  (6874, 72763)\t0.044356258585917956\n",
            "  (6874, 30963)\t0.03614392398012741\n",
            "  (6874, 55717)\t0.04938260250616437\n",
            "  (6874, 6718)\t0.0627911907367868\n",
            "  (6874, 23439)\t0.031456123416969385\n",
            "  (6874, 53772)\t0.03434468367883741\n",
            "  (6874, 6289)\t0.029888524107414938\n",
            "  (6874, 66394)\t0.05619793116748646\n"
          ]
        }
      ],
      "source": [
        "\n",
        "#converting the full body text to a matrix of tokens\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "\n",
        "vectorizer = TfidfVectorizer()\n",
        "x=vectorizer.fit_transform(x)\n",
        "#vectorizer.get_feature_names_out()\n",
        "#vectorizer.vocabulary_\n",
        "print(x)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "6keKS3FlYNe9"
      },
      "outputs": [],
      "source": [
        "#train and test data\n",
        "x_train,x_test,y_train,y_test= train_test_split(x,y,test_size=0.25, random_state=0)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3Q8IBFc3afuf"
      },
      "source": [
        "**NAive bayes classifier model**\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "V2HXNsI88zE0"
      },
      "outputs": [],
      "source": [
        "# naive bayes classifier model\n",
        "from sklearn.naive_bayes import MultinomialNB\n",
        "cs=MultinomialNB()\n",
        "classifier= MultinomialNB().fit(x_train,y_train)\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "vYan63sBYQpG",
        "outputId": "0af83851-29bd-4d19-b337-4ad5387ff54d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.97      0.94      0.96       912\n",
            "           1       0.93      0.97      0.95       807\n",
            "\n",
            "    accuracy                           0.95      1719\n",
            "   macro avg       0.95      0.96      0.95      1719\n",
            "weighted avg       0.95      0.95      0.95      1719\n",
            "\n",
            "Accuracy of naive based Model::  0.954043048283886\n"
          ]
        }
      ],
      "source": [
        "##classification report of naive bayes model\n",
        "from sklearn.metrics import classification_report,accuracy_score\n",
        "npred=classifier.predict(x_test)\n",
        "print(classification_report(y_test,npred))\n",
        "#print(npred)\n",
        "print('Accuracy of naive based Model:: ',accuracy_score(y_test,npred))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JlN0ySUubAeS"
      },
      "source": [
        "**Logistic Regression**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "C1-cxoq7X2EO",
        "outputId": "adcd713b-eaec-4433-c289-77f3932af824"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "LogisticRegression()"
            ]
          },
          "execution_count": null,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "from sklearn.linear_model import LogisticRegression\n",
        "lr=LogisticRegression()\n",
        "lr.fit(x_train,y_train)\n",
        "#lr.score(x_test,y_test)\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "Ka3yrT5bcwC0",
        "outputId": "3ea082c1-7748-45fb-c040-b8ecd1f6e902"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.98      0.98      0.98       912\n",
            "           1       0.98      0.98      0.98       807\n",
            "\n",
            "    accuracy                           0.98      1719\n",
            "   macro avg       0.98      0.98      0.98      1719\n",
            "weighted avg       0.98      0.98      0.98      1719\n",
            "\n",
            "accuracy of logistic regression model::  0.9802210587550901\n"
          ]
        }
      ],
      "source": [
        "lr_pred=lr.predict(x_test)\n",
        "print(classification_report(y_test,lr_pred))\n",
        "\n",
        "print(\"accuracy of logistic regression model:: \",accuracy_score(y_test,lr_pred))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ucqVeJmFgy9o"
      },
      "source": [
        "**decision tree classification**\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "tWyvMWLFd0xl",
        "outputId": "21eef088-954d-467d-a35d-a104da7a0b4d"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "DecisionTreeClassifier()"
            ]
          },
          "execution_count": null,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "\n",
        "from sklearn.tree import DecisionTreeClassifier\n",
        "dt=DecisionTreeClassifier()\n",
        "dt.fit(x_train,y_train)\n",
        "#dt.score(x_test,y_test)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "ujUjiDqTeWHK",
        "outputId": "2fdb4620-7de7-43e4-9809-39658eb4ac1f"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.97      0.97      0.97       912\n",
            "           1       0.97      0.97      0.97       807\n",
            "\n",
            "    accuracy                           0.97      1719\n",
            "   macro avg       0.97      0.97      0.97      1719\n",
            "weighted avg       0.97      0.97      0.97      1719\n",
            "\n",
            "accuracy of decision tree model::  0.9685863874345549\n"
          ]
        }
      ],
      "source": [
        "\n",
        "\n",
        "dt_pred=dt.predict(x_test)\n",
        "print(classification_report(y_test,dt_pred))\n",
        "\n",
        "print(\"accuracy of decision tree model:: \",accuracy_score(y_test,dt_pred))\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HEIiRwyGh2D6"
      },
      "source": [
        "**Gradient Boosting Classifier**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "YqTazYKbhzzs",
        "outputId": "fc727f83-bcb7-4cc7-db42-d910de0674dc"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "GradientBoostingClassifier(random_state=0)"
            ]
          },
          "execution_count": null,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "\n",
        "from sklearn.ensemble import GradientBoostingClassifier\n",
        "gc=GradientBoostingClassifier(random_state=0)\n",
        "gc.fit(x_train,y_train)\n",
        "#gc.score(x_test,y_test)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "30iJSqf7ijzP",
        "outputId": "42300f4d-2571-4d3c-8115-1ef2ad37bd1a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.99      0.97      0.98       912\n",
            "           1       0.97      0.98      0.98       807\n",
            "\n",
            "    accuracy                           0.98      1719\n",
            "   macro avg       0.98      0.98      0.98      1719\n",
            "weighted avg       0.98      0.98      0.98      1719\n",
            "\n",
            "accuracy of gradient boosting  model::  0.9778941244909831\n"
          ]
        }
      ],
      "source": [
        "\n",
        "gc_pred=gc.predict(x_test)\n",
        "print(classification_report(y_test,gc_pred))\n",
        "\n",
        "print(\"accuracy of gradient boosting  model:: \",accuracy_score(y_test,gc_pred))\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xfblNa5Ykl1E"
      },
      "source": [
        "**RAndom forest classifier**\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "9Yd0FuXtkli-",
        "outputId": "1d577f7f-668b-4897-ad10-f9be0b2885bd"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "RandomForestClassifier()"
            ]
          },
          "execution_count": null,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "from sklearn.ensemble import RandomForestClassifier\n",
        "rfc=RandomForestClassifier()\n",
        "rfc.fit(x_train,y_train)\n",
        "#rfc.score(x_test,y_test)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "LrShPTANld2u",
        "outputId": "de716ae7-f4eb-47f9-a08f-dab8b036f74e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.99      0.97      0.98       912\n",
            "           1       0.97      0.98      0.98       807\n",
            "\n",
            "    accuracy                           0.98      1719\n",
            "   macro avg       0.98      0.98      0.98      1719\n",
            "weighted avg       0.98      0.98      0.98      1719\n",
            "\n",
            "accuracy of Random Forest  model::  0.9778941244909831\n"
          ]
        }
      ],
      "source": [
        "rfc_pred=gc.predict(x_test)\n",
        "print(classification_report(y_test,rfc_pred))\n",
        "\n",
        "print(\"accuracy of Random Forest  model:: \",accuracy_score(y_test,gc_pred))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TQBRugsyq7pE"
      },
      "source": [
        "**Manual Testing Section**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "qN3C-MEZrDR9"
      },
      "outputs": [],
      "source": [
        "true = [0]\n",
        "false = [0]\n",
        "def output_lable(n):\n",
        "    if n == 0:\n",
        "      false[0]+=1\n",
        "      return \"Fake News\"\n",
        "    elif n == 1:\n",
        "      true[0]+=1\n",
        "      return \"It's a Valid News\"\n",
        "    \n",
        "def manual_testing(news):\n",
        "    testing_news = {\"text\":[news]}\n",
        "    new_def_test = pd.DataFrame(testing_news)\n",
        "    new_def_test[\"text\"] = new_def_test[\"text\"].apply(process) \n",
        "    new_x_test = new_def_test[\"text\"]\n",
        "    new_xv_test =vectorizer.transform(new_x_test)\n",
        "    pred_nbc= classifier.predict(new_xv_test)\n",
        "    pred_LR = lr.predict(new_xv_test)\n",
        "    pred_DT = dt.predict(new_xv_test)\n",
        "    pred_GBC = gc.predict(new_xv_test)\n",
        "    pred_RFC = rfc.predict(new_xv_test)\n",
        "    return print(\"\\n\\nNB Prediction: {} \\nLR Prediction: {} \\nDT Prediction: {} \\nGBC Prediction: {} \\nRFC Prediction: {}\".format(output_lable(pred_nbc[0]),output_lable(pred_LR[0]), \n",
        "                                                                                                              output_lable(pred_DT[0]), \n",
        "                                                                                                              output_lable(pred_GBC[0]), \n",
        "                                                                                                              output_lable(pred_RFC[0])))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "eMf9GoJjrYiV"
      },
      "outputs": [],
      "source": [
        "news = (input())\n",
        "news=str(news)\n",
        "manual_testing(news)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DaRcvL9dkGYG"
      },
      "outputs": [],
      "source": [
        "print(true)\n",
        "print(false)\n",
        "if(true[0]>false[0]):\n",
        "  print(\"Approximately It's a  Valid News\")\n",
        "else:\n",
        "  print(\"Approximately It's a  Fake News\")\n",
        "  "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-5ivr2rlq72g"
      },
      "source": []
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyPG+sVuUjVhdhA3AU+OITZ3",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}