Clean_and_Filter.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Clean and Filter.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyN1OGhMuH3KcPVfzodMqHhe"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xjK6tPY1PPDq"
      },
      "source": [
        "#Clean and Filter"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1hW_d2yQkXV1"
      },
      "source": [
        "import pandas as pd\n",
        "data = pd.read_csv(\"/content/all_articles_More.csv\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BV7h1gea9kpg",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f527424d-e0f1-4d58-d944-56245a9c4ccc"
      },
      "source": [
        "import nltk\n",
        "nltk.download('stopwords')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UrmQA6Yw9xya",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 764
        },
        "outputId": "fcababde-0586-4492-dbc5-ef29ad06612f"
      },
      "source": [
        "import string \n",
        "from nltk.corpus import stopwords\n",
        "\n",
        "stop = stopwords.words('english')\n",
        "\n",
        "#convert tweets to lower case\n",
        "data['preprocess_data'] = data['Sent'].str.lower()\n",
        "\n",
        "#url removes\n",
        "data['preprocess_data'] = data['preprocess_data'].str.replace(r'(https|http)?:\\/(\\w|\\.|\\/|\\?|\\=|\\&|\\%)*\\b','')\n",
        "data['preprocess_data'] = data['preprocess_data'].str.replace(r'www\\.\\S+\\.com','')\n",
        "\n",
        "#removes retweets & cc\n",
        "data['preprocess_data'] = data['preprocess_data'].str.replace(r'rt|cc', '')\n",
        "\n",
        "#hashtags removes\n",
        "data['preprocess_data'] = data['preprocess_data'].str.replace(r'#\\S+', '')\n",
        "\n",
        "#user mention removes\n",
        "data['preprocess_data'] = data['preprocess_data'].str.replace(r'@\\S+', '')\n",
        "\n",
        "#emoji \n",
        "data['preprocess_data'] = data['preprocess_data'].str.replace(r'[^\\x00-\\x7F]+', '')\n",
        "\n",
        "#html tags\n",
        "data['preprocess_data'] = data['preprocess_data'].str.replace(r'<.*?>', '')\n",
        "\n",
        "#removes extra spaces\n",
        "data['preprocess_data'] = data['preprocess_data'].str.replace(r' +', ' ')\n",
        "\n",
        "#punctuation\n",
        "data['preprocess_data'] = data['preprocess_data'].str.replace('[{}]'.format(string.punctuation), '')\n",
        "\n",
        "#stop words removes\n",
        "data['preprocess_data'] = data['preprocess_data'].apply(lambda x: ' '.join([word for word in x.split() if word not in (stop)]))\n",
        "\n",
        "\n",
        "#convert preprocessed list words to string \n",
        "data['preprocess_str'] = data['preprocess_data'].apply(' '.join)\n",
        "\n",
        "data"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Sent</th>\n",
              "      <th>Len</th>\n",
              "      <th>label</th>\n",
              "      <th>preprocess_data</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>['Preeclampsia (PE) is a hypertensive disorder of pregnancy that can cause detrimental obstetric outcomes if not managed properly.</td>\n",
              "      <td>17</td>\n",
              "      <td>Coronary artery disease</td>\n",
              "      <td>['preeclampsia (pe) hypertensive disorder pregnancy cause detrimental obstetric outcomes managed properly.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Current evidence demonstrates higher risk for long-term cardiovascular disease in preeclamptic women.</td>\n",
              "      <td>10</td>\n",
              "      <td>Coronary artery disease</td>\n",
              "      <td>current evidence demonstrates higher risk long-term cardiovascular disease preeclamptic women.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Even in uncomplicated pregnancies, the heart work overload often reveals subtle cardiac defects or abnormalities, which otherwise remain undiagnosed in women without a history of pregnancy.</td>\n",
              "      <td>24</td>\n",
              "      <td>Coronary artery disease</td>\n",
              "      <td>even uncomplicated pregnancies, heart work overload often reveals subtle cardiac defects abnormalities, otherwise remain undiagnosed women without history pregnancy.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Pathophysiologic patterns occurring in PE patients resemble biochemical responses observed in cases of cardiovascular disease.</td>\n",
              "      <td>13</td>\n",
              "      <td>Coronary artery disease</td>\n",
              "      <td>pathophysiologic patterns occurring pe patients resemble biochemical responses observed cases cardiovascular disease.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>It has been estimated that women with an obstetric history of PE are more likely to develop coronary artery disease in the long run.</td>\n",
              "      <td>23</td>\n",
              "      <td>Coronary artery disease</td>\n",
              "      <td>estimated women obstetric history pe likely develop coronary artery disease long run.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17620</th>\n",
              "      <td>['OBJECTIVES: Pizza eating has been favourably related to the risk of cardiovascular disease, but the data are limited.</td>\n",
              "      <td>17</td>\n",
              "      <td>unsaturated fats disease</td>\n",
              "      <td>['objectives: pizza eating favourably related risk cardiovascular disease, data limited.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17621</th>\n",
              "      <td>To evaluate the potential role  of pizza consumption on the risk of acute myocardial infarction (AMI), we considered data from an Italian study. DESIGN: We conducted a hospital-based case-control study on 507 cases of nonfatal AMI and 478 controls in Milan, Italy, between 1995 and 1999. RESULTS: The multivariate odds ratios were 0.78 for occasional, 0.62 for regular  and 0.44 for frequent eaters.</td>\n",
              "      <td>59</td>\n",
              "      <td>unsaturated fats disease</td>\n",
              "      <td>evaluate potential role pizza consumption risk acute myocardial infarction (ami), considered data italian study. design: conducted hospital-based case-control study 507 cases nonfatal ami 478 controls milan, italy, 1995 1999. results: multivariate odds ratios 0.78 occasional, 0.62 regular 0.44 frequent eaters.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17622</th>\n",
              "      <td>The estimates were similar across strata of age, sex, smoking and other major covariates. CONCLUSIONS: Some of the ingredients of pizza have been shown to have a favourable influence on the risk of cardiovascular disease.</td>\n",
              "      <td>32</td>\n",
              "      <td>unsaturated fats disease</td>\n",
              "      <td>estimates similar across strata age, sex, smoking major covariates. conclusions: ingredients pizza shown favourable influence risk cardiovascular disease.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17623</th>\n",
              "      <td>However, there is no  single explanation for the present findings.</td>\n",
              "      <td>10</td>\n",
              "      <td>unsaturated fats disease</td>\n",
              "      <td>however, single explanation present findings.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17624</th>\n",
              "      <td>Pizza may in fact represent a general indicator of Italian diet, that has been shown to have potential cardiovascular benefits.']</td>\n",
              "      <td>18</td>\n",
              "      <td>unsaturated fats disease</td>\n",
              "      <td>pizza may fact represent general indicator italian diet, shown potential cardiovascular benefits.']</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>17625 rows × 4 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                                                                                                                                                                                                                                                                                                                                                                                  Sent  ...                                                                                                                                                                                                                                                                                                          preprocess_data\n",
              "0                                                                                                                                                                                                                                                                                   ['Preeclampsia (PE) is a hypertensive disorder of pregnancy that can cause detrimental obstetric outcomes if not managed properly.  ...                                                                                                                                                                                                               ['preeclampsia (pe) hypertensive disorder pregnancy cause detrimental obstetric outcomes managed properly.\n",
              "1                                                                                                                                                                                                                                                                                                                Current evidence demonstrates higher risk for long-term cardiovascular disease in preeclamptic women.  ...                                                                                                                                                                                                                           current evidence demonstrates higher risk long-term cardiovascular disease preeclamptic women.\n",
              "2                                                                                                                                                                                                                        Even in uncomplicated pregnancies, the heart work overload often reveals subtle cardiac defects or abnormalities, which otherwise remain undiagnosed in women without a history of pregnancy.  ...                                                                                                                                                    even uncomplicated pregnancies, heart work overload often reveals subtle cardiac defects abnormalities, otherwise remain undiagnosed women without history pregnancy.\n",
              "3                                                                                                                                                                                                                                                                                       Pathophysiologic patterns occurring in PE patients resemble biochemical responses observed in cases of cardiovascular disease.  ...                                                                                                                                                                                                    pathophysiologic patterns occurring pe patients resemble biochemical responses observed cases cardiovascular disease.\n",
              "4                                                                                                                                                                                                                                                                                 It has been estimated that women with an obstetric history of PE are more likely to develop coronary artery disease in the long run.  ...                                                                                                                                                                                                                                    estimated women obstetric history pe likely develop coronary artery disease long run.\n",
              "...                                                                                                                                                                                                                                                                                                                                                                                                                ...  ...                                                                                                                                                                                                                                                                                                                      ...\n",
              "17620                                                                                                                                                                                                                                                                                          ['OBJECTIVES: Pizza eating has been favourably related to the risk of cardiovascular disease, but the data are limited.  ...                                                                                                                                                                                                                                 ['objectives: pizza eating favourably related risk cardiovascular disease, data limited.\n",
              "17621  To evaluate the potential role  of pizza consumption on the risk of acute myocardial infarction (AMI), we considered data from an Italian study. DESIGN: We conducted a hospital-based case-control study on 507 cases of nonfatal AMI and 478 controls in Milan, Italy, between 1995 and 1999. RESULTS: The multivariate odds ratios were 0.78 for occasional, 0.62 for regular  and 0.44 for frequent eaters.  ...  evaluate potential role pizza consumption risk acute myocardial infarction (ami), considered data italian study. design: conducted hospital-based case-control study 507 cases nonfatal ami 478 controls milan, italy, 1995 1999. results: multivariate odds ratios 0.78 occasional, 0.62 regular 0.44 frequent eaters.\n",
              "17622                                                                                                                                                                                    The estimates were similar across strata of age, sex, smoking and other major covariates. CONCLUSIONS: Some of the ingredients of pizza have been shown to have a favourable influence on the risk of cardiovascular disease.  ...                                                                                                                                                               estimates similar across strata age, sex, smoking major covariates. conclusions: ingredients pizza shown favourable influence risk cardiovascular disease.\n",
              "17623                                                                                                                                                                                                                                                                                                                                               However, there is no  single explanation for the present findings.  ...                                                                                                                                                                                                                                                                            however, single explanation present findings.\n",
              "17624                                                                                                                                                                                                                                                                                Pizza may in fact represent a general indicator of Italian diet, that has been shown to have potential cardiovascular benefits.']  ...                                                                                                                                                                                                                      pizza may fact represent general indicator italian diet, shown potential cardiovascular benefits.']\n",
              "\n",
              "[17625 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gJNLOMxSfgGY"
      },
      "source": [
        "#https://shubham-agnihotri.medium.com/sentimental-analysis-using-rule-based-approach-text-blob-over-twitter-data-part-i-391edb24aa58\n",
        "from textblob import TextBlob\n",
        "from tweepy import OAuthHandler, Stream, Cursor, API\n",
        "from tweepy.streaming import StreamListener\n",
        "import tweepy\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import json\n",
        "import time\n",
        "import re\n",
        "import os.path\n",
        "\n",
        "class Analyze_Tweet():\n",
        "\n",
        "    def Analyze_Sentiment(self,tweet):\n",
        "        try:\n",
        "            analysis_result = TextBlob(tweet)\n",
        "            if analysis_result.sentiment.polarity > 0:\n",
        "                return \"Positive\"\n",
        "            elif analysis_result.sentiment.polarity == 0:\n",
        "                return \"Neutral\"\n",
        "            else:\n",
        "                return \"Negative\"\n",
        "                \n",
        "        except Exception as e:\n",
        "            print(\"Exception as Analyze_Tweet Class on function Analyze_Sentiment: \",e)\n",
        "            return \"Nil\"\n",
        "\n",
        "    def Analyze_Sentiment_value(self,tweet):\n",
        "        try:\n",
        "            analysis_result = TextBlob(tweet)\n",
        "            return analysis_result.sentiment.polarity \n",
        "               \n",
        "        except Exception as e:\n",
        "            print(\"Exception as Analyze_Tweet Class on function Analyze_Sentiment: \",e)\n",
        "            return \"Nil\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Kf50qA2lcn7y"
      },
      "source": [
        "class Analyze_Tweet2():\n",
        "\n",
        "    def Analyze_subjectivity(self,tweet):\n",
        "        try:\n",
        "            analysis_result = TextBlob(tweet)\n",
        "            if analysis_result.sentiment.subjectivity > 0.5:\n",
        "                return \"Subjective\"\n",
        "            elif analysis_result.sentiment.subjectivity == 0.5:\n",
        "                return \"Neutral\"\n",
        "            else:\n",
        "                return \"Objective\"\n",
        "                \n",
        "        except Exception as e:\n",
        "            print(\"Exception as Analyze_Tweet2 Class on function Analyze_subjectivity: \",e)\n",
        "            return \"Nil\"\n",
        "\n",
        "    def Analyze_subjectivity_value(self,tweet):\n",
        "        try:\n",
        "            analysis_result = TextBlob(tweet)\n",
        "            return analysis_result.sentiment.subjectivity\n",
        "               \n",
        "        except Exception as e:\n",
        "            print(\"Exception as Analyze_Tweet Class on function Analyze_Sentiment: \",e)\n",
        "            return \"Nil\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Xj2o6b8Ng8Sl",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c8fe6d59-5ce5-49e1-c491-c45fa6ea13b2"
      },
      "source": [
        "a = Analyze_Tweet()\n",
        "data['sentiments'] = np.array([a.Analyze_Sentiment(row[3]) for index,row in data.iterrows()])\n",
        "print(data.columns)\n",
        "df2 = data.filter(['tweet','sentiments'],axis = 1)\n",
        "print(df2)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Index(['Sent', 'Len', 'label', 'preprocess_data', 'sentiments'], dtype='object')\n",
            "      sentiments\n",
            "0        Neutral\n",
            "1       Positive\n",
            "2       Negative\n",
            "3        Neutral\n",
            "4       Negative\n",
            "...          ...\n",
            "17620   Negative\n",
            "17621   Positive\n",
            "17622   Positive\n",
            "17623   Negative\n",
            "17624   Positive\n",
            "\n",
            "[17625 rows x 1 columns]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ijVwyPH8nfrd",
        "outputId": "c672e4bf-eddd-44e8-bbb9-ad281894898c"
      },
      "source": [
        "a = Analyze_Tweet()\n",
        "data['sentiments_val'] = np.array([a.Analyze_Sentiment_value(row[0]) for index,row in data.iterrows()])\n",
        "print(data.columns)\n",
        "df2 = data.filter(['tweet','sentiments_val'],axis = 1)\n",
        "print(df2)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Index(['Sent', 'Len', 'label', 'preprocess_data', 'sentiments',\n",
            "       'sentiments_val'],\n",
            "      dtype='object')\n",
            "       sentiments_val\n",
            "0            0.000000\n",
            "1            0.125000\n",
            "2           -0.191667\n",
            "3            0.000000\n",
            "4            0.150000\n",
            "...               ...\n",
            "17620       -0.035714\n",
            "17621        0.116667\n",
            "17622       -0.020833\n",
            "17623        0.017857\n",
            "17624        0.016667\n",
            "\n",
            "[17625 rows x 1 columns]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ik3siWZe_AiU",
        "outputId": "a039618e-15e2-4813-a0e1-c1f5c822013d"
      },
      "source": [
        "data['sentiments'].value_counts()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Positive    8289\n",
              "Neutral     5455\n",
              "Negative    3881\n",
              "Name: sentiments, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "59D3Ax-kdQ3i",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8b48dc72-63e4-4965-84a2-b83b91a2181f"
      },
      "source": [
        "a = Analyze_Tweet2()\n",
        "data['subjective'] = np.array([a.Analyze_subjectivity(row[3]) for index,row in data.iterrows()])\n",
        "print(data.columns)\n",
        "df2 = data.filter(['tweet','subjective'],axis = 1)\n",
        "print(df2)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Index(['Sent', 'Len', 'label', 'preprocess_data', 'sentiments',\n",
            "       'sentiments_val', 'subjective'],\n",
            "      dtype='object')\n",
            "       subjective\n",
            "0       Objective\n",
            "1       Objective\n",
            "2       Objective\n",
            "3       Objective\n",
            "4      Subjective\n",
            "...           ...\n",
            "17620   Objective\n",
            "17621   Objective\n",
            "17622   Objective\n",
            "17623   Objective\n",
            "17624     Neutral\n",
            "\n",
            "[17625 rows x 1 columns]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kXc6Npzxnqcn",
        "outputId": "79e89e98-67f7-4ff7-ee08-56cddbdfdc61"
      },
      "source": [
        "a = Analyze_Tweet2()\n",
        "data['subjective_val'] = np.array([a.Analyze_subjectivity_value(row[0]) for index,row in data.iterrows()])\n",
        "print(data.columns)\n",
        "df2 = data.filter(['tweet','subjective_val'],axis = 1)\n",
        "print(df2)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Index(['Sent', 'Len', 'label', 'preprocess_data', 'sentiments',\n",
            "       'sentiments_val', 'subjective', 'subjective_val'],\n",
            "      dtype='object')\n",
            "       subjective_val\n",
            "0            0.100000\n",
            "1            0.450000\n",
            "2            0.250000\n",
            "3            0.000000\n",
            "4            0.633333\n",
            "...               ...\n",
            "17620        0.271429\n",
            "17621        0.400321\n",
            "17622        0.425000\n",
            "17623        0.107143\n",
            "17624        0.500000\n",
            "\n",
            "[17625 rows x 1 columns]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "74b7U35Y_PDA",
        "outputId": "7f5572fb-c85e-4311-875d-654d45e1ba04"
      },
      "source": [
        "data['subjective'].value_counts()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Objective     12010\n",
              "Subjective     4582\n",
              "Neutral        1033\n",
              "Name: subjective, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 42
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yibiWDYotyHx"
      },
      "source": [
        "**Vader**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-3gcqDHpt_d6"
      },
      "source": [
        "pip install vaderSentiment"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SxGgIzsItv-0"
      },
      "source": [
        "# import SentimentIntensityAnalyzer class\n",
        "# from vaderSentiment.vaderSentiment module.\n",
        "from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer\n",
        "\n",
        "# function to print sentiments\n",
        "# of the sentence.\n",
        "def sentiment_scores(sentence):\n",
        "\n",
        "\t# Create a SentimentIntensityAnalyzer object.\n",
        "\tsid_obj = SentimentIntensityAnalyzer()\n",
        "\n",
        "\n",
        "\tsentiment_dict = sid_obj.polarity_scores(sentence)\n",
        "\t\n",
        "\t# decide sentiment as positive, negative and neutral\n",
        "\tif sentiment_dict['compound'] >= 0.05 :\n",
        "\t\treturn \"Positive\"#,sentiment_dict['compound'] *100\n",
        "\n",
        "\telif sentiment_dict['compound'] <= - 0.05 :\n",
        "\t\treturn \"Negative\"#,sentiment_dict['compound'] *100\n",
        "\n",
        "\telse :\n",
        "\t\treturn \"Neutral\"#,sentiment_dict['compound'] *100\n",
        "\n",
        "\n",
        "def sentiment_scores2(sentence):\n",
        "\n",
        "\t# Create a SentimentIntensityAnalyzer object.\n",
        "\tsid_obj = SentimentIntensityAnalyzer()\n",
        "\n",
        "\t# which contains pos, neg, neu, and compound scores.\n",
        "\tsentiment_dict = sid_obj.polarity_scores(sentence)\n",
        "\n",
        "\treturn sentiment_dict['compound']#,sentiment_dict['compound'] *100\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ubuo-939uWW3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "f532d77c-be12-4052-e4e7-9d446801f75b"
      },
      "source": [
        "\n",
        "data['vader_sentiments']= np.array([sentiment_scores(row[0]) for index,row in data.iterrows()])\n",
        "\n",
        "data['vader_sentiments_val']= np.array([sentiment_scores2(row[0]) for index,row in data.iterrows()])\n",
        "\n",
        "data\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Sent</th>\n",
              "      <th>Len</th>\n",
              "      <th>label</th>\n",
              "      <th>preprocess_data</th>\n",
              "      <th>sentiments</th>\n",
              "      <th>sentiments_val</th>\n",
              "      <th>subjective</th>\n",
              "      <th>subjective_val</th>\n",
              "      <th>vader_sentiments</th>\n",
              "      <th>vader_sentiments_val</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>['Preeclampsia (PE) is a hypertensive disorder of pregnancy that can cause detrimental obstetric outcomes if not managed properly.</td>\n",
              "      <td>17</td>\n",
              "      <td>Coronary artery disease</td>\n",
              "      <td>['preeclampsia (pe) hypertensive disorder pregnancy cause detrimental obstetric outcomes managed properly.</td>\n",
              "      <td>Neutral</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>Objective</td>\n",
              "      <td>0.100000</td>\n",
              "      <td>Negative</td>\n",
              "      <td>-0.4019</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Current evidence demonstrates higher risk for long-term cardiovascular disease in preeclamptic women.</td>\n",
              "      <td>10</td>\n",
              "      <td>Coronary artery disease</td>\n",
              "      <td>current evidence demonstrates higher risk long-term cardiovascular disease preeclamptic women.</td>\n",
              "      <td>Positive</td>\n",
              "      <td>0.125000</td>\n",
              "      <td>Objective</td>\n",
              "      <td>0.450000</td>\n",
              "      <td>Negative</td>\n",
              "      <td>-0.2732</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Even in uncomplicated pregnancies, the heart work overload often reveals subtle cardiac defects or abnormalities, which otherwise remain undiagnosed in women without a history of pregnancy.</td>\n",
              "      <td>24</td>\n",
              "      <td>Coronary artery disease</td>\n",
              "      <td>even uncomplicated pregnancies, heart work overload often reveals subtle cardiac defects abnormalities, otherwise remain undiagnosed women without history pregnancy.</td>\n",
              "      <td>Negative</td>\n",
              "      <td>-0.191667</td>\n",
              "      <td>Objective</td>\n",
              "      <td>0.250000</td>\n",
              "      <td>Neutral</td>\n",
              "      <td>0.0000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Pathophysiologic patterns occurring in PE patients resemble biochemical responses observed in cases of cardiovascular disease.</td>\n",
              "      <td>13</td>\n",
              "      <td>Coronary artery disease</td>\n",
              "      <td>pathophysiologic patterns occurring pe patients resemble biochemical responses observed cases cardiovascular disease.</td>\n",
              "      <td>Neutral</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>Objective</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>Neutral</td>\n",
              "      <td>0.0000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>It has been estimated that women with an obstetric history of PE are more likely to develop coronary artery disease in the long run.</td>\n",
              "      <td>23</td>\n",
              "      <td>Coronary artery disease</td>\n",
              "      <td>estimated women obstetric history pe likely develop coronary artery disease long run.</td>\n",
              "      <td>Negative</td>\n",
              "      <td>0.150000</td>\n",
              "      <td>Subjective</td>\n",
              "      <td>0.633333</td>\n",
              "      <td>Neutral</td>\n",
              "      <td>0.0000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17620</th>\n",
              "      <td>['OBJECTIVES: Pizza eating has been favourably related to the risk of cardiovascular disease, but the data are limited.</td>\n",
              "      <td>17</td>\n",
              "      <td>unsaturated fats disease</td>\n",
              "      <td>['objectives: pizza eating favourably related risk cardiovascular disease, data limited.</td>\n",
              "      <td>Negative</td>\n",
              "      <td>-0.035714</td>\n",
              "      <td>Objective</td>\n",
              "      <td>0.271429</td>\n",
              "      <td>Negative</td>\n",
              "      <td>-0.4404</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17621</th>\n",
              "      <td>To evaluate the potential role  of pizza consumption on the risk of acute myocardial infarction (AMI), we considered data from an Italian study. DESIGN: We conducted a hospital-based case-control study on 507 cases of nonfatal AMI and 478 controls in Milan, Italy, between 1995 and 1999. RESULTS: The multivariate odds ratios were 0.78 for occasional, 0.62 for regular  and 0.44 for frequent eaters.</td>\n",
              "      <td>59</td>\n",
              "      <td>unsaturated fats disease</td>\n",
              "      <td>evaluate potential role pizza consumption risk acute myocardial infarction (ami), considered data italian study. design: conducted hospital-based case-control study 507 cases nonfatal ami 478 controls milan, italy, 1995 1999. results: multivariate odds ratios 0.78 occasional, 0.62 regular 0.44 frequent eaters.</td>\n",
              "      <td>Positive</td>\n",
              "      <td>0.116667</td>\n",
              "      <td>Objective</td>\n",
              "      <td>0.400321</td>\n",
              "      <td>Negative</td>\n",
              "      <td>-0.2732</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17622</th>\n",
              "      <td>The estimates were similar across strata of age, sex, smoking and other major covariates. CONCLUSIONS: Some of the ingredients of pizza have been shown to have a favourable influence on the risk of cardiovascular disease.</td>\n",
              "      <td>32</td>\n",
              "      <td>unsaturated fats disease</td>\n",
              "      <td>estimates similar across strata age, sex, smoking major covariates. conclusions: ingredients pizza shown favourable influence risk cardiovascular disease.</td>\n",
              "      <td>Positive</td>\n",
              "      <td>-0.020833</td>\n",
              "      <td>Objective</td>\n",
              "      <td>0.425000</td>\n",
              "      <td>Negative</td>\n",
              "      <td>-0.2732</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17623</th>\n",
              "      <td>However, there is no  single explanation for the present findings.</td>\n",
              "      <td>10</td>\n",
              "      <td>unsaturated fats disease</td>\n",
              "      <td>however, single explanation present findings.</td>\n",
              "      <td>Negative</td>\n",
              "      <td>0.017857</td>\n",
              "      <td>Objective</td>\n",
              "      <td>0.107143</td>\n",
              "      <td>Negative</td>\n",
              "      <td>-0.2960</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17624</th>\n",
              "      <td>Pizza may in fact represent a general indicator of Italian diet, that has been shown to have potential cardiovascular benefits.']</td>\n",
              "      <td>18</td>\n",
              "      <td>unsaturated fats disease</td>\n",
              "      <td>pizza may fact represent general indicator italian diet, shown potential cardiovascular benefits.']</td>\n",
              "      <td>Positive</td>\n",
              "      <td>0.016667</td>\n",
              "      <td>Neutral</td>\n",
              "      <td>0.500000</td>\n",
              "      <td>Positive</td>\n",
              "      <td>0.3818</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>17625 rows × 10 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                                                                                                                                                                                                                                                                                                                                                                                  Sent  ...  vader_sentiments_val\n",
              "0                                                                                                                                                                                                                                                                                   ['Preeclampsia (PE) is a hypertensive disorder of pregnancy that can cause detrimental obstetric outcomes if not managed properly.  ...               -0.4019\n",
              "1                                                                                                                                                                                                                                                                                                                Current evidence demonstrates higher risk for long-term cardiovascular disease in preeclamptic women.  ...               -0.2732\n",
              "2                                                                                                                                                                                                                        Even in uncomplicated pregnancies, the heart work overload often reveals subtle cardiac defects or abnormalities, which otherwise remain undiagnosed in women without a history of pregnancy.  ...                0.0000\n",
              "3                                                                                                                                                                                                                                                                                       Pathophysiologic patterns occurring in PE patients resemble biochemical responses observed in cases of cardiovascular disease.  ...                0.0000\n",
              "4                                                                                                                                                                                                                                                                                 It has been estimated that women with an obstetric history of PE are more likely to develop coronary artery disease in the long run.  ...                0.0000\n",
              "...                                                                                                                                                                                                                                                                                                                                                                                                                ...  ...                   ...\n",
              "17620                                                                                                                                                                                                                                                                                          ['OBJECTIVES: Pizza eating has been favourably related to the risk of cardiovascular disease, but the data are limited.  ...               -0.4404\n",
              "17621  To evaluate the potential role  of pizza consumption on the risk of acute myocardial infarction (AMI), we considered data from an Italian study. DESIGN: We conducted a hospital-based case-control study on 507 cases of nonfatal AMI and 478 controls in Milan, Italy, between 1995 and 1999. RESULTS: The multivariate odds ratios were 0.78 for occasional, 0.62 for regular  and 0.44 for frequent eaters.  ...               -0.2732\n",
              "17622                                                                                                                                                                                    The estimates were similar across strata of age, sex, smoking and other major covariates. CONCLUSIONS: Some of the ingredients of pizza have been shown to have a favourable influence on the risk of cardiovascular disease.  ...               -0.2732\n",
              "17623                                                                                                                                                                                                                                                                                                                                               However, there is no  single explanation for the present findings.  ...               -0.2960\n",
              "17624                                                                                                                                                                                                                                                                                Pizza may in fact represent a general indicator of Italian diet, that has been shown to have potential cardiovascular benefits.']  ...                0.3818\n",
              "\n",
              "[17625 rows x 10 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 46
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XrFfSYOhkVgO",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3b90da40-367a-4bdc-d3a5-b4eee465004d"
      },
      "source": [
        "data['vader_sentiments'].value_counts()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Positive    8213\n",
              "Negative    5217\n",
              "Neutral     4195\n",
              "Name: vader_sentiments, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 48
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Zm8AvOUQMlOj"
      },
      "source": [
        "##Flair Context Filtration"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GWDRHMe3M1tQ"
      },
      "source": [
        "pip install flair"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "v_YOqBzkqRvb"
      },
      "source": [
        "**Zero Shot Predictions**\n",
        "\n",
        "using TARS to get if statement about disease or not\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nU5fj2T1nJ_C"
      },
      "source": [
        "from flair.models.text_classification_model import TARSClassifier\n",
        "from flair.data import Sentence\n",
        "\n",
        "# 1. Load our pre-trained TARS model for English\n",
        "tars = TARSClassifier.load('tars-base')\n",
        "\n",
        "classes = ['drug','symptoms','heart disease','disease','treatment']\n",
        "flair_tars=[]\n",
        "flair_tars_len=[]\n",
        "\n",
        "for i in range(len(data['Sent'])):\n",
        "    sentence = Sentence(data['Sent'][i])\n",
        "    tars.predict_zero_shot(sentence, classes)\n",
        "    print(i)\n",
        "    flair_tars.append([sent for sent in sentence.labels])\n",
        "    flair_tars_len.append(len([sent for sent in sentence.labels]))\n",
        "\n",
        "data['flair_tars']=flair_tars\n",
        "data['flair_tars_len']=flair_tars_len\n",
        "data['flair_tars']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 731
        },
        "id": "FcPLDIP2qkt4",
        "outputId": "3323fc13-663c-4b48-896a-eedcf2424aa4"
      },
      "source": [
        "pd.set_option('display.max_colwidth', None)\n",
        "data[['flair_tars','Sent','label']]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>flair_tars</th>\n",
              "      <th>Sent</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>[heart disease (0.6824), treatment (0.6469), symptoms (0.8362), disease (0.9684)]</td>\n",
              "      <td>['Preeclampsia (PE) is a hypertensive disorder of pregnancy that can cause detrimental obstetric outcomes if not managed properly.</td>\n",
              "      <td>Coronary artery disease</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>[heart disease (0.9719), disease (0.9598)]</td>\n",
              "      <td>Current evidence demonstrates higher risk for long-term cardiovascular disease in preeclamptic women.</td>\n",
              "      <td>Coronary artery disease</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>[heart disease (0.9775), symptoms (0.896), disease (0.9628)]</td>\n",
              "      <td>Even in uncomplicated pregnancies, the heart work overload often reveals subtle cardiac defects or abnormalities, which otherwise remain undiagnosed in women without a history of pregnancy.</td>\n",
              "      <td>Coronary artery disease</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>[heart disease (0.965), treatment (0.9188), symptoms (0.7273), disease (0.9858), drug (0.8282)]</td>\n",
              "      <td>Pathophysiologic patterns occurring in PE patients resemble biochemical responses observed in cases of cardiovascular disease.</td>\n",
              "      <td>Coronary artery disease</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>[heart disease (0.9899), treatment (0.694), symptoms (0.9561), disease (0.9822)]</td>\n",
              "      <td>It has been estimated that women with an obstetric history of PE are more likely to develop coronary artery disease in the long run.</td>\n",
              "      <td>Coronary artery disease</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17620</th>\n",
              "      <td>[heart disease (0.9741), treatment (0.951), symptoms (0.9365), disease (0.991), drug (0.916)]</td>\n",
              "      <td>['OBJECTIVES: Pizza eating has been favourably related to the risk of cardiovascular disease, but the data are limited.</td>\n",
              "      <td>unsaturated fats disease</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17621</th>\n",
              "      <td>[heart disease (0.9446), disease (0.9951), drug (0.8412)]</td>\n",
              "      <td>To evaluate the potential role  of pizza consumption on the risk of acute myocardial infarction (AMI), we considered data from an Italian study. DESIGN: We conducted a hospital-based case-control study on 507 cases of nonfatal AMI and 478 controls in Milan, Italy, between 1995 and 1999. RESULTS: The multivariate odds ratios were 0.78 for occasional, 0.62 for regular  and 0.44 for frequent eaters.</td>\n",
              "      <td>unsaturated fats disease</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17622</th>\n",
              "      <td>[]</td>\n",
              "      <td>The estimates were similar across strata of age, sex, smoking and other major covariates. CONCLUSIONS: Some of the ingredients of pizza have been shown to have a favourable influence on the risk of cardiovascular disease.</td>\n",
              "      <td>unsaturated fats disease</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17623</th>\n",
              "      <td>[]</td>\n",
              "      <td>However, there is no  single explanation for the present findings.</td>\n",
              "      <td>unsaturated fats disease</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17624</th>\n",
              "      <td>[heart disease (0.7919)]</td>\n",
              "      <td>Pizza may in fact represent a general indicator of Italian diet, that has been shown to have potential cardiovascular benefits.']</td>\n",
              "      <td>unsaturated fats disease</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>17625 rows × 3 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                                                            flair_tars  ...                     label\n",
              "0                    [heart disease (0.6824), treatment (0.6469), symptoms (0.8362), disease (0.9684)]  ...   Coronary artery disease\n",
              "1                                                           [heart disease (0.9719), disease (0.9598)]  ...   Coronary artery disease\n",
              "2                                         [heart disease (0.9775), symptoms (0.896), disease (0.9628)]  ...   Coronary artery disease\n",
              "3      [heart disease (0.965), treatment (0.9188), symptoms (0.7273), disease (0.9858), drug (0.8282)]  ...   Coronary artery disease\n",
              "4                     [heart disease (0.9899), treatment (0.694), symptoms (0.9561), disease (0.9822)]  ...   Coronary artery disease\n",
              "...                                                                                                ...  ...                       ...\n",
              "17620    [heart disease (0.9741), treatment (0.951), symptoms (0.9365), disease (0.991), drug (0.916)]  ...  unsaturated fats disease\n",
              "17621                                        [heart disease (0.9446), disease (0.9951), drug (0.8412)]  ...  unsaturated fats disease\n",
              "17622                                                                                               []  ...  unsaturated fats disease\n",
              "17623                                                                                               []  ...  unsaturated fats disease\n",
              "17624                                                                         [heart disease (0.7919)]  ...  unsaturated fats disease\n",
              "\n",
              "[17625 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A-YqTYgp2n70"
      },
      "source": [
        "data[data['flair_tars_len']==0]"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}