Emotional_Analysis.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Copy of Pill-Recommendation.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "3d3cb6730c5d4056bb6dc7311c6c54fd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_3b87966b87cd4cf28ba3d8b27e61db0f",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_3ba53b52c9944089ba16e23aeb747f36",
              "IPY_MODEL_bc60033e665f4b8fae5f2afe04f3dc3b"
            ]
          }
        },
        "3b87966b87cd4cf28ba3d8b27e61db0f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3ba53b52c9944089ba16e23aeb747f36": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_11b4e4c9151340f6b03746e0c1db6faa",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 156432,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 156432,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8a2a40681b8d495db88a4df7c6dfe400"
          }
        },
        "bc60033e665f4b8fae5f2afe04f3dc3b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_642a584be5194fae9be0761c5011ad3b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 156432/156432 [4:52:54&lt;00:00,  8.90it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ff1e315a5a6041cbbbf0ad6a838e8dd8"
          }
        },
        "11b4e4c9151340f6b03746e0c1db6faa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8a2a40681b8d495db88a4df7c6dfe400": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "642a584be5194fae9be0761c5011ad3b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ff1e315a5a6041cbbbf0ad6a838e8dd8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/harsh-ux/PRML-project/blob/main/Emotional_Analysis.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SwrqSdYmivmg"
      },
      "source": [
        "#Installing Libraries and Dependencies"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UQIyDqj0J9ql",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a544a1e9-1f03-4271-95e5-c84107d183f0"
      },
      "source": [
        "import time\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import seaborn as sns\n",
        "from matplotlib import pyplot as plt\n",
        "from sklearn.svm import SVC\n",
        "from sklearn.model_selection import train_test_split, StratifiedKFold, cross_val_score\n",
        "from sklearn.pipeline import make_pipeline, Pipeline\n",
        "from sklearn.metrics import classification_report\n",
        "from sklearn.model_selection import GridSearchCV\n",
        "from sklearn.metrics import accuracy_score, f1_score, roc_curve, auc, confusion_matrix, roc_auc_score, recall_score, precision_score\n",
        "\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "from sklearn.naive_bayes import MultinomialNB\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from wordcloud import WordCloud, STOPWORDS\n",
        "from nltk import word_tokenize\n",
        "from nltk.stem.snowball import SnowballStemmer\n",
        "from tqdm import tqdm_notebook as tqdm\n",
        "from tqdm import trange\n",
        "from sklearn.preprocessing import Normalizer\n",
        "import nltk\n",
        "nltk.download('punkt')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pNn52moROsPj",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a956ad9a-5ed2-4ef1-c514-6a55303d0dce"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive',force_remount=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uQdeIbnki2s6"
      },
      "source": [
        "#Dataset Description"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HT8I0ceWXwjt",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "277e2dfa-3a23-4bbc-ae7b-1050ec5b7551"
      },
      "source": [
        "# train = pd.read_csv('drive/My Drive/drugsComTrain_raw.csv')\n",
        "# test = pd.read_csv('drive/My Drive/drugsComTest_raw.csv')\n",
        "train = pd.read_csv('drive/My Drive/drugsCom_raw/drugsComTrain_raw.csv')\n",
        "test = pd.read_csv('drive/My Drive/drugsCom_raw/drugsComTest_raw.csv')\n",
        "# getting the shapes\n",
        "print(\"Shape of train :\", train.shape)\n",
        "print(\"Shape of test :\", test.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Shape of train : (161297, 7)\n",
            "Shape of test : (53766, 7)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H-KXObLyYE9q",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 195
        },
        "outputId": "2d7eac72-2711-4e25-8b56-60f78e714e28"
      },
      "source": [
        "train.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>uniqueID</th>\n",
              "      <th>drugName</th>\n",
              "      <th>condition</th>\n",
              "      <th>review</th>\n",
              "      <th>rating</th>\n",
              "      <th>date</th>\n",
              "      <th>usefulCount</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>206461</td>\n",
              "      <td>Valsartan</td>\n",
              "      <td>Left Ventricular Dysfunction</td>\n",
              "      <td>\"It has no side effect, I take it in combinati...</td>\n",
              "      <td>9</td>\n",
              "      <td>20-May-12</td>\n",
              "      <td>27</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>95260</td>\n",
              "      <td>Guanfacine</td>\n",
              "      <td>ADHD</td>\n",
              "      <td>\"My son is halfway through his fourth week of ...</td>\n",
              "      <td>8</td>\n",
              "      <td>27-Apr-10</td>\n",
              "      <td>192</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>92703</td>\n",
              "      <td>Lybrel</td>\n",
              "      <td>Birth Control</td>\n",
              "      <td>\"I used to take another oral contraceptive, wh...</td>\n",
              "      <td>5</td>\n",
              "      <td>14-Dec-09</td>\n",
              "      <td>17</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>138000</td>\n",
              "      <td>Ortho Evra</td>\n",
              "      <td>Birth Control</td>\n",
              "      <td>\"This is my first time using any form of birth...</td>\n",
              "      <td>8</td>\n",
              "      <td>3-Nov-15</td>\n",
              "      <td>10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>35696</td>\n",
              "      <td>Buprenorphine / naloxone</td>\n",
              "      <td>Opiate Dependence</td>\n",
              "      <td>\"Suboxone has completely turned my life around...</td>\n",
              "      <td>9</td>\n",
              "      <td>27-Nov-16</td>\n",
              "      <td>37</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   uniqueID                  drugName  ...       date usefulCount\n",
              "0    206461                 Valsartan  ...  20-May-12          27\n",
              "1     95260                Guanfacine  ...  27-Apr-10         192\n",
              "2     92703                    Lybrel  ...  14-Dec-09          17\n",
              "3    138000                Ortho Evra  ...   3-Nov-15          10\n",
              "4     35696  Buprenorphine / naloxone  ...  27-Nov-16          37\n",
              "\n",
              "[5 rows x 7 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 195
        },
        "id": "DKpAg1evaL5T",
        "outputId": "4c11bf69-c825-4d37-e1d0-7ba9b613625a"
      },
      "source": [
        "test.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>uniqueID</th>\n",
              "      <th>drugName</th>\n",
              "      <th>condition</th>\n",
              "      <th>review</th>\n",
              "      <th>rating</th>\n",
              "      <th>date</th>\n",
              "      <th>usefulCount</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>163740</td>\n",
              "      <td>Mirtazapine</td>\n",
              "      <td>Depression</td>\n",
              "      <td>\"I&amp;#039;ve tried a few antidepressants over th...</td>\n",
              "      <td>10</td>\n",
              "      <td>28-Feb-12</td>\n",
              "      <td>22</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>206473</td>\n",
              "      <td>Mesalamine</td>\n",
              "      <td>Crohn's Disease, Maintenance</td>\n",
              "      <td>\"My son has Crohn&amp;#039;s disease and has done ...</td>\n",
              "      <td>8</td>\n",
              "      <td>17-May-09</td>\n",
              "      <td>17</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>159672</td>\n",
              "      <td>Bactrim</td>\n",
              "      <td>Urinary Tract Infection</td>\n",
              "      <td>\"Quick reduction of symptoms\"</td>\n",
              "      <td>9</td>\n",
              "      <td>29-Sep-17</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>39293</td>\n",
              "      <td>Contrave</td>\n",
              "      <td>Weight Loss</td>\n",
              "      <td>\"Contrave combines drugs that were used for al...</td>\n",
              "      <td>9</td>\n",
              "      <td>5-Mar-17</td>\n",
              "      <td>35</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>97768</td>\n",
              "      <td>Cyclafem 1 / 35</td>\n",
              "      <td>Birth Control</td>\n",
              "      <td>\"I have been on this birth control for one cyc...</td>\n",
              "      <td>9</td>\n",
              "      <td>22-Oct-15</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   uniqueID         drugName  ...       date usefulCount\n",
              "0    163740      Mirtazapine  ...  28-Feb-12          22\n",
              "1    206473       Mesalamine  ...  17-May-09          17\n",
              "2    159672          Bactrim  ...  29-Sep-17           3\n",
              "3     39293         Contrave  ...   5-Mar-17          35\n",
              "4     97768  Cyclafem 1 / 35  ...  22-Oct-15           4\n",
              "\n",
              "[5 rows x 7 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "z0R27T0waVC8",
        "outputId": "4a6827ff-4640-49f3-bb76-6fa8feca4faa"
      },
      "source": [
        "# as both the dataset contains same columns we can combine them for better analysis\n",
        "data = pd.concat([train, test])\n",
        "\n",
        "# checking the shape\n",
        "data.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(215063, 7)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YE4VJbWeHV3P"
      },
      "source": [
        "# feature engineering \n",
        "# let's make a new column review sentiment \n",
        "\n",
        "data.loc[(data['rating'] >= 7), 'Review_Sentiment'] = 2\n",
        "data.loc[(data['rating']== 5), 'Review_Sentiment'] = 1\n",
        "data.loc[(data['rating']== 6), 'Review_Sentiment'] = 1\n",
        "data.loc[(data['rating']<5), 'Review_Sentiment'] = 0\n",
        "data['Review_Sentiment'].value_counts()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Fa53J8pXQZR3"
      },
      "source": [
        "# Data Preprocessing"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WCbskXHWjYGI"
      },
      "source": [
        "##Basic Data Info"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5wI66HLlaqe0"
      },
      "source": [
        "Check for null values"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-pzoIy09aoFV",
        "outputId": "fc8b3f21-98fb-4b4c-926b-a7ca537ea104"
      },
      "source": [
        "data.isnull().any()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "uniqueID            False\n",
              "drugName            False\n",
              "condition            True\n",
              "review              False\n",
              "rating              False\n",
              "date                False\n",
              "usefulCount         False\n",
              "Review_Sentiment    False\n",
              "Year                False\n",
              "month               False\n",
              "day                 False\n",
              "dtype: bool"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cz-1dTzKa9JP",
        "outputId": "e53c1729-22c3-4b9a-c553-8ec895ff1b37"
      },
      "source": [
        "# we will delete the rows so that the data does not overfits \n",
        "\n",
        "data = data.dropna(axis = 0)\n",
        "\n",
        "# checking the new shape of the data\n",
        "data.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(213869, 11)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XKDGZDYxcPNo",
        "outputId": "a2b33e86-d926-4b83-b15c-19d163171569"
      },
      "source": [
        "# importing the important libraries \n",
        "\n",
        "import re\n",
        "from bs4 import BeautifulSoup\n",
        "\n",
        "nltk.download('stopwords')\n",
        "from nltk.corpus import stopwords\n",
        "from nltk.stem.snowball import SnowballStemmer\n",
        "from nltk.stem.porter import PorterStemmer"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H-927tAecSS-"
      },
      "source": [
        "# removing some stopwords from the list of stopwords as they are important for drug recommendation\n",
        "\n",
        "stops = set(stopwords.words('english'))\n",
        "\n",
        "not_stop = [\"aren't\",\"couldn't\",\"didn't\",\"doesn't\",\"don't\",\"hadn't\",\"hasn't\",\"haven't\",\"isn't\",\"mightn't\",\n",
        "            \"mustn't\",\"needn't\",\"no\",\"nor\",\"not\",\"shan't\",\"shouldn't\",\"wasn't\",\"weren't\",\"wouldn't\"]\n",
        "for i in not_stop:\n",
        "    stops.remove(i)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OketBEXTcm-2",
        "outputId": "0e8fa9ac-29ea-4be7-ffd2-a1fb1b88ea48"
      },
      "source": [
        "data.columns"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Index(['uniqueID', 'drugName', 'condition', 'review', 'rating', 'date',\n",
              "       'usefulCount', 'Review_Sentiment', 'Year', 'month', 'day'],\n",
              "      dtype='object')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 639
        },
        "id": "oxuSrIqgcp31",
        "outputId": "dae3800b-7b7b-4887-802a-5a75bac8bd00"
      },
      "source": [
        "df_condition = data.groupby(['condition'])['drugName'].nunique().sort_values(ascending=False)\n",
        "df_condition = pd.DataFrame(df_condition).reset_index()\n",
        "df_condition.tail(20)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>condition</th>\n",
              "      <th>drugName</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>896</th>\n",
              "      <td>Short Stature</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>897</th>\n",
              "      <td>Hemangioma</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>898</th>\n",
              "      <td>Short Stature for Age</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>899</th>\n",
              "      <td>Sleep Paralysis</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>900</th>\n",
              "      <td>Gestational Diabetes</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>901</th>\n",
              "      <td>Gastric Cance</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>902</th>\n",
              "      <td>Gallbladder Disease</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>903</th>\n",
              "      <td>Dermatitis Herpetiformis</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>904</th>\n",
              "      <td>Somatoform Pain Disorde</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>905</th>\n",
              "      <td>Esophageal Variceal Hemorrhage Prophylaxis</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>906</th>\n",
              "      <td>Esophageal Spasm</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>907</th>\n",
              "      <td>Sporotrichosis</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>908</th>\n",
              "      <td>Epididymitis, Sexually Transmitted</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>909</th>\n",
              "      <td>Pertussis</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>910</th>\n",
              "      <td>Ehrlichiosis</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>911</th>\n",
              "      <td>Steroid Responsive Inflammatory Conditions</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>912</th>\n",
              "      <td>Ectopic Pregnancy</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>913</th>\n",
              "      <td>Diagnostic Bronchograms</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>914</th>\n",
              "      <td>Systemic Candidiasis</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>915</th>\n",
              "      <td>Epicondylitis, Tennis Elbow</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                      condition  drugName\n",
              "896                               Short Stature         1\n",
              "897                                  Hemangioma         1\n",
              "898                       Short Stature for Age         1\n",
              "899                             Sleep Paralysis         1\n",
              "900                        Gestational Diabetes         1\n",
              "901                               Gastric Cance         1\n",
              "902                         Gallbladder Disease         1\n",
              "903                    Dermatitis Herpetiformis         1\n",
              "904                     Somatoform Pain Disorde         1\n",
              "905  Esophageal Variceal Hemorrhage Prophylaxis         1\n",
              "906                            Esophageal Spasm         1\n",
              "907                              Sporotrichosis         1\n",
              "908          Epididymitis, Sexually Transmitted         1\n",
              "909                                   Pertussis         1\n",
              "910                                Ehrlichiosis         1\n",
              "911  Steroid Responsive Inflammatory Conditions         1\n",
              "912                           Ectopic Pregnancy         1\n",
              "913                     Diagnostic Bronchograms         1\n",
              "914                        Systemic Candidiasis         1\n",
              "915                 Epicondylitis, Tennis Elbow         1"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pflJ6Q65c7pV"
      },
      "source": [
        "Removing medical conditions which have less than 5 drug associated with it in the dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mvkCOuhNc03T"
      },
      "source": [
        "# setting a df with conditions with less than 5 drugs\n",
        "df_condition_1 = df_condition[df_condition['drugName']<5].reset_index()\n",
        "\n",
        "all_list = set(data.index)\n",
        "\n",
        "# deleting them\n",
        "condition_list = []\n",
        "for i,j in enumerate(data['condition']):\n",
        "    for c in list(df_condition_1['condition']):\n",
        "        if j == c:\n",
        "            condition_list.append(i)\n",
        "            \n",
        "new_idx = all_list.difference(set(condition_list))\n",
        "data = data.iloc[list(new_idx)].reset_index()\n",
        "del data['index']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JAgoNQ4IdHSZ"
      },
      "source": [
        "removing the conditions with the word \"\\span\" in it.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y6FCE4DRdDdF"
      },
      "source": [
        "all_list = set(data.index)\n",
        "span_list = []\n",
        "for i,j in enumerate(data['condition']):\n",
        "    if '</span>' in j:\n",
        "        span_list.append(i)\n",
        "new_idx = all_list.difference(set(span_list))\n",
        "data = data.iloc[list(new_idx)].reset_index()\n",
        "del data['index']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BcePLbJ9dJEB",
        "outputId": "c31d6778-f83b-4699-b5cc-1e127ca86d0e"
      },
      "source": [
        "data.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(156432, 11)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 47
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "m2zxwNQDdhyv"
      },
      "source": [
        "## Applying data cleanup with -\n",
        "\n",
        "\n",
        "1.   Removing HTML tags\n",
        "2.   Removing non letters\n",
        "3.   Convert all to lower case\n",
        "4.   Remove stopwords\n",
        "5.   Apply stemming\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7CMWLiGBdO7L"
      },
      "source": [
        "stemmer = SnowballStemmer('english')\n",
        "\n",
        "def review_to_words(raw_review):\n",
        "    # 1. Delete HTML \n",
        "    review_text = BeautifulSoup(raw_review, 'html.parser').get_text()\n",
        "    # 2. Make a space\n",
        "    letters_only = re.sub('[^a-zA-Z]', ' ', review_text)\n",
        "    # 3. lower letters\n",
        "    words = letters_only.lower().split()\n",
        "    # 4. Stopwords \n",
        "    meaningful_words = [w for w in words if not w in stops]\n",
        "    # 5. Stemming\n",
        "    stemming_words = [stemmer.stem(w) for w in meaningful_words]\n",
        "    # 6. space join words\n",
        "    return( ' '.join(stemming_words))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RxvlgxTcdS87",
        "outputId": "f7a03916-fb62-40a1-fdb4-4f88d4ffc0f8"
      },
      "source": [
        "%time data['review_clean'] = data['review'].apply(review_to_words)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 1min 58s, sys: 991 ms, total: 1min 59s\n",
            "Wall time: 1min 59s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Z7pTDpAl25pb"
      },
      "source": [
        "#Analysing Emotions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EYWCTmB73An9",
        "outputId": "65fd1fd3-e47f-47c1-9032-2d9cba872f11"
      },
      "source": [
        "# Make data directory if it doesn't exist\n",
        "!mkdir -p nrcdata\n",
        "!wget -nc https://nyc3.digitaloceanspaces.com/ml-files-distro/v1/upshot-trump-emolex/data/NRC-emotion-lexicon-wordlevel-alphabetized-v0.92.txt -P nrcdata"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2021-05-15 10:17:11--  https://nyc3.digitaloceanspaces.com/ml-files-distro/v1/upshot-trump-emolex/data/NRC-emotion-lexicon-wordlevel-alphabetized-v0.92.txt\n",
            "Resolving nyc3.digitaloceanspaces.com (nyc3.digitaloceanspaces.com)... 162.243.189.2\n",
            "Connecting to nyc3.digitaloceanspaces.com (nyc3.digitaloceanspaces.com)|162.243.189.2|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 2581050 (2.5M) [text/plain]\n",
            "Saving to: ‘nrcdata/NRC-emotion-lexicon-wordlevel-alphabetized-v0.92.txt’\n",
            "\n",
            "NRC-emotion-lexicon 100%[===================>]   2.46M  --.-KB/s    in 0.1s    \n",
            "\n",
            "2021-05-15 10:17:11 (21.3 MB/s) - ‘nrcdata/NRC-emotion-lexicon-wordlevel-alphabetized-v0.92.txt’ saved [2581050/2581050]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 402
        },
        "id": "NDgJ14cD317O",
        "outputId": "31289f48-1d73-4c9a-db52-ed262c1c8d42"
      },
      "source": [
        "filepath = \"/content/nrcdata/NRC-emotion-lexicon-wordlevel-alphabetized-v0.92.txt\"\n",
        "emolex_df = pd.read_csv(filepath,  names=[\"word\", \"emotion\", \"association\"], skiprows=45, sep='\\t', keep_default_na=False)\n",
        "emolex_df.head(12)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>word</th>\n",
              "      <th>emotion</th>\n",
              "      <th>association</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>aback</td>\n",
              "      <td>anger</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>aback</td>\n",
              "      <td>anticipation</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>aback</td>\n",
              "      <td>disgust</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>aback</td>\n",
              "      <td>fear</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>aback</td>\n",
              "      <td>joy</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>aback</td>\n",
              "      <td>negative</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>aback</td>\n",
              "      <td>positive</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>aback</td>\n",
              "      <td>sadness</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>aback</td>\n",
              "      <td>surprise</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>aback</td>\n",
              "      <td>trust</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>abacus</td>\n",
              "      <td>anger</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>abacus</td>\n",
              "      <td>anticipation</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      word       emotion  association\n",
              "0    aback         anger            0\n",
              "1    aback  anticipation            0\n",
              "2    aback       disgust            0\n",
              "3    aback          fear            0\n",
              "4    aback           joy            0\n",
              "5    aback      negative            0\n",
              "6    aback      positive            0\n",
              "7    aback       sadness            0\n",
              "8    aback      surprise            0\n",
              "9    aback         trust            0\n",
              "10  abacus         anger            0\n",
              "11  abacus  anticipation            0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 58
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "n7M-QAwk4zNY",
        "outputId": "01a96d56-51ec-4a1a-ac1e-178235e3e222"
      },
      "source": [
        "emolex_df.emotion.unique()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['anger', 'anticipation', 'disgust', 'fear', 'joy', 'negative',\n",
              "       'positive', 'sadness', 'surprise', 'trust'], dtype=object)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 59
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "i9wCpdSx40uv",
        "outputId": "5ce04718-afed-4978-a185-4ed7ca12df98"
      },
      "source": [
        "emolex_df.emotion.value_counts()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "positive        14182\n",
              "trust           14182\n",
              "negative        14182\n",
              "anticipation    14182\n",
              "disgust         14182\n",
              "fear            14182\n",
              "anger           14182\n",
              "joy             14182\n",
              "surprise        14182\n",
              "sadness         14182\n",
              "Name: emotion, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 60
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XGY5IZ7v45_s",
        "outputId": "e9a736b8-c7be-4c77-9ea4-b4b7a6230054"
      },
      "source": [
        "emolex_df[emolex_df.association == 1].emotion.value_counts()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "negative        3324\n",
              "positive        2312\n",
              "fear            1476\n",
              "anger           1247\n",
              "trust           1231\n",
              "sadness         1191\n",
              "disgust         1058\n",
              "anticipation     839\n",
              "joy              689\n",
              "surprise         534\n",
              "Name: emotion, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 61
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 195
        },
        "id": "cBIlXcUN4_Mh",
        "outputId": "b888d564-a651-4480-b89c-f199be89842d"
      },
      "source": [
        "emolex_words = emolex_df.pivot(index='word', columns='emotion', values='association').reset_index()\n",
        "emolex_words.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th>emotion</th>\n",
              "      <th>word</th>\n",
              "      <th>anger</th>\n",
              "      <th>anticipation</th>\n",
              "      <th>disgust</th>\n",
              "      <th>fear</th>\n",
              "      <th>joy</th>\n",
              "      <th>negative</th>\n",
              "      <th>positive</th>\n",
              "      <th>sadness</th>\n",
              "      <th>surprise</th>\n",
              "      <th>trust</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>aback</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>abacus</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>abandon</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>abandoned</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>abandonment</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "emotion         word  anger  anticipation  ...  sadness  surprise  trust\n",
              "0              aback      0             0  ...        0         0      0\n",
              "1             abacus      0             0  ...        0         0      1\n",
              "2            abandon      0             0  ...        1         0      0\n",
              "3          abandoned      1             0  ...        1         0      0\n",
              "4        abandonment      1             0  ...        1         1      0\n",
              "\n",
              "[5 rows x 11 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 62
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W_H0iJAu6kR6"
      },
      "source": [
        "def extract_review_emotion(df,column):\n",
        "    new_df = df.copy()\n",
        "    \n",
        "    emotions = emolex_words.columns.drop('word')\n",
        "    emo_df = pd.DataFrame(0, index=df.index, columns=emotions) \n",
        "    stemmer = SnowballStemmer(\"english\")   \n",
        "    with tqdm(total=len(list(new_df.iterrows()))) as pbar:\n",
        "        for i, row in new_df.iterrows():\n",
        "            pbar.update(1)\n",
        "            document = word_tokenize(new_df.loc[i][column])\n",
        "            for word in document:\n",
        "                word = stemmer.stem(word.lower())\n",
        "                emo_score = emolex_words[emolex_words.word == word]\n",
        "                if not emo_score.empty:\n",
        "                    for emotion in list(emotions):\n",
        "                        emo_df.at[i, emotion] += emo_score[emotion]\n",
        "\n",
        "    new_df = pd.concat([new_df, emo_df], axis=1)\n",
        "    return new_df"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 116,
          "referenced_widgets": [
            "3d3cb6730c5d4056bb6dc7311c6c54fd",
            "3b87966b87cd4cf28ba3d8b27e61db0f",
            "3ba53b52c9944089ba16e23aeb747f36",
            "bc60033e665f4b8fae5f2afe04f3dc3b",
            "11b4e4c9151340f6b03746e0c1db6faa",
            "8a2a40681b8d495db88a4df7c6dfe400",
            "642a584be5194fae9be0761c5011ad3b",
            "ff1e315a5a6041cbbbf0ad6a838e8dd8"
          ]
        },
        "id": "50wcd_rP7Fqn",
        "outputId": "8a87da20-d583-4e5d-c24b-7f8b1639f754"
      },
      "source": [
        "emotion_df = extract_review_emotion(data, 'review_clean')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:7: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
            "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n",
            "  import sys\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "3d3cb6730c5d4056bb6dc7311c6c54fd",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=156432.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6jtU5AV9vLln"
      },
      "source": [
        "emo = emotion_df.groupby(['drugName']).sum()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "19rHUY2bvhRa",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 477
        },
        "outputId": "5902f851-bd2c-4dab-bff2-05692843b1bf"
      },
      "source": [
        "emo.head(10)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>uniqueID</th>\n",
              "      <th>rating</th>\n",
              "      <th>usefulCount</th>\n",
              "      <th>Review_Sentiment</th>\n",
              "      <th>Year</th>\n",
              "      <th>month</th>\n",
              "      <th>day</th>\n",
              "      <th>anger</th>\n",
              "      <th>anticipation</th>\n",
              "      <th>disgust</th>\n",
              "      <th>fear</th>\n",
              "      <th>joy</th>\n",
              "      <th>negative</th>\n",
              "      <th>positive</th>\n",
              "      <th>sadness</th>\n",
              "      <th>surprise</th>\n",
              "      <th>trust</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>drugName</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>A + D Cracked Skin Relief</th>\n",
              "      <td>163935</td>\n",
              "      <td>10</td>\n",
              "      <td>6</td>\n",
              "      <td>2.0</td>\n",
              "      <td>2009</td>\n",
              "      <td>12</td>\n",
              "      <td>7</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>3</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>A / B Otic</th>\n",
              "      <td>131173</td>\n",
              "      <td>10</td>\n",
              "      <td>20</td>\n",
              "      <td>2.0</td>\n",
              "      <td>2009</td>\n",
              "      <td>9</td>\n",
              "      <td>23</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Abacavir / dolutegravir / lamivudine</th>\n",
              "      <td>3748194</td>\n",
              "      <td>437</td>\n",
              "      <td>653</td>\n",
              "      <td>88.0</td>\n",
              "      <td>106842</td>\n",
              "      <td>341</td>\n",
              "      <td>911</td>\n",
              "      <td>25</td>\n",
              "      <td>100</td>\n",
              "      <td>52</td>\n",
              "      <td>45</td>\n",
              "      <td>50</td>\n",
              "      <td>100</td>\n",
              "      <td>132</td>\n",
              "      <td>62</td>\n",
              "      <td>22</td>\n",
              "      <td>103</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Abacavir / lamivudine / zidovudine</th>\n",
              "      <td>117239</td>\n",
              "      <td>9</td>\n",
              "      <td>1</td>\n",
              "      <td>2.0</td>\n",
              "      <td>2014</td>\n",
              "      <td>9</td>\n",
              "      <td>6</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Abatacept</th>\n",
              "      <td>1447899</td>\n",
              "      <td>136</td>\n",
              "      <td>976</td>\n",
              "      <td>29.0</td>\n",
              "      <td>38272</td>\n",
              "      <td>97</td>\n",
              "      <td>278</td>\n",
              "      <td>11</td>\n",
              "      <td>25</td>\n",
              "      <td>10</td>\n",
              "      <td>25</td>\n",
              "      <td>23</td>\n",
              "      <td>45</td>\n",
              "      <td>43</td>\n",
              "      <td>25</td>\n",
              "      <td>9</td>\n",
              "      <td>25</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Abilify</th>\n",
              "      <td>39992508</td>\n",
              "      <td>2758</td>\n",
              "      <td>17179</td>\n",
              "      <td>556.0</td>\n",
              "      <td>849650</td>\n",
              "      <td>2829</td>\n",
              "      <td>6864</td>\n",
              "      <td>318</td>\n",
              "      <td>1025</td>\n",
              "      <td>330</td>\n",
              "      <td>695</td>\n",
              "      <td>624</td>\n",
              "      <td>1175</td>\n",
              "      <td>1121</td>\n",
              "      <td>661</td>\n",
              "      <td>406</td>\n",
              "      <td>661</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Abilify Discmelt</th>\n",
              "      <td>240464</td>\n",
              "      <td>16</td>\n",
              "      <td>19</td>\n",
              "      <td>4.0</td>\n",
              "      <td>4021</td>\n",
              "      <td>6</td>\n",
              "      <td>35</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Abilify Maintena</th>\n",
              "      <td>750287</td>\n",
              "      <td>31</td>\n",
              "      <td>27</td>\n",
              "      <td>6.0</td>\n",
              "      <td>8064</td>\n",
              "      <td>24</td>\n",
              "      <td>46</td>\n",
              "      <td>6</td>\n",
              "      <td>11</td>\n",
              "      <td>5</td>\n",
              "      <td>7</td>\n",
              "      <td>8</td>\n",
              "      <td>10</td>\n",
              "      <td>12</td>\n",
              "      <td>7</td>\n",
              "      <td>8</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Abiraterone</th>\n",
              "      <td>1699270</td>\n",
              "      <td>82</td>\n",
              "      <td>360</td>\n",
              "      <td>16.0</td>\n",
              "      <td>18125</td>\n",
              "      <td>64</td>\n",
              "      <td>152</td>\n",
              "      <td>3</td>\n",
              "      <td>5</td>\n",
              "      <td>4</td>\n",
              "      <td>6</td>\n",
              "      <td>3</td>\n",
              "      <td>10</td>\n",
              "      <td>8</td>\n",
              "      <td>6</td>\n",
              "      <td>3</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>AbobotulinumtoxinA</th>\n",
              "      <td>397353</td>\n",
              "      <td>21</td>\n",
              "      <td>23</td>\n",
              "      <td>5.0</td>\n",
              "      <td>6043</td>\n",
              "      <td>16</td>\n",
              "      <td>38</td>\n",
              "      <td>1</td>\n",
              "      <td>5</td>\n",
              "      <td>0</td>\n",
              "      <td>4</td>\n",
              "      <td>3</td>\n",
              "      <td>7</td>\n",
              "      <td>5</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                      uniqueID  rating  ...  surprise  trust\n",
              "drugName                                                ...                 \n",
              "A + D Cracked Skin Relief               163935      10  ...         0      2\n",
              "A / B Otic                              131173      10  ...         1      1\n",
              "Abacavir / dolutegravir / lamivudine   3748194     437  ...        22    103\n",
              "Abacavir / lamivudine / zidovudine      117239       9  ...         1      1\n",
              "Abatacept                              1447899     136  ...         9     25\n",
              "Abilify                               39992508    2758  ...       406    661\n",
              "Abilify Discmelt                        240464      16  ...         0      0\n",
              "Abilify Maintena                        750287      31  ...         8      8\n",
              "Abiraterone                            1699270      82  ...         3      3\n",
              "AbobotulinumtoxinA                      397353      21  ...         1      3\n",
              "\n",
              "[10 rows x 17 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 71
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gX-wGuQ3vrJS"
      },
      "source": [
        "emo.to_csv('emotion_groupby11.csv',index=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S2qlV98vsj96"
      },
      "source": [
        "emotion_df.to_csv('emotion_sentiment.csv',index=False)"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}