predicting-tags-for-stackoverflow-deep-learning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "a7cf70d54413735fe632aaa5df43f34e353e0ff4",
    "papermill": {
     "duration": 0.058089,
     "end_time": "2020-09-29T10:42:01.835511",
     "exception": false,
     "start_time": "2020-09-29T10:42:01.777422",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "**PART 1: Cleaning Data and Exploratory Data Analysis**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_kg_hide-input": true,
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2020-09-29T10:42:01.958545Z",
     "iopub.status.busy": "2020-09-29T10:42:01.957795Z",
     "iopub.status.idle": "2020-09-29T10:42:03.627682Z",
     "shell.execute_reply": "2020-09-29T10:42:03.626581Z"
    },
    "papermill": {
     "duration": 1.735358,
     "end_time": "2020-09-29T10:42:03.627797",
     "exception": false,
     "start_time": "2020-09-29T10:42:01.892439",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "\n",
    "from string import punctuation\n",
    "import matplotlib.pyplot as plt\n",
    "from nltk.corpus import stopwords\n",
    "import warnings, pickle, time, re, nltk\n",
    "from nltk.stem.wordnet import WordNetLemmatizer\n",
    "\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.metrics import make_scorer\n",
    "from sklearn.metrics import hamming_loss\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import MultiLabelBinarizer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_uuid": "dc1c92bf4166de1e2ce4bfb229b40b32eb94374a",
    "execution": {
     "iopub.execute_input": "2020-09-29T10:42:03.752274Z",
     "iopub.status.busy": "2020-09-29T10:42:03.751578Z",
     "iopub.status.idle": "2020-09-29T10:42:03.757754Z",
     "shell.execute_reply": "2020-09-29T10:42:03.758727Z"
    },
    "papermill": {
     "duration": 0.073498,
     "end_time": "2020-09-29T10:42:03.758870",
     "exception": false,
     "start_time": "2020-09-29T10:42:03.685372",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Answers.csv', 'Tags.csv', 'Questions.csv']\n"
     ]
    }
   ],
   "source": [
    "import os \n",
    "print(os.listdir(\"../input\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "_uuid": "120edf8b2306a6391ad992cf3f48598739ec1940",
    "execution": {
     "iopub.execute_input": "2020-09-29T10:42:03.879941Z",
     "iopub.status.busy": "2020-09-29T10:42:03.879332Z",
     "iopub.status.idle": "2020-09-29T10:42:27.382625Z",
     "shell.execute_reply": "2020-09-29T10:42:27.381445Z"
    },
    "papermill": {
     "duration": 23.565231,
     "end_time": "2020-09-29T10:42:27.382752",
     "exception": false,
     "start_time": "2020-09-29T10:42:03.817521",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"../input/Questions.csv\", encoding=\"ISO-8859-1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "_uuid": "40c3e09d38aecf03ce13fd0d88809e80a67d3d1a",
    "execution": {
     "iopub.execute_input": "2020-09-29T10:42:27.516623Z",
     "iopub.status.busy": "2020-09-29T10:42:27.515607Z",
     "iopub.status.idle": "2020-09-29T10:42:27.527954Z",
     "shell.execute_reply": "2020-09-29T10:42:27.528445Z"
    },
    "papermill": {
     "duration": 0.087647,
     "end_time": "2020-09-29T10:42:27.528584",
     "exception": false,
     "start_time": "2020-09-29T10:42:27.440937",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1264216, 7)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>OwnerUserId</th>\n",
       "      <th>CreationDate</th>\n",
       "      <th>ClosedDate</th>\n",
       "      <th>Score</th>\n",
       "      <th>Title</th>\n",
       "      <th>Body</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>80</td>\n",
       "      <td>26.0</td>\n",
       "      <td>2008-08-01T13:57:07Z</td>\n",
       "      <td>NaN</td>\n",
       "      <td>26</td>\n",
       "      <td>SQLStatement.execute() - multiple queries in o...</td>\n",
       "      <td>&lt;p&gt;I've written a database generation script i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>90</td>\n",
       "      <td>58.0</td>\n",
       "      <td>2008-08-01T14:41:24Z</td>\n",
       "      <td>2012-12-26T03:45:49Z</td>\n",
       "      <td>144</td>\n",
       "      <td>Good branching and merging tutorials for Torto...</td>\n",
       "      <td>&lt;p&gt;Are there any really good tutorials explain...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>120</td>\n",
       "      <td>83.0</td>\n",
       "      <td>2008-08-01T15:50:08Z</td>\n",
       "      <td>NaN</td>\n",
       "      <td>21</td>\n",
       "      <td>ASP.NET Site Maps</td>\n",
       "      <td>&lt;p&gt;Has anyone got experience creating &lt;strong&gt;...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>180</td>\n",
       "      <td>2089740.0</td>\n",
       "      <td>2008-08-01T18:42:19Z</td>\n",
       "      <td>NaN</td>\n",
       "      <td>53</td>\n",
       "      <td>Function for creating color wheels</td>\n",
       "      <td>&lt;p&gt;This is something I've pseudo-solved many t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>260</td>\n",
       "      <td>91.0</td>\n",
       "      <td>2008-08-01T23:22:08Z</td>\n",
       "      <td>NaN</td>\n",
       "      <td>49</td>\n",
       "      <td>Adding scripting functionality to .NET applica...</td>\n",
       "      <td>&lt;p&gt;I have a little game written in C#. It uses...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Id  OwnerUserId          CreationDate            ClosedDate  Score  \\\n",
       "0   80         26.0  2008-08-01T13:57:07Z                   NaN     26   \n",
       "1   90         58.0  2008-08-01T14:41:24Z  2012-12-26T03:45:49Z    144   \n",
       "2  120         83.0  2008-08-01T15:50:08Z                   NaN     21   \n",
       "3  180    2089740.0  2008-08-01T18:42:19Z                   NaN     53   \n",
       "4  260         91.0  2008-08-01T23:22:08Z                   NaN     49   \n",
       "\n",
       "                                               Title  \\\n",
       "0  SQLStatement.execute() - multiple queries in o...   \n",
       "1  Good branching and merging tutorials for Torto...   \n",
       "2                                  ASP.NET Site Maps   \n",
       "3                 Function for creating color wheels   \n",
       "4  Adding scripting functionality to .NET applica...   \n",
       "\n",
       "                                                Body  \n",
       "0  <p>I've written a database generation script i...  \n",
       "1  <p>Are there any really good tutorials explain...  \n",
       "2  <p>Has anyone got experience creating <strong>...  \n",
       "3  <p>This is something I've pseudo-solved many t...  \n",
       "4  <p>I have a little game written in C#. It uses...  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(df.shape)\n",
    "df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "_uuid": "9beed1bea78dd1ed930caeef2923aef03420e1cc",
    "execution": {
     "iopub.execute_input": "2020-09-29T10:42:27.652629Z",
     "iopub.status.busy": "2020-09-29T10:42:27.651960Z",
     "iopub.status.idle": "2020-09-29T10:42:29.136297Z",
     "shell.execute_reply": "2020-09-29T10:42:29.136946Z"
    },
    "papermill": {
     "duration": 1.549354,
     "end_time": "2020-09-29T10:42:29.137143",
     "exception": false,
     "start_time": "2020-09-29T10:42:27.587789",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "tags = pd.read_csv(\"../input/Tags.csv\", encoding=\"ISO-8859-1\", dtype={'Tag': str})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "_uuid": "3f4a7e6a7ee2745fc207f19fccb9c0dbc3a571a4",
    "execution": {
     "iopub.execute_input": "2020-09-29T10:42:29.304724Z",
     "iopub.status.busy": "2020-09-29T10:42:29.304095Z",
     "iopub.status.idle": "2020-09-29T10:42:29.312244Z",
     "shell.execute_reply": "2020-09-29T10:42:29.311595Z"
    },
    "papermill": {
     "duration": 0.097067,
     "end_time": "2020-09-29T10:42:29.312346",
     "exception": false,
     "start_time": "2020-09-29T10:42:29.215279",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3750994, 2)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>Tag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>80</td>\n",
       "      <td>flex</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>80</td>\n",
       "      <td>actionscript-3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>80</td>\n",
       "      <td>air</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>90</td>\n",
       "      <td>svn</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>90</td>\n",
       "      <td>tortoisesvn</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Id             Tag\n",
       "0  80            flex\n",
       "1  80  actionscript-3\n",
       "2  80             air\n",
       "3  90             svn\n",
       "4  90     tortoisesvn"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(tags.shape)\n",
    "tags.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "_uuid": "ab191660e879734a6c26d1c1f2afd0e1218a50fc",
    "execution": {
     "iopub.execute_input": "2020-09-29T10:42:29.481805Z",
     "iopub.status.busy": "2020-09-29T10:42:29.480977Z",
     "iopub.status.idle": "2020-09-29T10:42:30.053336Z",
     "shell.execute_reply": "2020-09-29T10:42:30.052463Z"
    },
    "papermill": {
     "duration": 0.66566,
     "end_time": "2020-09-29T10:42:30.053442",
     "exception": false,
     "start_time": "2020-09-29T10:42:29.387782",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Id                    0\n",
       "OwnerUserId       14454\n",
       "CreationDate          0\n",
       "ClosedDate      1208257\n",
       "Score                 0\n",
       "Title                 0\n",
       "Body                  0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "_uuid": "b71d45e2c86152d95f22679f01d8f855b042a62e",
    "execution": {
     "iopub.execute_input": "2020-09-29T10:42:30.522280Z",
     "iopub.status.busy": "2020-09-29T10:42:30.180124Z",
     "iopub.status.idle": "2020-09-29T10:42:30.538225Z",
     "shell.execute_reply": "2020-09-29T10:42:30.538739Z"
    },
    "papermill": {
     "duration": 0.424063,
     "end_time": "2020-09-29T10:42:30.538869",
     "exception": false,
     "start_time": "2020-09-29T10:42:30.114806",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Id        0\n",
       "Tag    1113\n",
       "dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tags.isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "ec6a03e07edd3ce3de1f20f073bdf18ffca87f4c",
    "papermill": {
     "duration": 0.061815,
     "end_time": "2020-09-29T10:42:30.662143",
     "exception": false,
     "start_time": "2020-09-29T10:42:30.600328",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "First, what I want to do is to merge both dataframes. In order to do that, I'll have to group tags by the id of the post since a post can have multiple tags. I'll just use the groupeby function and then merge the dataframes on the id. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "_uuid": "933761c8b2bdb668e0bca3c9ea9cab36dd1a4391",
    "execution": {
     "iopub.execute_input": "2020-09-29T10:42:30.793649Z",
     "iopub.status.busy": "2020-09-29T10:42:30.792698Z",
     "iopub.status.idle": "2020-09-29T10:42:32.083003Z",
     "shell.execute_reply": "2020-09-29T10:42:32.081971Z"
    },
    "papermill": {
     "duration": 1.358948,
     "end_time": "2020-09-29T10:42:32.083159",
     "exception": false,
     "start_time": "2020-09-29T10:42:30.724211",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "tags['Tag'] = tags['Tag'].astype(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "_kg_hide-output": true,
    "_uuid": "267d7c8f01a3b427ab139d689ce5d13ab8820913",
    "execution": {
     "iopub.execute_input": "2020-09-29T10:42:32.216398Z",
     "iopub.status.busy": "2020-09-29T10:42:32.215438Z",
     "iopub.status.idle": "2020-09-29T10:43:11.792228Z",
     "shell.execute_reply": "2020-09-29T10:43:11.791653Z"
    },
    "papermill": {
     "duration": 39.647163,
     "end_time": "2020-09-29T10:43:11.792349",
     "exception": false,
     "start_time": "2020-09-29T10:42:32.145186",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "grouped_tags = tags.groupby(\"Id\")['Tag'].apply(lambda tags: ' '.join(tags)).reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "_uuid": "d4096130e47ec904f9bbe27822d1ddf403a675dc",
    "execution": {
     "iopub.execute_input": "2020-09-29T10:43:11.925438Z",
     "iopub.status.busy": "2020-09-29T10:43:11.924606Z",
     "iopub.status.idle": "2020-09-29T10:43:11.927874Z",
     "shell.execute_reply": "2020-09-29T10:43:11.928378Z"
    },
    "papermill": {
     "duration": 0.074211,
     "end_time": "2020-09-29T10:43:11.928502",
     "exception": false,
     "start_time": "2020-09-29T10:43:11.854291",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>Tag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>80</td>\n",
       "      <td>flex actionscript-3 air</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>90</td>\n",
       "      <td>svn tortoisesvn branch branching-and-merging</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>120</td>\n",
       "      <td>sql asp.net sitemap</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>180</td>\n",
       "      <td>algorithm language-agnostic colors color-space</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>260</td>\n",
       "      <td>c# .net scripting compiler-construction</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Id                                             Tag\n",
       "0   80                         flex actionscript-3 air\n",
       "1   90    svn tortoisesvn branch branching-and-merging\n",
       "2  120                             sql asp.net sitemap\n",
       "3  180  algorithm language-agnostic colors color-space\n",
       "4  260         c# .net scripting compiler-construction"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grouped_tags.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "_uuid": "2612dea031562d420f8dac14801d167721d2bcc9",
    "execution": {
     "iopub.execute_input": "2020-09-29T10:43:12.065096Z",
     "iopub.status.busy": "2020-09-29T10:43:12.063646Z",
     "iopub.status.idle": "2020-09-29T10:43:12.136435Z",
     "shell.execute_reply": "2020-09-29T10:43:12.137290Z"
    },
    "papermill": {
     "duration": 0.147055,
     "end_time": "2020-09-29T10:43:12.137480",
     "exception": false,
     "start_time": "2020-09-29T10:43:11.990425",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df.drop(columns=['OwnerUserId', 'CreationDate', 'ClosedDate'], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "_uuid": "bd794a28df6e19818c754702ed5f3745262d866f",
    "execution": {
     "iopub.execute_input": "2020-09-29T10:43:12.386113Z",
     "iopub.status.busy": "2020-09-29T10:43:12.385289Z",
     "iopub.status.idle": "2020-09-29T10:43:13.173496Z",
     "shell.execute_reply": "2020-09-29T10:43:13.172312Z"
    },
    "papermill": {
     "duration": 0.888917,
     "end_time": "2020-09-29T10:43:13.173700",
     "exception": false,
     "start_time": "2020-09-29T10:43:12.284783",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df = df.merge(grouped_tags, on='Id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "_uuid": "132c4501747d2832eaf8d95f5d9cb3acaadc97f4",
    "execution": {
     "iopub.execute_input": "2020-09-29T10:43:13.384984Z",
     "iopub.status.busy": "2020-09-29T10:43:13.383059Z",
     "iopub.status.idle": "2020-09-29T10:43:13.399019Z",
     "shell.execute_reply": "2020-09-29T10:43:13.400313Z"
    },
    "papermill": {
     "duration": 0.136383,
     "end_time": "2020-09-29T10:43:13.400474",
     "exception": false,
     "start_time": "2020-09-29T10:43:13.264091",
     "status": "completed"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1264216, 5)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>Score</th>\n",
       "      <th>Title</th>\n",
       "      <th>Body</th>\n",
       "      <th>Tag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>80</td>\n",
       "      <td>26</td>\n",
       "      <td>SQLStatement.execute() - multiple queries in o...</td>\n",
       "      <td>&lt;p&gt;I've written a database generation script i...</td>\n",
       "      <td>flex actionscript-3 air</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>90</td>\n",
       "      <td>144</td>\n",
       "      <td>Good branching and merging tutorials for Torto...</td>\n",
       "      <td>&lt;p&gt;Are there any really good tutorials explain...</td>\n",
       "      <td>svn tortoisesvn branch branching-and-merging</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>120</td>\n",
       "      <td>21</td>\n",
       "      <td>ASP.NET Site Maps</td>\n",
       "      <td>&lt;p&gt;Has anyone got experience creating &lt;strong&gt;...</td>\n",
       "      <td>sql asp.net sitemap</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>180</td>\n",
       "      <td>53</td>\n",
       "      <td>Function for creating color wheels</td>\n",
       "      <td>&lt;p&gt;This is something I've pseudo-solved many t...</td>\n",
       "      <td>algorithm language-agnostic colors color-space</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>260</td>\n",
       "      <td>49</td>\n",
       "      <td>Adding scripting functionality to .NET applica...</td>\n",
       "      <td>&lt;p&gt;I have a little game written in C#. It uses...</td>\n",
       "      <td>c# .net scripting compiler-construction</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Id  Score                                              Title  \\\n",
       "0   80     26  SQLStatement.execute() - multiple queries in o...   \n",
       "1   90    144  Good branching and merging tutorials for Torto...   \n",
       "2  120     21                                  ASP.NET Site Maps   \n",
       "3  180     53                 Function for creating color wheels   \n",
       "4  260     49  Adding scripting functionality to .NET applica...   \n",
       "\n",
       "                                                Body  \\\n",
       "0  <p>I've written a database generation script i...   \n",
       "1  <p>Are there any really good tutorials explain...   \n",
       "2  <p>Has anyone got experience creating <strong>...   \n",
       "3  <p>This is something I've pseudo-solved many t...   \n",
       "4  <p>I have a little game written in C#. It uses...   \n",
       "\n",
       "                                              Tag  \n",
       "0                         flex actionscript-3 air  \n",
       "1    svn tortoisesvn branch branching-and-merging  \n",
       "2                             sql asp.net sitemap  \n",
       "3  algorithm language-agnostic colors color-space  \n",
       "4         c# .net scripting compiler-construction  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(df.shape)\n",
    "df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-09-29T10:43:14.015279Z",
     "iopub.status.busy": "2020-09-29T10:43:13.582869Z",
     "iopub.status.idle": "2020-09-29T10:43:14.151183Z",
     "shell.execute_reply": "2020-09-29T10:43:14.150672Z"
    },
    "papermill": {
     "duration": 0.660305,
     "end_time": "2020-09-29T10:43:14.151288",
     "exception": false,
     "start_time": "2020-09-29T10:43:13.490983",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Id       0\n",
       "Score    0\n",
       "Title    0\n",
       "Body     0\n",
       "Tag      0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.063312,
     "end_time": "2020-09-29T10:43:14.279243",
     "exception": false,
     "start_time": "2020-09-29T10:43:14.215931",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "All the rows have some Tag associated with it."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "df4a7b2e88af524f4bd778c223d580de76c57cf7",
    "papermill": {
     "duration": 0.07008,
     "end_time": "2020-09-29T10:43:14.413510",
     "exception": false,
     "start_time": "2020-09-29T10:43:14.343430",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Now, I'll take only quesions with score greater than 5 because posts will probably be with a better quality and will be better tagged since they have lots of upvotes. \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "_uuid": "ecebe90606c70e84e4e87e1661e49a11c37e529f",
    "execution": {
     "iopub.execute_input": "2020-09-29T10:43:14.546691Z",
     "iopub.status.busy": "2020-09-29T10:43:14.545938Z",
     "iopub.status.idle": "2020-09-29T10:43:14.804631Z",
     "shell.execute_reply": "2020-09-29T10:43:14.804008Z"
    },
    "papermill": {
     "duration": 0.327046,
     "end_time": "2020-09-29T10:43:14.804740",
     "exception": false,
     "start_time": "2020-09-29T10:43:14.477694",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df = df[df['Score']>5].copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "e462e56ec2d84a0f3810d53bee3715cb570907f1",
    "papermill": {
     "duration": 0.065676,
     "end_time": "2020-09-29T10:43:14.934391",
     "exception": false,
     "start_time": "2020-09-29T10:43:14.868715",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "**1.2 Cleaning Data**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "_uuid": "1550d62d082219a8c8564f8a95afb121d9096c67",
    "execution": {
     "iopub.execute_input": "2020-09-29T10:43:15.069376Z",
     "iopub.status.busy": "2020-09-29T10:43:15.068405Z",
     "iopub.status.idle": "2020-09-29T10:43:15.621763Z",
     "shell.execute_reply": "2020-09-29T10:43:15.621247Z"
    },
    "papermill": {
     "duration": 0.623261,
     "end_time": "2020-09-29T10:43:15.621875",
     "exception": false,
     "start_time": "2020-09-29T10:43:14.998614",
     "status": "completed"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Duplicate entries: 0\n"
     ]
    }
   ],
   "source": [
    "print('Duplicate entries: {}'.format(df.duplicated().sum()))\n",
    "df.drop_duplicates(inplace = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "097a6864ce893217365a3dca22bb68b442746740",
    "papermill": {
     "duration": 0.067475,
     "end_time": "2020-09-29T10:43:15.763892",
     "exception": false,
     "start_time": "2020-09-29T10:43:15.696417",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "This is a very good dataset since there are no missing values or duplicate values. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "_uuid": "6ed21b85891cb57be8273aa41443e6c8198c0d0c",
    "execution": {
     "iopub.execute_input": "2020-09-29T10:43:15.915875Z",
     "iopub.status.busy": "2020-09-29T10:43:15.904881Z",
     "iopub.status.idle": "2020-09-29T10:43:15.919609Z",
     "shell.execute_reply": "2020-09-29T10:43:15.918917Z"
    },
    "papermill": {
     "duration": 0.085919,
     "end_time": "2020-09-29T10:43:15.919730",
     "exception": false,
     "start_time": "2020-09-29T10:43:15.833811",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df.drop(columns=['Id', 'Score'], inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "d3a6dc67414c2faf61fc32adf821d5d6c5a00e4e",
    "papermill": {
     "duration": 0.064259,
     "end_time": "2020-09-29T10:43:16.051853",
     "exception": false,
     "start_time": "2020-09-29T10:43:15.987594",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Tags"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "7d52b54ca515eb7ef5a9545b9f722090abc4c683",
    "papermill": {
     "duration": 0.064374,
     "end_time": "2020-09-29T10:43:16.180755",
     "exception": false,
     "start_time": "2020-09-29T10:43:16.116381",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Let's do some cleaning on the tags' column. Furthermore, I decided to keep the 100 most popular tags because I'll be easier to predict the right tag from 100 words than from 14,000 and because we want to keep macro tags and not be too specific since it's only a recommendation for a post, the user can add more specific tags himself. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-09-29T10:43:16.319807Z",
     "iopub.status.busy": "2020-09-29T10:43:16.319119Z",
     "iopub.status.idle": "2020-09-29T10:43:16.323789Z",
     "shell.execute_reply": "2020-09-29T10:43:16.323226Z"
    },
    "papermill": {
     "duration": 0.077978,
     "end_time": "2020-09-29T10:43:16.323888",
     "exception": false,
     "start_time": "2020-09-29T10:43:16.245910",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "_uuid": "f52f8b1079382f88516dd1df91aeaebe6d4f1dc4",
    "execution": {
     "iopub.execute_input": "2020-09-29T10:43:16.465398Z",
     "iopub.status.busy": "2020-09-29T10:43:16.464617Z",
     "iopub.status.idle": "2020-09-29T10:43:16.469373Z",
     "shell.execute_reply": "2020-09-29T10:43:16.468862Z"
    },
    "papermill": {
     "duration": 0.080612,
     "end_time": "2020-09-29T10:43:16.469491",
     "exception": false,
     "start_time": "2020-09-29T10:43:16.388879",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Title</th>\n",
       "      <th>Body</th>\n",
       "      <th>Tag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>SQLStatement.execute() - multiple queries in o...</td>\n",
       "      <td>&lt;p&gt;I've written a database generation script i...</td>\n",
       "      <td>flex actionscript-3 air</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Good branching and merging tutorials for Torto...</td>\n",
       "      <td>&lt;p&gt;Are there any really good tutorials explain...</td>\n",
       "      <td>svn tortoisesvn branch branching-and-merging</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ASP.NET Site Maps</td>\n",
       "      <td>&lt;p&gt;Has anyone got experience creating &lt;strong&gt;...</td>\n",
       "      <td>sql asp.net sitemap</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Function for creating color wheels</td>\n",
       "      <td>&lt;p&gt;This is something I've pseudo-solved many t...</td>\n",
       "      <td>algorithm language-agnostic colors color-space</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Adding scripting functionality to .NET applica...</td>\n",
       "      <td>&lt;p&gt;I have a little game written in C#. It uses...</td>\n",
       "      <td>c# .net scripting compiler-construction</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               Title  \\\n",
       "0  SQLStatement.execute() - multiple queries in o...   \n",
       "1  Good branching and merging tutorials for Torto...   \n",
       "2                                  ASP.NET Site Maps   \n",
       "3                 Function for creating color wheels   \n",
       "4  Adding scripting functionality to .NET applica...   \n",
       "\n",
       "                                                Body  \\\n",
       "0  <p>I've written a database generation script i...   \n",
       "1  <p>Are there any really good tutorials explain...   \n",
       "2  <p>Has anyone got experience creating <strong>...   \n",
       "3  <p>This is something I've pseudo-solved many t...   \n",
       "4  <p>I have a little game written in C#. It uses...   \n",
       "\n",
       "                                              Tag  \n",
       "0                         flex actionscript-3 air  \n",
       "1    svn tortoisesvn branch branching-and-merging  \n",
       "2                             sql asp.net sitemap  \n",
       "3  algorithm language-agnostic colors color-space  \n",
       "4         c# .net scripting compiler-construction  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "_kg_hide-output": true,
    "_uuid": "d112c6cca7ac7b8fecfafdecfe3c75820a8b6f8c",
    "execution": {
     "iopub.execute_input": "2020-09-29T10:43:16.606996Z",
     "iopub.status.busy": "2020-09-29T10:43:16.606029Z",
     "iopub.status.idle": "2020-09-29T10:43:16.761352Z",
     "shell.execute_reply": "2020-09-29T10:43:16.760769Z"
    },
    "papermill": {
     "duration": 0.226506,
     "end_time": "2020-09-29T10:43:16.761463",
     "exception": false,
     "start_time": "2020-09-29T10:43:16.534957",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df['Tag'] = df['Tag'].apply(lambda x: x.split())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "_uuid": "84505cde455a6d528b08e72688caa6c092893388",
    "execution": {
     "iopub.execute_input": "2020-09-29T10:43:16.914532Z",
     "iopub.status.busy": "2020-09-29T10:43:16.913510Z",
     "iopub.status.idle": "2020-09-29T10:43:16.921874Z",
     "shell.execute_reply": "2020-09-29T10:43:16.922346Z"
    },
    "papermill": {
     "duration": 0.095082,
     "end_time": "2020-09-29T10:43:16.922486",
     "exception": false,
     "start_time": "2020-09-29T10:43:16.827404",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "all_tags = [item for sublist in df['Tag'].values for item in sublist]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "_uuid": "79be7667451d41ac7cd9ef52b7edae19a8c9e846",
    "execution": {
     "iopub.execute_input": "2020-09-29T10:43:17.067328Z",
     "iopub.status.busy": "2020-09-29T10:43:17.065727Z",
     "iopub.status.idle": "2020-09-29T10:43:17.069605Z",
     "shell.execute_reply": "2020-09-29T10:43:17.070106Z"
    },
    "papermill": {
     "duration": 0.076729,
     "end_time": "2020-09-29T10:43:17.070249",
     "exception": false,
     "start_time": "2020-09-29T10:43:16.993520",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "224129"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(all_tags)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "_uuid": "0a7e3c76664f2bbcbab52d1ecbf445e8451d877d",
    "execution": {
     "iopub.execute_input": "2020-09-29T10:43:17.234162Z",
     "iopub.status.busy": "2020-09-29T10:43:17.233279Z",
     "iopub.status.idle": "2020-09-29T10:43:17.236985Z",
     "shell.execute_reply": "2020-09-29T10:43:17.237486Z"
    },
    "papermill": {
     "duration": 0.096701,
     "end_time": "2020-09-29T10:43:17.237623",
     "exception": false,
     "start_time": "2020-09-29T10:43:17.140922",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14883"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_set = set(all_tags)\n",
    "unique_tags = list(my_set)\n",
    "len(unique_tags)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-09-29T10:43:17.402458Z",
     "iopub.status.busy": "2020-09-29T10:43:17.400758Z",
     "iopub.status.idle": "2020-09-29T10:43:17.403256Z",
     "shell.execute_reply": "2020-09-29T10:43:17.403726Z"
    },
    "papermill": {
     "duration": 0.099414,
     "end_time": "2020-09-29T10:43:17.403849",
     "exception": false,
     "start_time": "2020-09-29T10:43:17.304435",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "counts = Counter(all_tags)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-09-29T10:43:17.546893Z",
     "iopub.status.busy": "2020-09-29T10:43:17.546232Z",
     "iopub.status.idle": "2020-09-29T10:43:17.551230Z",
     "shell.execute_reply": "2020-09-29T10:43:17.550740Z"
    },
    "papermill": {
     "duration": 0.080094,
     "end_time": "2020-09-29T10:43:17.551331",
     "exception": false,
     "start_time": "2020-09-29T10:43:17.471237",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('c#', 6907), ('java', 6862), ('javascript', 5560), ('android', 5052), ('python', 4442), ('c++', 4369), ('php', 3024), ('jquery', 2770), ('.net', 2705), ('ios', 2685), ('html', 2129), ('css', 1980), ('c', 1845), ('iphone', 1781), ('objective-c', 1768), ('ruby-on-rails', 1524), ('sql', 1389), ('asp.net', 1302), ('mysql', 1286), ('ruby', 1249), ('r', 1120), ('git', 1013), ('asp.net-mvc', 1004), ('linux', 935), ('sql-server', 907)]\n"
     ]
    }
   ],
   "source": [
    "print(counts.most_common(25))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-09-29T10:43:17.710984Z",
     "iopub.status.busy": "2020-09-29T10:43:17.704133Z",
     "iopub.status.idle": "2020-09-29T10:43:17.719780Z",
     "shell.execute_reply": "2020-09-29T10:43:17.720815Z"
    },
    "papermill": {
     "duration": 0.102682,
     "end_time": "2020-09-29T10:43:17.720996",
     "exception": false,
     "start_time": "2020-09-29T10:43:17.618314",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "65608\n"
     ]
    }
   ],
   "source": [
    "s=0\n",
    "for i in counts.most_common(25):\n",
    "    s+=i[1]\n",
    "print(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "_uuid": "56cecaeb6a313c81202a08fbcca65e33da95b3be",
    "execution": {
     "iopub.execute_input": "2020-09-29T10:43:17.866944Z",
     "iopub.status.busy": "2020-09-29T10:43:17.866270Z",
     "iopub.status.idle": "2020-09-29T10:43:17.869796Z",
     "shell.execute_reply": "2020-09-29T10:43:17.869273Z"
    },
    "papermill": {
     "duration": 0.079828,
     "end_time": "2020-09-29T10:43:17.869892",
     "exception": false,
     "start_time": "2020-09-29T10:43:17.790064",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "frequencies_words = counts.most_common(25)\n",
    "tags_features = [word[0] for word in frequencies_words]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "_uuid": "a97a5e5d2e1031394382558d4dc3b54530400005",
    "execution": {
     "iopub.execute_input": "2020-09-29T10:43:18.056223Z",
     "iopub.status.busy": "2020-09-29T10:43:18.055308Z",
     "iopub.status.idle": "2020-09-29T10:43:18.066498Z",
     "shell.execute_reply": "2020-09-29T10:43:18.067201Z"
    },
    "papermill": {
     "duration": 0.086821,
     "end_time": "2020-09-29T10:43:18.067380",
     "exception": false,
     "start_time": "2020-09-29T10:43:17.980559",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['c#', 'java', 'javascript', 'android', 'python', 'c++', 'php', 'jquery', '.net', 'ios', 'html', 'css', 'c', 'iphone', 'objective-c', 'ruby-on-rails', 'sql', 'asp.net', 'mysql', 'ruby', 'r', 'git', 'asp.net-mvc', 'linux', 'sql-server']\n"
     ]
    }
   ],
   "source": [
    "print(tags_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "_uuid": "3dabce5e92dc9314e7042384add46fc2c31f3112",
    "execution": {
     "iopub.execute_input": "2020-09-29T10:43:18.283781Z",
     "iopub.status.busy": "2020-09-29T10:43:18.282829Z",
     "iopub.status.idle": "2020-09-29T10:43:18.290385Z",
     "shell.execute_reply": "2020-09-29T10:43:18.291474Z"
    },
    "papermill": {
     "duration": 0.121221,
     "end_time": "2020-09-29T10:43:18.291652",
     "exception": false,
     "start_time": "2020-09-29T10:43:18.170431",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def most_common(tags):\n",
    "    tags_filtered = []\n",
    "    for i in range(0, len(tags)):\n",
    "        if tags[i] in tags_features:\n",
    "            tags_filtered.append(tags[i])\n",
    "    return tags_filtered"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "_uuid": "e1b1f8eefaf9b94b5bffdbd295b5770fae7b2984",
    "execution": {
     "iopub.execute_input": "2020-09-29T10:43:18.506927Z",
     "iopub.status.busy": "2020-09-29T10:43:18.506111Z",
     "iopub.status.idle": "2020-09-29T10:43:18.792841Z",
     "shell.execute_reply": "2020-09-29T10:43:18.792206Z"
    },
    "papermill": {
     "duration": 0.4012,
     "end_time": "2020-09-29T10:43:18.792965",
     "exception": false,
     "start_time": "2020-09-29T10:43:18.391765",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df['Tag'] = df['Tag'].apply(lambda x: most_common(x))\n",
    "df['Tag'] = df['Tag'].apply(lambda x: x if len(x)>0 else None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-09-29T10:43:18.943147Z",
     "iopub.status.busy": "2020-09-29T10:43:18.942481Z",
     "iopub.status.idle": "2020-09-29T10:43:18.948770Z",
     "shell.execute_reply": "2020-09-29T10:43:18.948254Z"
    },
    "papermill": {
     "duration": 0.087192,
     "end_time": "2020-09-29T10:43:18.948870",
     "exception": false,
     "start_time": "2020-09-29T10:43:18.861678",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Title</th>\n",
       "      <th>Body</th>\n",
       "      <th>Tag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>SQLStatement.execute() - multiple queries in o...</td>\n",
       "      <td>&lt;p&gt;I've written a database generation script i...</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Good branching and merging tutorials for Torto...</td>\n",
       "      <td>&lt;p&gt;Are there any really good tutorials explain...</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ASP.NET Site Maps</td>\n",
       "      <td>&lt;p&gt;Has anyone got experience creating &lt;strong&gt;...</td>\n",
       "      <td>[sql, asp.net]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Function for creating color wheels</td>\n",
       "      <td>&lt;p&gt;This is something I've pseudo-solved many t...</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Adding scripting functionality to .NET applica...</td>\n",
       "      <td>&lt;p&gt;I have a little game written in C#. It uses...</td>\n",
       "      <td>[c#, .net]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               Title  \\\n",
       "0  SQLStatement.execute() - multiple queries in o...   \n",
       "1  Good branching and merging tutorials for Torto...   \n",
       "2                                  ASP.NET Site Maps   \n",
       "3                 Function for creating color wheels   \n",
       "4  Adding scripting functionality to .NET applica...   \n",
       "\n",
       "                                                Body             Tag  \n",
       "0  <p>I've written a database generation script i...            None  \n",
       "1  <p>Are there any really good tutorials explain...            None  \n",
       "2  <p>Has anyone got experience creating <strong>...  [sql, asp.net]  \n",
       "3  <p>This is something I've pseudo-solved many t...            None  \n",
       "4  <p>I have a little game written in C#. It uses...      [c#, .net]  "
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "_uuid": "1f9ffd9e9e0dfe7c2e4c595e31559e2f5ec513f4",
    "execution": {
     "iopub.execute_input": "2020-09-29T10:43:19.102183Z",
     "iopub.status.busy": "2020-09-29T10:43:19.101199Z",
     "iopub.status.idle": "2020-09-29T10:43:19.111850Z",
     "shell.execute_reply": "2020-09-29T10:43:19.111327Z"
    },
    "papermill": {
     "duration": 0.093339,
     "end_time": "2020-09-29T10:43:19.111956",
     "exception": false,
     "start_time": "2020-09-29T10:43:19.018617",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df.dropna(subset=['Tag'], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "_uuid": "608e36814e0b2befb41bb9dc6eca073edf48fbe4",
    "execution": {
     "iopub.execute_input": "2020-09-29T10:43:19.259414Z",
     "iopub.status.busy": "2020-09-29T10:43:19.257986Z",
     "iopub.status.idle": "2020-09-29T10:43:19.261931Z",
     "shell.execute_reply": "2020-09-29T10:43:19.261431Z"
    },
    "papermill": {
     "duration": 0.079148,
     "end_time": "2020-09-29T10:43:19.262028",
     "exception": false,
     "start_time": "2020-09-29T10:43:19.182880",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(52418, 3)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "0240dc179aeabfdd8bbd6638d0e1b3ae5cb56877",
    "papermill": {
     "duration": 0.070429,
     "end_time": "2020-09-29T10:43:19.408315",
     "exception": false,
     "start_time": "2020-09-29T10:43:19.337886",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "**1.2.2 Body**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "598869f86fd4f9a6fe1e41f9acce43d392cb50aa",
    "papermill": {
     "duration": 0.070895,
     "end_time": "2020-09-29T10:43:19.554068",
     "exception": false,
     "start_time": "2020-09-29T10:43:19.483173",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "In the next two columns: Body and Title, I'll use lots of text processing:\n",
    "* Removing html format \n",
    "* Lowering text\n",
    "* Transforming abbreviations \n",
    "* Removing punctuation (but keeping words like c# since it's the most popular tag)\n",
    "* Lemmatizing words\n",
    "* Removing stop words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-09-29T10:43:19.710007Z",
     "iopub.status.busy": "2020-09-29T10:43:19.709123Z",
     "iopub.status.idle": "2020-09-29T10:43:30.925529Z",
     "shell.execute_reply": "2020-09-29T10:43:30.924892Z"
    },
    "papermill": {
     "duration": 11.301758,
     "end_time": "2020-09-29T10:43:30.925641",
     "exception": false,
     "start_time": "2020-09-29T10:43:19.623883",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting git+https://github.com/laxmimerit/preprocess_kgptalkie.git\r\n",
      "  Cloning https://github.com/laxmimerit/preprocess_kgptalkie.git to /tmp/pip-req-build-qy6zuiyh\r\n",
      "Building wheels for collected packages: preprocess-kgptalkie\r\n",
      "  Building wheel for preprocess-kgptalkie (setup.py) ... \u001b[?25l-\b \b\\\b \bdone\r\n",
      "\u001b[?25h  Created wheel for preprocess-kgptalkie: filename=preprocess_kgptalkie-0.1.0-py3-none-any.whl size=11713 sha256=3f762f494990aae906c8666b335f82459bf67d796118ed1a0176a2bcdc53f179\r\n",
      "  Stored in directory: /tmp/pip-ephem-wheel-cache-dizmbfct/wheels/0d/b3/29/bfe3deffda68980088d17b81331be6667e837ffb4a071bae82\r\n",
      "Successfully built preprocess-kgptalkie\r\n",
      "Installing collected packages: preprocess-kgptalkie\r\n",
      "Successfully installed preprocess-kgptalkie-0.1.0\r\n"
     ]
    }
   ],
   "source": [
    "!pip install git+https://github.com/laxmimerit/preprocess_kgptalkie.git"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-09-29T10:43:31.078122Z",
     "iopub.status.busy": "2020-09-29T10:43:31.077217Z",
     "iopub.status.idle": "2020-09-29T10:43:34.310347Z",
     "shell.execute_reply": "2020-09-29T10:43:34.311140Z"
    },
    "papermill": {
     "duration": 3.31201,
     "end_time": "2020-09-29T10:43:34.311335",
     "exception": false,
     "start_time": "2020-09-29T10:43:30.999325",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import preprocess_kgptalkie as ps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-09-29T10:43:34.474052Z",
     "iopub.status.busy": "2020-09-29T10:43:34.473361Z",
     "iopub.status.idle": "2020-09-29T10:43:34.476983Z",
     "shell.execute_reply": "2020-09-29T10:43:34.476457Z"
    },
    "papermill": {
     "duration": 0.084802,
     "end_time": "2020-09-29T10:43:34.477095",
     "exception": false,
     "start_time": "2020-09-29T10:43:34.392293",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def get_clean(x):\n",
    "    x = str(x).lower().replace('\\\\', '').replace('_', ' ')\n",
    "    x = ps.cont_exp(x)\n",
    "    x = ps.remove_emails(x)\n",
    "    x = ps.remove_urls(x)\n",
    "    x = ps.remove_html_tags(x)\n",
    "    x = ps.remove_accented_chars(x)\n",
    "    x = ps.remove_special_chars(x)\n",
    "    x = re.sub(\"(.)\\\\1{2,}\", \"\\\\1\", x)\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-09-29T10:43:34.675470Z",
     "iopub.status.busy": "2020-09-29T10:43:34.674578Z",
     "iopub.status.idle": "2020-09-29T10:45:05.529652Z",
     "shell.execute_reply": "2020-09-29T10:45:05.530194Z"
    },
    "papermill": {
     "duration": 90.980512,
     "end_time": "2020-09-29T10:45:05.530352",
     "exception": false,
     "start_time": "2020-09-29T10:43:34.549840",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df['Body'] = df['Body'].apply(lambda x: get_clean(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-09-29T10:45:05.714761Z",
     "iopub.status.busy": "2020-09-29T10:45:05.713808Z",
     "iopub.status.idle": "2020-09-29T10:45:38.929266Z",
     "shell.execute_reply": "2020-09-29T10:45:38.928681Z"
    },
    "papermill": {
     "duration": 33.321149,
     "end_time": "2020-09-29T10:45:38.929396",
     "exception": false,
     "start_time": "2020-09-29T10:45:05.608247",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df['Title'] = df['Title'].apply(lambda x: get_clean(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-09-29T10:45:39.144191Z",
     "iopub.status.busy": "2020-09-29T10:45:39.141436Z",
     "iopub.status.idle": "2020-09-29T10:45:39.146986Z",
     "shell.execute_reply": "2020-09-29T10:45:39.146505Z"
    },
    "papermill": {
     "duration": 0.143774,
     "end_time": "2020-09-29T10:45:39.147113",
     "exception": false,
     "start_time": "2020-09-29T10:45:39.003339",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df['Text'] = df['Title'] + \" \" +  df['Body']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-09-29T10:45:39.310503Z",
     "iopub.status.busy": "2020-09-29T10:45:39.309533Z",
     "iopub.status.idle": "2020-09-29T10:45:39.314028Z",
     "shell.execute_reply": "2020-09-29T10:45:39.313512Z"
    },
    "papermill": {
     "duration": 0.092316,
     "end_time": "2020-09-29T10:45:39.314148",
     "exception": false,
     "start_time": "2020-09-29T10:45:39.221832",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Title</th>\n",
       "      <th>Body</th>\n",
       "      <th>Tag</th>\n",
       "      <th>Text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>aspnet site mapostscripoint</td>\n",
       "      <td>has anyone got experetweethat isnce creating s...</td>\n",
       "      <td>[sql, asp.net]</td>\n",
       "      <td>aspnet site mapostscripoint has anyone got exp...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>adding scripointing fyounctionality to net apa...</td>\n",
       "      <td>i have a little game written in c it youses a ...</td>\n",
       "      <td>[c#, .net]</td>\n",
       "      <td>adding scripointing fyounctionality to net apa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>shoyould i youse nested classes in this case</td>\n",
       "      <td>i am working on a collection of classes yoused...</td>\n",
       "      <td>[c++]</td>\n",
       "      <td>shoyould i youse nested classes in this case i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>homexamplerown consyoumpointion of web servi ses</td>\n",
       "      <td>i have been writing a few web servi ses for a ...</td>\n",
       "      <td>[.net]</td>\n",
       "      <td>homexamplerown consyoumpointion of web servi s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>deploying sqyouarel server databases from test...</td>\n",
       "      <td>i wonder how yoyou gyouys manage deployment of...</td>\n",
       "      <td>[sql-server]</td>\n",
       "      <td>deploying sqyouarel server databases from test...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               Title  \\\n",
       "2                        aspnet site mapostscripoint   \n",
       "4  adding scripointing fyounctionality to net apa...   \n",
       "5       shoyould i youse nested classes in this case   \n",
       "6   homexamplerown consyoumpointion of web servi ses   \n",
       "7  deploying sqyouarel server databases from test...   \n",
       "\n",
       "                                                Body             Tag  \\\n",
       "2  has anyone got experetweethat isnce creating s...  [sql, asp.net]   \n",
       "4  i have a little game written in c it youses a ...      [c#, .net]   \n",
       "5  i am working on a collection of classes yoused...           [c++]   \n",
       "6  i have been writing a few web servi ses for a ...          [.net]   \n",
       "7  i wonder how yoyou gyouys manage deployment of...    [sql-server]   \n",
       "\n",
       "                                                Text  \n",
       "2  aspnet site mapostscripoint has anyone got exp...  \n",
       "4  adding scripointing fyounctionality to net apa...  \n",
       "5  shoyould i youse nested classes in this case i...  \n",
       "6  homexamplerown consyoumpointion of web servi s...  \n",
       "7  deploying sqyouarel server databases from test...  "
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "119c8c35293cccd9627b58544acaaf06b4cb32d2",
    "papermill": {
     "duration": 0.074386,
     "end_time": "2020-09-29T10:45:39.469503",
     "exception": false,
     "start_time": "2020-09-29T10:45:39.395117",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "**PART 2: Classical classifiers**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "4d7d89ca6323645d6bf722bd572852edc2b9571c",
    "papermill": {
     "duration": 0.073935,
     "end_time": "2020-09-29T10:45:39.633829",
     "exception": false,
     "start_time": "2020-09-29T10:45:39.559894",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "**2.1 Data preparation**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "_uuid": "8640de349b18ea297f15c3d3eed3836d3ed19e14",
    "execution": {
     "iopub.execute_input": "2020-09-29T10:45:39.787515Z",
     "iopub.status.busy": "2020-09-29T10:45:39.786832Z",
     "iopub.status.idle": "2020-09-29T10:45:39.791156Z",
     "shell.execute_reply": "2020-09-29T10:45:39.790653Z"
    },
    "papermill": {
     "duration": 0.083131,
     "end_time": "2020-09-29T10:45:39.791262",
     "exception": false,
     "start_time": "2020-09-29T10:45:39.708131",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "y = df['Tag']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "_uuid": "a4669f647cb5a147710294f1116ac4e2f29f50d7",
    "execution": {
     "iopub.execute_input": "2020-09-29T10:45:39.985715Z",
     "iopub.status.busy": "2020-09-29T10:45:39.970402Z",
     "iopub.status.idle": "2020-09-29T10:45:40.034095Z",
     "shell.execute_reply": "2020-09-29T10:45:40.033563Z"
    },
    "papermill": {
     "duration": 0.168482,
     "end_time": "2020-09-29T10:45:40.034209",
     "exception": false,
     "start_time": "2020-09-29T10:45:39.865727",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "multilabel = MultiLabelBinarizer()\n",
    "y = multilabel.fit_transform(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-09-29T10:45:40.188972Z",
     "iopub.status.busy": "2020-09-29T10:45:40.188179Z",
     "iopub.status.idle": "2020-09-29T10:45:40.191982Z",
     "shell.execute_reply": "2020-09-29T10:45:40.191518Z"
    },
    "papermill": {
     "duration": 0.0832,
     "end_time": "2020-09-29T10:45:40.192103",
     "exception": false,
     "start_time": "2020-09-29T10:45:40.108903",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(52418, 25)"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "_uuid": "e8fd01469710da8a69fc45de00d37901d570814a",
    "execution": {
     "iopub.execute_input": "2020-09-29T10:45:40.386284Z",
     "iopub.status.busy": "2020-09-29T10:45:40.370849Z",
     "iopub.status.idle": "2020-09-29T10:45:51.887186Z",
     "shell.execute_reply": "2020-09-29T10:45:51.885882Z"
    },
    "papermill": {
     "duration": 11.619932,
     "end_time": "2020-09-29T10:45:51.887304",
     "exception": false,
     "start_time": "2020-09-29T10:45:40.267372",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "tfidf = TfidfVectorizer(analyzer = 'word', max_features=1000)\n",
    "X = tfidf.fit_transform(df['Text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "_uuid": "1219d6cdc1f1d27c90910659f1be21a917f9288e",
    "execution": {
     "iopub.execute_input": "2020-09-29T10:45:52.045289Z",
     "iopub.status.busy": "2020-09-29T10:45:52.043351Z",
     "iopub.status.idle": "2020-09-29T10:45:52.066079Z",
     "shell.execute_reply": "2020-09-29T10:45:52.065562Z"
    },
    "papermill": {
     "duration": 0.103296,
     "end_time": "2020-09-29T10:45:52.066205",
     "exception": false,
     "start_time": "2020-09-29T10:45:51.962909",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.3, random_state = 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "6755c0843211713e0b4bf90cb4618cfdaa9fef85",
    "papermill": {
     "duration": 0.075812,
     "end_time": "2020-09-29T10:45:52.217379",
     "exception": false,
     "start_time": "2020-09-29T10:45:52.141567",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "**2.2 One vs Rest**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "c4448315a460e141e1c8b079b809c1040e4fc050",
    "papermill": {
     "duration": 0.074904,
     "end_time": "2020-09-29T10:45:52.367879",
     "exception": false,
     "start_time": "2020-09-29T10:45:52.292975",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "To evaluate our models, I'll use the jacard score since it's the best fitted for multi label classification. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "_kg_hide-input": true,
    "_uuid": "03e10593770e8b27cb2a9574efcc9cd8b6dabbd6",
    "execution": {
     "iopub.execute_input": "2020-09-29T10:45:52.527733Z",
     "iopub.status.busy": "2020-09-29T10:45:52.526829Z",
     "iopub.status.idle": "2020-09-29T10:45:52.529741Z",
     "shell.execute_reply": "2020-09-29T10:45:52.529244Z"
    },
    "papermill": {
     "duration": 0.085983,
     "end_time": "2020-09-29T10:45:52.529843",
     "exception": false,
     "start_time": "2020-09-29T10:45:52.443860",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def avg_jacard(y_true,y_pred):\n",
    "    \n",
    "    jacard = np.minimum(y_true,y_pred).sum(axis=1) / np.maximum(y_true,y_pred).sum(axis=1)\n",
    "    \n",
    "    return jacard.mean()*100\n",
    "\n",
    "def print_score(y_pred, clf):\n",
    "    print(\"Clf: \", clf.__class__.__name__)\n",
    "    print(\"Jacard score: {}\".format(avg_jacard(y_test, y_pred)))\n",
    "    print(\"Hamming loss: {}\".format(hamming_loss(y_pred, y_test)*100))\n",
    "    print(\"---\")    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "_kg_hide-input": true,
    "_uuid": "6fb3326bc2c812cd991ca969762e54190b548150",
    "execution": {
     "iopub.execute_input": "2020-09-29T10:45:52.688672Z",
     "iopub.status.busy": "2020-09-29T10:45:52.687667Z",
     "iopub.status.idle": "2020-09-29T10:46:19.607703Z",
     "shell.execute_reply": "2020-09-29T10:46:19.608682Z"
    },
    "papermill": {
     "duration": 27.003395,
     "end_time": "2020-09-29T10:46:19.608857",
     "exception": false,
     "start_time": "2020-09-29T10:45:52.605462",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Clf:  SGDClassifier\n",
      "Jacard score: 42.20123786510661\n",
      "Hamming loss: 3.246343634745008\n",
      "---\n",
      "Clf:  LogisticRegression\n",
      "Jacard score: 44.32192971300183\n",
      "Hamming loss: 3.220653694518632\n",
      "---\n",
      "Clf:  LinearSVC\n",
      "Jacard score: 50.1352325236339\n",
      "Hamming loss: 3.0281063207427192\n",
      "---\n"
     ]
    }
   ],
   "source": [
    "sgd = SGDClassifier()\n",
    "lr = LogisticRegression()\n",
    "svc = LinearSVC()\n",
    "rf = RandomForestClassifier()\n",
    "\n",
    "for classifier in [sgd, lr, svc]:\n",
    "    clf = OneVsRestClassifier(classifier)\n",
    "    clf.fit(X_train, y_train)\n",
    "    y_pred = clf.predict(X_test)\n",
    "    print_score(y_pred, classifier)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "c14aca676e8fe5e65beba7b40ce7d6295e8a4242",
    "papermill": {
     "duration": 0.076023,
     "end_time": "2020-09-29T10:46:19.761952",
     "exception": false,
     "start_time": "2020-09-29T10:46:19.685929",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "**2.6 Confusion matrix**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "_kg_hide-output": true,
    "_uuid": "444d6f85d6c9c121c3ecf4654cb7a37e21974321",
    "execution": {
     "iopub.execute_input": "2020-09-29T10:46:19.923200Z",
     "iopub.status.busy": "2020-09-29T10:46:19.922264Z",
     "iopub.status.idle": "2020-09-29T10:46:20.568909Z",
     "shell.execute_reply": "2020-09-29T10:46:20.570069Z"
    },
    "papermill": {
     "duration": 0.731377,
     "end_time": "2020-09-29T10:46:20.570248",
     "exception": false,
     "start_time": "2020-09-29T10:46:19.838871",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ".net\n",
      "[[14861    52]\n",
      " [  685   128]]\n",
      "\n",
      "android\n",
      "[[14166    34]\n",
      " [  322  1204]]\n",
      "\n",
      "asp.net\n",
      "[[15325    34]\n",
      " [  250   117]]\n",
      "\n",
      "asp.net-mvc\n",
      "[[15392    46]\n",
      " [  139   149]]\n",
      "\n",
      "c\n",
      "[[15080    74]\n",
      " [  438   134]]\n",
      "\n",
      "c#\n",
      "[[13331   331]\n",
      " [ 1325   739]]\n",
      "\n",
      "c++\n",
      "[[14259   171]\n",
      " [  773   523]]\n",
      "\n",
      "css\n",
      "[[15030    60]\n",
      " [  234   402]]\n",
      "\n",
      "git\n",
      "[[15398    10]\n",
      " [   46   272]]\n",
      "\n",
      "html\n",
      "[[14945   132]\n",
      " [  428   221]]\n",
      "\n",
      "ios\n",
      "[[14808   102]\n",
      " [  401   415]]\n",
      "\n",
      "iphone\n",
      "[[15137    60]\n",
      " [  382   147]]\n",
      "\n",
      "java\n",
      "[[13496   151]\n",
      " [  887  1192]]\n",
      "\n",
      "javascript\n",
      "[[13875   212]\n",
      " [  778   861]]\n",
      "\n",
      "jquery\n",
      "[[14826    66]\n",
      " [  295   539]]\n",
      "\n",
      "linux\n",
      "[[15408    39]\n",
      " [  181    98]]\n",
      "\n",
      "mysql\n",
      "[[15307    27]\n",
      " [  143   249]]\n",
      "\n",
      "objective-c\n",
      "[[15146    63]\n",
      " [  440    77]]\n",
      "\n",
      "php\n",
      "[[14740    69]\n",
      " [  324   593]]\n",
      "\n",
      "python\n",
      "[[14309    66]\n",
      " [  400   951]]\n",
      "\n",
      "r\n",
      "[[15353    37]\n",
      " [  222   114]]\n",
      "\n",
      "ruby\n",
      "[[15311    30]\n",
      " [  218   167]]\n",
      "\n",
      "ruby-on-rails\n",
      "[[15246    25]\n",
      " [  167   288]]\n",
      "\n",
      "sql\n",
      "[[15231    75]\n",
      " [  270   150]]\n",
      "\n",
      "sql-server\n",
      "[[15430    56]\n",
      " [  135   105]]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for i in range(y_train.shape[1]):\n",
    "    print(multilabel.classes_[i])\n",
    "    print(confusion_matrix(y_test[:,i], y_pred[:,i]))\n",
    "    print(\"\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.11973,
     "end_time": "2020-09-29T10:46:20.821150",
     "exception": false,
     "start_time": "2020-09-29T10:46:20.701420",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "#### Data Store"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-09-29T10:46:21.087728Z",
     "iopub.status.busy": "2020-09-29T10:46:21.086987Z",
     "iopub.status.idle": "2020-09-29T10:46:23.742014Z",
     "shell.execute_reply": "2020-09-29T10:46:23.741086Z"
    },
    "papermill": {
     "duration": 2.791864,
     "end_time": "2020-09-29T10:46:23.742173",
     "exception": false,
     "start_time": "2020-09-29T10:46:20.950309",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df[['Text', 'Tag']].to_csv('stackoverflow.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.07811,
     "end_time": "2020-09-29T10:46:23.898094",
     "exception": false,
     "start_time": "2020-09-29T10:46:23.819984",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Deep Learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-09-29T10:46:24.061283Z",
     "iopub.status.busy": "2020-09-29T10:46:24.059519Z",
     "iopub.status.idle": "2020-09-29T10:46:24.061930Z",
     "shell.execute_reply": "2020-09-29T10:46:24.062392Z"
    },
    "papermill": {
     "duration": 0.086033,
     "end_time": "2020-09-29T10:46:24.062536",
     "exception": false,
     "start_time": "2020-09-29T10:46:23.976503",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.preprocessing import MultiLabelBinarizer\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import hamming_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-09-29T10:46:24.224968Z",
     "iopub.status.busy": "2020-09-29T10:46:24.224187Z",
     "iopub.status.idle": "2020-09-29T10:46:29.217500Z",
     "shell.execute_reply": "2020-09-29T10:46:29.216594Z"
    },
    "papermill": {
     "duration": 5.077138,
     "end_time": "2020-09-29T10:46:29.217626",
     "exception": false,
     "start_time": "2020-09-29T10:46:24.140488",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras.preprocessing.text import Tokenizer\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense,Flatten,Embedding,Activation, Dropout\n",
    "from tensorflow.keras.layers import Conv1D, MaxPooling1D, GlobalMaxPooling1D "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-09-29T10:46:29.381456Z",
     "iopub.status.busy": "2020-09-29T10:46:29.379643Z",
     "iopub.status.idle": "2020-09-29T10:46:29.382210Z",
     "shell.execute_reply": "2020-09-29T10:46:29.382674Z"
    },
    "papermill": {
     "duration": 0.085669,
     "end_time": "2020-09-29T10:46:29.382789",
     "exception": false,
     "start_time": "2020-09-29T10:46:29.297120",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "import ast"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-09-29T10:46:29.546567Z",
     "iopub.status.busy": "2020-09-29T10:46:29.545704Z",
     "iopub.status.idle": "2020-09-29T10:46:30.125577Z",
     "shell.execute_reply": "2020-09-29T10:46:30.125023Z"
    },
    "papermill": {
     "duration": 0.664543,
     "end_time": "2020-09-29T10:46:30.125690",
     "exception": false,
     "start_time": "2020-09-29T10:46:29.461147",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv('./stackoverflow.csv', index_col = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-09-29T10:46:30.295724Z",
     "iopub.status.busy": "2020-09-29T10:46:30.294945Z",
     "iopub.status.idle": "2020-09-29T10:46:30.298158Z",
     "shell.execute_reply": "2020-09-29T10:46:30.298634Z"
    },
    "papermill": {
     "duration": 0.0943,
     "end_time": "2020-09-29T10:46:30.298759",
     "exception": false,
     "start_time": "2020-09-29T10:46:30.204459",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "      <th>Tag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>aspnet site mapostscripoint has anyone got exp...</td>\n",
       "      <td>['sql', 'asp.net']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>adding scripointing fyounctionality to net apa...</td>\n",
       "      <td>['c#', '.net']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>shoyould i youse nested classes in this case i...</td>\n",
       "      <td>['c++']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>homexamplerown consyoumpointion of web servi s...</td>\n",
       "      <td>['.net']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>deploying sqyouarel server databases from test...</td>\n",
       "      <td>['sql-server']</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                Text                 Tag\n",
       "2  aspnet site mapostscripoint has anyone got exp...  ['sql', 'asp.net']\n",
       "4  adding scripointing fyounctionality to net apa...      ['c#', '.net']\n",
       "5  shoyould i youse nested classes in this case i...             ['c++']\n",
       "6  homexamplerown consyoumpointion of web servi s...            ['.net']\n",
       "7  deploying sqyouarel server databases from test...      ['sql-server']"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-09-29T10:46:30.474310Z",
     "iopub.status.busy": "2020-09-29T10:46:30.464095Z",
     "iopub.status.idle": "2020-09-29T10:46:30.991265Z",
     "shell.execute_reply": "2020-09-29T10:46:30.990697Z"
    },
    "papermill": {
     "duration": 0.611711,
     "end_time": "2020-09-29T10:46:30.991370",
     "exception": false,
     "start_time": "2020-09-29T10:46:30.379659",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df['Tag'] = df['Tag'].apply(lambda x: ast.literal_eval(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-09-29T10:46:31.158767Z",
     "iopub.status.busy": "2020-09-29T10:46:31.157868Z",
     "iopub.status.idle": "2020-09-29T10:46:31.162944Z",
     "shell.execute_reply": "2020-09-29T10:46:31.162472Z"
    },
    "papermill": {
     "duration": 0.092073,
     "end_time": "2020-09-29T10:46:31.163062",
     "exception": false,
     "start_time": "2020-09-29T10:46:31.070989",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2          [sql, asp.net]\n",
       "4              [c#, .net]\n",
       "5                   [c++]\n",
       "6                  [.net]\n",
       "7            [sql-server]\n",
       "                ...      \n",
       "1262668             [c++]\n",
       "1262834             [c++]\n",
       "1262915          [python]\n",
       "1263065          [python]\n",
       "1263454             [c++]\n",
       "Name: Tag, Length: 52418, dtype: object"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Tag']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-09-29T10:46:31.328997Z",
     "iopub.status.busy": "2020-09-29T10:46:31.328377Z",
     "iopub.status.idle": "2020-09-29T10:46:31.333726Z",
     "shell.execute_reply": "2020-09-29T10:46:31.334234Z"
    },
    "papermill": {
     "duration": 0.090736,
     "end_time": "2020-09-29T10:46:31.334360",
     "exception": false,
     "start_time": "2020-09-29T10:46:31.243624",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['sql', 'asp.net']"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Tag'].iloc[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-09-29T10:46:31.599057Z",
     "iopub.status.busy": "2020-09-29T10:46:31.578709Z",
     "iopub.status.idle": "2020-09-29T10:46:31.711683Z",
     "shell.execute_reply": "2020-09-29T10:46:31.712344Z"
    },
    "papermill": {
     "duration": 0.270689,
     "end_time": "2020-09-29T10:46:31.712545",
     "exception": false,
     "start_time": "2020-09-29T10:46:31.441856",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "multilabel = MultiLabelBinarizer()\n",
    "y = multilabel.fit_transform(df['Tag'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-09-29T10:46:31.997717Z",
     "iopub.status.busy": "2020-09-29T10:46:31.996797Z",
     "iopub.status.idle": "2020-09-29T10:46:32.000438Z",
     "shell.execute_reply": "2020-09-29T10:46:31.998466Z"
    },
    "papermill": {
     "duration": 0.172934,
     "end_time": "2020-09-29T10:46:32.000561",
     "exception": false,
     "start_time": "2020-09-29T10:46:31.827627",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['.net', 'android', 'asp.net', 'asp.net-mvc', 'c', 'c#', 'c++',\n",
       "       'css', 'git', 'html', 'ios', 'iphone', 'java', 'javascript',\n",
       "       'jquery', 'linux', 'mysql', 'objective-c', 'php', 'python', 'r',\n",
       "       'ruby', 'ruby-on-rails', 'sql', 'sql-server'], dtype=object)"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "multilabel.classes_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-09-29T10:46:32.218513Z",
     "iopub.status.busy": "2020-09-29T10:46:32.217755Z",
     "iopub.status.idle": "2020-09-29T10:46:32.222680Z",
     "shell.execute_reply": "2020-09-29T10:46:32.223294Z"
    },
    "papermill": {
     "duration": 0.113667,
     "end_time": "2020-09-29T10:46:32.223456",
     "exception": false,
     "start_time": "2020-09-29T10:46:32.109789",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "text = df['Text'].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-09-29T10:46:32.451138Z",
     "iopub.status.busy": "2020-09-29T10:46:32.450389Z",
     "iopub.status.idle": "2020-09-29T10:46:40.391431Z",
     "shell.execute_reply": "2020-09-29T10:46:40.390813Z"
    },
    "papermill": {
     "duration": 8.059171,
     "end_time": "2020-09-29T10:46:40.391546",
     "exception": false,
     "start_time": "2020-09-29T10:46:32.332375",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "token = Tokenizer()\n",
    "token.fit_on_texts(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-09-29T10:46:40.560461Z",
     "iopub.status.busy": "2020-09-29T10:46:40.559582Z",
     "iopub.status.idle": "2020-09-29T10:46:40.563599Z",
     "shell.execute_reply": "2020-09-29T10:46:40.563095Z"
    },
    "papermill": {
     "duration": 0.090619,
     "end_time": "2020-09-29T10:46:40.563702",
     "exception": false,
     "start_time": "2020-09-29T10:46:40.473083",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 0, 1, ..., 0, 1, 0],\n",
       "       [1, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       ...,\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0]])"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-09-29T10:46:40.732453Z",
     "iopub.status.busy": "2020-09-29T10:46:40.731799Z",
     "iopub.status.idle": "2020-09-29T10:46:40.737260Z",
     "shell.execute_reply": "2020-09-29T10:46:40.736692Z"
    },
    "papermill": {
     "duration": 0.091861,
     "end_time": "2020-09-29T10:46:40.737372",
     "exception": false,
     "start_time": "2020-09-29T10:46:40.645511",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(52418, 25)"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-09-29T10:46:40.908028Z",
     "iopub.status.busy": "2020-09-29T10:46:40.907259Z",
     "iopub.status.idle": "2020-09-29T10:46:40.911117Z",
     "shell.execute_reply": "2020-09-29T10:46:40.910455Z"
    },
    "papermill": {
     "duration": 0.091954,
     "end_time": "2020-09-29T10:46:40.911254",
     "exception": false,
     "start_time": "2020-09-29T10:46:40.819300",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "572005"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(token.word_counts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-09-29T10:46:41.082907Z",
     "iopub.status.busy": "2020-09-29T10:46:41.082111Z",
     "iopub.status.idle": "2020-09-29T10:46:41.085894Z",
     "shell.execute_reply": "2020-09-29T10:46:41.085424Z"
    },
    "papermill": {
     "duration": 0.092063,
     "end_time": "2020-09-29T10:46:41.085993",
     "exception": false,
     "start_time": "2020-09-29T10:46:40.993930",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "572006"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab_size = len(token.word_index) + 1 #https://keras.io/api/layers/core_layers/embedding/\n",
    "vocab_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-09-29T10:46:41.297294Z",
     "iopub.status.busy": "2020-09-29T10:46:41.287001Z",
     "iopub.status.idle": "2020-09-29T10:46:47.620200Z",
     "shell.execute_reply": "2020-09-29T10:46:47.619630Z"
    },
    "papermill": {
     "duration": 6.450736,
     "end_time": "2020-09-29T10:46:47.620317",
     "exception": false,
     "start_time": "2020-09-29T10:46:41.169581",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "encoded_text = token.texts_to_sequences(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-09-29T10:46:47.807729Z",
     "iopub.status.busy": "2020-09-29T10:46:47.806259Z",
     "iopub.status.idle": "2020-09-29T10:46:48.931232Z",
     "shell.execute_reply": "2020-09-29T10:46:48.930654Z"
    },
    "papermill": {
     "duration": 1.226171,
     "end_time": "2020-09-29T10:46:48.931338",
     "exception": false,
     "start_time": "2020-09-29T10:46:47.705167",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "max_length = 100\n",
    "X = pad_sequences(encoded_text, maxlen=max_length, padding='post')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-09-29T10:46:49.105731Z",
     "iopub.status.busy": "2020-09-29T10:46:49.104852Z",
     "iopub.status.idle": "2020-09-29T10:46:49.110055Z",
     "shell.execute_reply": "2020-09-29T10:46:49.109544Z"
    },
    "papermill": {
     "duration": 0.094213,
     "end_time": "2020-09-29T10:46:49.110153",
     "exception": false,
     "start_time": "2020-09-29T10:46:49.015940",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((52418, 100), (52418, 25))"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape, y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-09-29T10:46:49.289795Z",
     "iopub.status.busy": "2020-09-29T10:46:49.288798Z",
     "iopub.status.idle": "2020-09-29T10:46:49.303199Z",
     "shell.execute_reply": "2020-09-29T10:46:49.302657Z"
    },
    "papermill": {
     "duration": 0.106911,
     "end_time": "2020-09-29T10:46:49.303303",
     "exception": false,
     "start_time": "2020-09-29T10:46:49.196392",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state = 42, test_size = 0.3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.084971,
     "end_time": "2020-09-29T10:46:49.474137",
     "exception": false,
     "start_time": "2020-09-29T10:46:49.389166",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Model Building"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-09-29T10:46:49.650332Z",
     "iopub.status.busy": "2020-09-29T10:46:49.649398Z",
     "iopub.status.idle": "2020-09-29T10:46:49.652302Z",
     "shell.execute_reply": "2020-09-29T10:46:49.651801Z"
    },
    "papermill": {
     "duration": 0.092922,
     "end_time": "2020-09-29T10:46:49.652406",
     "exception": false,
     "start_time": "2020-09-29T10:46:49.559484",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras.optimizers import Adam\n",
    "from tensorflow.keras.callbacks import EarlyStopping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-09-29T10:46:49.833742Z",
     "iopub.status.busy": "2020-09-29T10:46:49.832834Z",
     "iopub.status.idle": "2020-09-29T10:46:49.835683Z",
     "shell.execute_reply": "2020-09-29T10:46:49.835145Z"
    },
    "papermill": {
     "duration": 0.09807,
     "end_time": "2020-09-29T10:46:49.835779",
     "exception": false,
     "start_time": "2020-09-29T10:46:49.737709",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "vec_size = 50\n",
    "def get_model():\n",
    "    model = Sequential()\n",
    "    model.add(Embedding(vocab_size, vec_size, input_length=max_length))\n",
    "\n",
    "    model.add(Conv1D(32, 2, activation = 'relu'))\n",
    "    model.add(MaxPooling1D(2))\n",
    "    model.add(Dropout(0.2))\n",
    "\n",
    "    model.add(Conv1D(64, 3, activation = 'relu'))\n",
    "    model.add(MaxPooling1D(2))\n",
    "    model.add(Dropout(0.3))\n",
    "\n",
    "\n",
    "    model.add(Dense(128, activation='relu'))\n",
    "\n",
    "    model.add(GlobalMaxPooling1D())\n",
    "\n",
    "    model.add(Dense(y.shape[1], activation='softmax'))\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-09-29T10:46:50.014979Z",
     "iopub.status.busy": "2020-09-29T10:46:50.014243Z",
     "iopub.status.idle": "2020-09-29T10:46:50.076923Z",
     "shell.execute_reply": "2020-09-29T10:46:50.076323Z"
    },
    "papermill": {
     "duration": 0.154754,
     "end_time": "2020-09-29T10:46:50.077026",
     "exception": false,
     "start_time": "2020-09-29T10:46:49.922272",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from keras import backend as K\n",
    "def avg_jacard(y_true,y_pred):\n",
    "    jacard = K.sum(K.minimum(y_true,y_pred)) / K.sum(K.maximum(y_true,y_pred))\n",
    "    return K.mean(jacard)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-09-29T10:46:50.256531Z",
     "iopub.status.busy": "2020-09-29T10:46:50.255668Z",
     "iopub.status.idle": "2020-09-29T10:46:50.258256Z",
     "shell.execute_reply": "2020-09-29T10:46:50.258689Z"
    },
    "papermill": {
     "duration": 0.094169,
     "end_time": "2020-09-29T10:46:50.258813",
     "exception": false,
     "start_time": "2020-09-29T10:46:50.164644",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "early = EarlyStopping(\n",
    "    monitor='val_loss', min_delta=0, patience=0, verbose=0, mode='auto',\n",
    "    baseline=None, restore_best_weights=False\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-09-29T10:46:50.438101Z",
     "iopub.status.busy": "2020-09-29T10:46:50.437398Z",
     "iopub.status.idle": "2020-09-29T11:07:19.659356Z",
     "shell.execute_reply": "2020-09-29T11:07:19.659838Z"
    },
    "papermill": {
     "duration": 1229.31479,
     "end_time": "2020-09-29T11:07:19.659992",
     "exception": false,
     "start_time": "2020-09-29T10:46:50.345202",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "287/287 [==============================] - 121s 421ms/step - loss: 602.4941 - avg_jacard: 0.0347 - val_loss: 3994.8645 - val_avg_jacard: 0.0637\n",
      "Epoch 2/10\n",
      "287/287 [==============================] - 120s 418ms/step - loss: 62801.2539 - avg_jacard: 0.0349 - val_loss: 166750.3125 - val_avg_jacard: 0.0112\n",
      "Epoch 3/10\n",
      "287/287 [==============================] - 121s 421ms/step - loss: 544542.5000 - avg_jacard: 0.0349 - val_loss: 921084.1875 - val_avg_jacard: 0.0112\n",
      "Epoch 4/10\n",
      "287/287 [==============================] - 121s 421ms/step - loss: 2280405.0000 - avg_jacard: 0.0362 - val_loss: 2976119.0000 - val_avg_jacard: 0.0450\n",
      "Epoch 5/10\n",
      "287/287 [==============================] - 121s 423ms/step - loss: 6201521.0000 - avg_jacard: 0.0343 - val_loss: 6821151.0000 - val_avg_jacard: 0.0613\n",
      "Epoch 6/10\n",
      "287/287 [==============================] - 122s 425ms/step - loss: 13558538.0000 - avg_jacard: 0.0346 - val_loss: 18103442.0000 - val_avg_jacard: 0.0382\n",
      "Epoch 7/10\n",
      "287/287 [==============================] - 124s 433ms/step - loss: 23324090.0000 - avg_jacard: 0.0358 - val_loss: 25764380.0000 - val_avg_jacard: 0.0504\n",
      "Epoch 8/10\n",
      "287/287 [==============================] - 122s 427ms/step - loss: 40633096.0000 - avg_jacard: 0.0360 - val_loss: 39462668.0000 - val_avg_jacard: 0.0382\n",
      "Epoch 9/10\n",
      "287/287 [==============================] - 122s 427ms/step - loss: 63403876.0000 - avg_jacard: 0.0349 - val_loss: 82375560.0000 - val_avg_jacard: 0.0382\n",
      "Epoch 10/10\n",
      "287/287 [==============================] - 122s 426ms/step - loss: 87818728.0000 - avg_jacard: 0.0355 - val_loss: 125060400.0000 - val_avg_jacard: 0.0382\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7ff9235e8090>"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = get_model()\n",
    "model.compile(optimizer='adam', loss = 'categorical_crossentropy', metrics = [avg_jacard])\n",
    "model.fit(X_train, y_train, epochs = 10, validation_data = (X_test, y_test), batch_size = 128)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 1.085032,
     "end_time": "2020-09-29T11:07:21.819901",
     "exception": false,
     "start_time": "2020-09-29T11:07:20.734869",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Model Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-09-29T11:07:23.988056Z",
     "iopub.status.busy": "2020-09-29T11:07:23.985990Z",
     "iopub.status.idle": "2020-09-29T11:07:23.988716Z",
     "shell.execute_reply": "2020-09-29T11:07:23.989194Z"
    },
    "papermill": {
     "duration": 1.070219,
     "end_time": "2020-09-29T11:07:23.989319",
     "exception": false,
     "start_time": "2020-09-29T11:07:22.919100",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "x = \"what is the best way to copy a database i always create a new empointy database afeatyouringer that backyoup and restore of the existing database into it byout is this really the best way as it seemiss very error prone and over compli seeated for me\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-09-29T11:07:26.486654Z",
     "iopub.status.busy": "2020-09-29T11:07:26.484828Z",
     "iopub.status.idle": "2020-09-29T11:07:26.490304Z",
     "shell.execute_reply": "2020-09-29T11:07:26.489394Z"
    },
    "papermill": {
     "duration": 1.089586,
     "end_time": "2020-09-29T11:07:26.490493",
     "exception": false,
     "start_time": "2020-09-29T11:07:25.400907",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def get_clean(x):\n",
    "    x = str(x).lower().replace('\\\\', '').replace('_', ' ')\n",
    "    x = ps.cont_exp(x)\n",
    "    x = ps.remove_emails(x)\n",
    "    x = ps.remove_urls(x)\n",
    "    x = ps.remove_html_tags(x)\n",
    "    x = ps.remove_accented_chars(x)\n",
    "    x = ps.remove_special_chars(x)\n",
    "    x = re.sub(\"(.)\\\\1{2,}\", \"\\\\1\", x)\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-09-29T11:07:28.612969Z",
     "iopub.status.busy": "2020-09-29T11:07:28.611105Z",
     "iopub.status.idle": "2020-09-29T11:07:28.613701Z",
     "shell.execute_reply": "2020-09-29T11:07:28.614181Z"
    },
    "papermill": {
     "duration": 1.062124,
     "end_time": "2020-09-29T11:07:28.614306",
     "exception": false,
     "start_time": "2020-09-29T11:07:27.552182",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def get_encoded(x):\n",
    "    x = token.texts_to_sequences([x])\n",
    "    x = pad_sequences(x, maxlen=max_length, padding = 'post')\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-09-29T11:07:30.741337Z",
     "iopub.status.busy": "2020-09-29T11:07:30.739633Z",
     "iopub.status.idle": "2020-09-29T11:07:30.742070Z",
     "shell.execute_reply": "2020-09-29T11:07:30.742550Z"
    },
    "papermill": {
     "duration": 1.072828,
     "end_time": "2020-09-29T11:07:30.742673",
     "exception": false,
     "start_time": "2020-09-29T11:07:29.669845",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "coded = get_encoded(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-09-29T11:07:32.928269Z",
     "iopub.status.busy": "2020-09-29T11:07:32.927353Z",
     "iopub.status.idle": "2020-09-29T11:07:33.111998Z",
     "shell.execute_reply": "2020-09-29T11:07:33.111220Z"
    },
    "papermill": {
     "duration": 1.262242,
     "end_time": "2020-09-29T11:07:33.112136",
     "exception": false,
     "start_time": "2020-09-29T11:07:31.849894",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([6])"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.predict_classes(coded)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-09-29T11:07:35.256887Z",
     "iopub.status.busy": "2020-09-29T11:07:35.255972Z",
     "iopub.status.idle": "2020-09-29T11:07:35.298975Z",
     "shell.execute_reply": "2020-09-29T11:07:35.298478Z"
    },
    "papermill": {
     "duration": 1.125228,
     "end_time": "2020-09-29T11:07:35.299158",
     "exception": false,
     "start_time": "2020-09-29T11:07:34.173930",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['c++'], dtype=object)"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "multilabel.classes_[model.predict_classes(coded)]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "papermill": {
   "duration": 1540.406199,
   "end_time": "2020-09-29T11:07:37.940437",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2020-09-29T10:41:57.534238",
   "version": "2.1.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}